source: main/trunk/gli/src/org/greenstone/gatherer/gui/FormatConversionDialog.java@ 29012

package org.greenstone.gatherer.gui;

import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
//import static org.greenstone.gatherer.cdm.Format4gs3Manager.NumberedJTextArea;
import org.greenstone.gatherer.cdm.CollectionConfigXMLReadWrite;
import org.greenstone.gatherer.util.Codec;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.util.XMLTools;

import org.fife.ui.rsyntaxtextarea.*;
import org.w3c.dom.*;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;

//import java.awt.Dimension;
import java.awt.*;
import java.awt.event.*;
import java.util.*;
import javax.swing.*;
import javax.swing.border.*;
import javax.swing.event.*;
import javax.swing.undo.*;


// TO DO:
// + StreamGobblers: http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html
// Need to test on Windows: the Ctrl-D/Ctrl-Z sent to formatconverter.exe. May not need to do anything special for Windows now that this code is using StreamGobblers to process the in/out streams of the process
// Need to use dictionary for labels
// convertFormat() for remote GS
// + HTML tidy (put all "<br/>" back to "<br>" in luce/etc/collectionConfig.xml and run again)
// Help tooltips on buttons
// + Undo, Redo buttons
// X Split class into dialog/widgets and data processing?
// http://www.java2s.com/Tutorial/Java/0240__Swing/SettingJOptionPanebuttonlabelstoFrench.htm

public class FormatConversionDialog extends ModalDialog
{
    public static final String GSF_FORMAT_GS2_TAG = "gsf:format-gs2";
    private static final String GSF_GS3_ROOT_TAG = "gsf:gs3-root";

    // The cmdline programs launched by this dialog
    private static final int XMLTIDY = 0;
    private static final int FORMATCONVERTER = 1;
    // Online HTML tidy for learning usage: http://infohound.net/tidy/
    private static final String[] xmltidy_cmd_args = {"tidy", "-config", Configuration.getGS3BinPath() + "tidyconfig.cfg", "-utf8", "-wrap", "0", "-raw", "-q"}; // {"tidy", "-xml", "-utf8"};
    private static final String[] formatconverter_cmd_args = {Configuration.getGS3BinPath() + "formatconverter", "--silent"};

    private static final Dimension SIZE = new Dimension(640,480);

    // the important member variables
    private File collect_cfg_file = null;
    private Document xml_file_doc = null;
    private NodeList gsf_format_gs2_list = null;    

    private int current_index = -1;
    private int dlgResult = OpenCollectionDialog.OK_OPTION;
    private int process_exitValue = -1;
    
    // widgets
    final String WARNING_TITLE = Dictionary.get("General.Warning") 
    + ": " + Dictionary.get("FormatConversionDialog.Invalid_XML")
    + " " + Dictionary.get("FormatConversionDialog.Invalid_XML_Warning_Title");

    private static final Border EMPTYBORDER = new EmptyBorder(5, 5, 5, 5);
    private JLabel section_label = null;
    private GLIButton cancel_button = null;
    private GLIButton next_button = null;
    private GLIButton accept_all_button = null;
    private GLIButton undo_button = null;
    private GLIButton redo_button = null;
    private GLIButton htmltidy_button = null;
    private GLIButton xmltidy_button = null;
    private NumberedJTextArea gs2_textarea = null;
    private NumberedJTextArea gs3_textarea = null;
    private JLabel count_label = null;
    private JLabel statusbar = null;
    private UndoManager undoManager;

    public FormatConversionDialog (File collect_cfg_file, Document xml_file_doc, NodeList gsf_format_gs2_list) {
    super(Gatherer.g_man, "", true);

    this.collect_cfg_file = collect_cfg_file;   
    this.xml_file_doc = xml_file_doc;
    this.gsf_format_gs2_list = gsf_format_gs2_list;

    this.setComponentOrientation(Dictionary.getOrientation());
    setSize(SIZE);
    setTitle(Dictionary.get(Dictionary.get("FormatConversionDialog.Title")));
    this.addWindowListener(new WindowClosingListener()); // the dialog's top right close button should behave as a Cancel press
    setDefaultCloseOperation(DISPOSE_ON_CLOSE);
    

    JPanel midbutton_panel = new JPanel(); // FlowLayout by default in a JPanel
    midbutton_panel.setComponentOrientation(Dictionary.getOrientation());
    JButton reconvert_button = new GLIButton(Dictionary.get("FormatConversionDialog.Reconvert"), Dictionary.get("FormatConversionDialog.Reconvert_Tooltip"));
    midbutton_panel.add(reconvert_button);
    reconvert_button.addActionListener(new ReconvertListener());
    reconvert_button.setAlignmentY(Component.CENTER_ALIGNMENT);

    section_label = new JLabel("Section Label Goes Here"); // title will be replaced, don't need to translate
    section_label.setBorder(EMPTYBORDER);
    section_label.setComponentOrientation(Dictionary.getOrientation());

    JPanel button1_panel = new JPanel();
    button1_panel.setComponentOrientation(Dictionary.getOrientation());
    undo_button = new GLIButton(Dictionary.get("General.Undo"), Dictionary.get("General.Undo_Tooltip"));
    undo_button.addActionListener(new UndoListener());
    undo_button.setEnabled(false);

    redo_button = new GLIButton(Dictionary.get("General.Redo"), Dictionary.get("General.Redo_Tooltip"));
    redo_button.addActionListener(new RedoListener());
    redo_button.setEnabled(false);

    xmltidy_button = new GLIButton(Dictionary.get("FormatConversionDialog.XHTML_Tidy"), Dictionary.get("FormatConversionDialog.XHTML_Tidy_Tooltip"));
    xmltidy_button.addActionListener(new XMLTidyButtonListener());

    button1_panel.add(undo_button);
    button1_panel.add(redo_button);
    button1_panel.add(xmltidy_button);

    JPanel button_panel = new JPanel(new FlowLayout());
    button_panel.setComponentOrientation(Dictionary.getOrientation());
    count_label = new JLabel("<count>"); // placeholder text for numbers, no need to translate
    cancel_button = new GLIButton(Dictionary.get("General.Cancel"), Dictionary.get("FormatConversionDialog.Cancel_Tooltip"));
    next_button = new GLIButton(Dictionary.get("FormatConversionDialog.Next"), Dictionary.get("FormatConversionDialog.Next_Tooltip"));
    accept_all_button = new GLIButton(Dictionary.get("FormatConversionDialog.Accept_All"), Dictionary.get("FormatConversionDialog.Accept_All_Tooltip"));
    cancel_button.addActionListener(new CancelButtonListener());
    next_button.addActionListener(new NextButtonListener());
    accept_all_button.addActionListener(new AcceptAllButtonListener());
    button_panel.add(cancel_button);
    button_panel.add(next_button);
    button_panel.add(accept_all_button);
    button_panel.add(count_label);


    // The undoable text areas. Adding the UndoableEditListener has to come after instantiation
    // of the undo and redo buttons, since the listener expects these buttons to already exist
    undoManager = new UndoManager();

    gs2_textarea = new NumberedJTextArea();
    gs3_textarea = new NumberedJTextArea();

    CustomUndoableEditListener customUndoableEditListener = new CustomUndoableEditListener();
    gs2_textarea.getDocument().addUndoableEditListener(customUndoableEditListener);
    gs3_textarea.getDocument().addUndoableEditListener(customUndoableEditListener);

    initTextArea(gs2_textarea);
    initTextArea(gs3_textarea);
    gs2_textarea.setToolTipText(Dictionary.get("FormatConversionDialog.GS2_Text_Tooltip"));
    gs3_textarea.setToolTipText(Dictionary.get("FormatConversionDialog.GS3_Text_Tooltip"));


    JPanel centre_panel = new JPanel();
    centre_panel.setLayout(new BoxLayout(centre_panel, BoxLayout.Y_AXIS));
    centre_panel.setComponentOrientation(Dictionary.getOrientation());
    centre_panel.setBorder(EMPTYBORDER);
    centre_panel.add(new JScrollPane(gs2_textarea));
    centre_panel.add(midbutton_panel);
    centre_panel.add(new JScrollPane(gs3_textarea));

    JPanel bottom_panel = new JPanel();
    bottom_panel.setLayout(new BoxLayout(bottom_panel, BoxLayout.Y_AXIS));
    bottom_panel.setComponentOrientation(Dictionary.getOrientation());
    bottom_panel.setBorder(EMPTYBORDER);

    bottom_panel.add(button1_panel);
    bottom_panel.add(button_panel);
    statusbar = new JLabel("");
    statusbar.setBorder(EMPTYBORDER);
    // http://stackoverflow.com/questions/2560784/how-to-center-elements-in-the-boxlayout-using-center-of-the-element
    statusbar.setAlignmentX(Component.CENTER_ALIGNMENT);
    bottom_panel.add(statusbar);

    // add all the widgets to the contentpane
    JPanel content_pane = (JPanel) getContentPane();
    content_pane.setComponentOrientation(Dictionary.getOrientation());
    content_pane.setLayout(new BorderLayout());
    //content_pane.setLayout(new GridLayout(7,1));
    //content_pane.setLayout(new BoxLayout(content_pane, BoxLayout.Y_AXIS));
    //content_pane.setBorder(EMPTYBORDER);

    content_pane.add(section_label, BorderLayout.NORTH);
    content_pane.add(centre_panel, BorderLayout.CENTER);
    content_pane.add(bottom_panel, BorderLayout.SOUTH);
    

    // Final dialog setup & positioning.
    getRootPane().setDefaultButton(next_button);
    Dimension screen_size = Configuration.screen_size;
    setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
    screen_size = null;
    //setVisible(true);
    }


    public static void initTextArea(NumberedJTextArea editor_textarea) {
    /* Fields specific to RSyntaxQuery inherited class */
    editor_textarea.setSyntaxEditingStyle(SyntaxConstants.SYNTAX_STYLE_XML);
    editor_textarea.setBracketMatchingEnabled(true);
    editor_textarea.setAnimateBracketMatching(true);
    editor_textarea.setAntiAliasingEnabled(true);
    editor_textarea.setAutoIndentEnabled(true);
    editor_textarea.setPaintMarkOccurrencesBorder(false);
    
    /* Standard fields to JTextArea */
    editor_textarea.setOpaque(false);
    editor_textarea.setBackground(Configuration.getColor("coloring.editable_background", false));
    editor_textarea.setCaretPosition(0);
    editor_textarea.setLineWrap(true);
    editor_textarea.setRows(11);
    editor_textarea.setWrapStyleWord(false);    
    }

    public int getDialogResult() {
    return dlgResult; // OK_OPTION or CANCEL_OPTION
    }
    
    //*************************PROCESSING FUNCTIONS***************************//
    public static int checkForGS2FormatStatements(File collect_cfg_file) {

    if(Gatherer.GS3 && collect_cfg_file.getAbsolutePath().endsWith(".xml")) {

        //System.err.println("*** Opening an xml config file");
        
        Document xml_file_doc = XMLTools.parseXMLFile(collect_cfg_file);
        Element root = xml_file_doc.getDocumentElement();

        // check if there are any <gsf:format-gs2 /> elements. If there are, then may need to process them
        
        //NodeList gsf_format_gs2_list = root.getElementsByTagNameNS("gsf", "format-gs2");
        NodeList gsf_format_gs2_list = root.getElementsByTagName(FormatConversionDialog.GSF_FORMAT_GS2_TAG);
        if(gsf_format_gs2_list != null && gsf_format_gs2_list.getLength() > 0) {

        // Sample the first of the <gsf:gs2-format/> elements to
        // check we don't have any CDataSections in the <gsf:gs2-format/> elements
        // If the first <gsf:gs2-format/> has a CDataSection, it means we've already 
        // converted it to GS3 format statements during an earlier GLI session.

        Node gs2format = gsf_format_gs2_list.item(0);
        //gs2format.normalize();
        NodeList children = gs2format.getChildNodes();

        for(int i = 0; i < children.getLength(); i++) {
            Node child = children.item(i);
            if(child.getNodeType() == Node.CDATA_SECTION_NODE) {    
            // there are GS2 statements in col config, but they've already been converted to GS3
            // can open the collection without going through the FormatConversionDialog
            return OpenCollectionDialog.OK_OPTION; 
            }
        }
        System.err.println("*** Found GS2 format statements in config file to be converted to GS3.");

        
        // if remote GS3, do we open the collection with the html-encoded GS2 format statements
        // or do we not allow the remote user to open such a collection at all?
        // For now we allow them to open it, but print a warning that conversions are not possible.
        
        if(Gatherer.isGsdlRemote) { // remote GS3
            System.err.println("*** Cannot convert GS2 collections from a remote GS3 server."); 
            return OpenCollectionDialog.OK_OPTION;
        }

        // If we get here, it means there were no CDataSections in the first (any) <gsf:gs2-format/>
        // This means it's the first time the GS2 collection is being opened in GLI 
        // Open the FormatCconversionDialog and convert the gs2 statements to gs3:
        FormatConversionDialog formatconversionDlg = new FormatConversionDialog(collect_cfg_file, xml_file_doc, gsf_format_gs2_list);
        formatconversionDlg.convertGS2FormatStatements();
        return formatconversionDlg.getDialogResult();
        
        }
    }   
    return OpenCollectionDialog.OK_OPTION; // no GS2 statements in col config, and can open the collection
    }


    /** 
     * runInteractiveProgram() runs a cmdline program that reads from stdinput 
     * until Ctrl-D is encountered. It outputs the result to stdout.
     *
     * The cmdline programs HTML Tidy and FormatConverter both behave the same way:
     * When the formatconverter binary is run in silent mode, it expects input
     * followed by a newline and then EOF (or if no newline, then 2 EOFs) 
     * which is Ctrl-D on Linux/Mac and Ctrl-Z on Windows. 
     * Then the cmdline program exits by printing the result of the conversion.
     *
     * Explicitly sending EOFs from java is no longer necessary, as the SendStreamGobbler 
     * Thread takes care of closing the process stream.
     *
     * HTMLTidy returns 0 if no warnings or errors, 1 if just Warnings, 2 if Errors (failure)
     * http://sourceforge.net/p/tidy/mailman/tidy-develop/thread/[email protected]/
     *
     * This code uses the StreamGobbler classes based on
     * http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
    */    
    public String runInteractiveProgram(int program, String inputstr) {
    String outputstr = "";
    String[] command_args;
    process_exitValue = -1;

    if(program == XMLTIDY) {
        command_args = xmltidy_cmd_args;
    } else if(program == FORMATCONVERTER) {
        command_args = formatconverter_cmd_args;
    } else { // unknown command
        return outputstr;
    }

    // Generate the formatconverter command     
    /*if (Gatherer.isGsdlRemote) {
      
      }*/

    try {       
        
        Runtime rt = Runtime.getRuntime();
        Process prcs = null;
        
        prcs = rt.exec(command_args);

        // send inputstr to process
        SendStreamGobbler inputGobbler = new SendStreamGobbler(prcs.getOutputStream(), inputstr);
        
        // monitor for any error messages
            ReadStreamGobbler errorGobbler = new ReadStreamGobbler(prcs.getErrorStream(), true);
            
            // monitor for the expected output line(s)
            ReadStreamGobbler outputGobbler = new ReadStreamGobbler(prcs.getInputStream());

            // kick them off
            inputGobbler.start();
            errorGobbler.start();
            outputGobbler.start();
                                    
            // any error???
            process_exitValue = prcs.waitFor();
            //System.out.println("ExitValue: " + exitVal); 

        // From the comments of 
        // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
        // To avoid running into nondeterministic failures to get the process output
        // if there's no waiting for the threads, call join() on each Thread (StreamGobbler) object:
        outputGobbler.join();
        errorGobbler.join();
        inputGobbler.join();
        
        outputstr = outputGobbler.getOutput();
        String errmsg = errorGobbler.getOutput();
        if(!errmsg.equals("")) {
        System.err.println("*** Process errorstream: \n" + errmsg + "\n****");
        }
       
    } catch(IOException ioe) {
        System.err.println("IOexception " + ioe.getMessage());
        //ioe.printStackTrace();
    } catch(InterruptedException ie) {
        System.err.println("Process InterruptedException " + ie.getMessage());
        //ie.printStackTrace();
    }

    return outputstr;
    }


    // http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html
    // http://stackoverflow.com/questions/481446/throws-exception-in-finally-blocks
    private void closeResource(Closeable resourceHandle) {
    try {
        if(resourceHandle != null) {
        resourceHandle.close();
        resourceHandle = null;
        }
    } catch(Exception e) {
        System.err.println("Exception closing resource: " + e.getMessage());
        e.printStackTrace();
    }
    }

    public void convertGS2FormatStatements() {
                
    // at this point, we know there are one or more <gsf:format-gs2 /> elements
    // process each of them as follows: unescape, then call formatconverter, then call html tidy on it
    //NodeList gsf_format_gs2_list = root.getElementsByTagNameNS("gsf", "format-gs2");

    int len = gsf_format_gs2_list.getLength();

    for(int i = 0; i < len; i++) {
            
        Element gs2format = (Element)gsf_format_gs2_list.item(i);
        String gs2formatstr = XMLTools.getElementTextValue(gs2format); // seems to already unescape the html entities
        //gs2formatstr = Codec.transform(gs2formatstr, Codec.ESCAPEDHTML_TO_UNESCAPED);
        
        processFormatStatement(i, gs2formatstr);
    } 

    increment(); // modifies the textareas to initialise them
    setVisible(true);
    }


    private String processFormatStatement(int i, String gs2formatstr) {
    String errorMsg = "";

    boolean startsWithTableCell = (gs2formatstr.toLowerCase().startsWith("<td")) ? true : false;

    //System.err.println("*** Found: " + gs2formatstr);

    String gs3formatstr = runInteractiveProgram(FORMATCONVERTER, gs2formatstr);
    gs3formatstr = gs3formatstr.replace("> <", "><");

    //System.err.println("*** Format is now: " + gs3formatstr);

    String gs3formatstr_notags = gs3formatstr;
    gs3formatstr = addSurroundingTags(gs3formatstr);


    String validationMsg = XMLTools.parseDOM(gs3formatstr);
    if(!validationMsg.startsWith(XMLTools.WELLFORMED)) {
        // Run Html Tidy in XML mode
        System.err.println("*** Needing to run HTML Tidy on: ");
        System.err.println(gs3formatstr_notags);        


        // HTMLTidy returns 0 if no warnings or errors, 1 if just Warnings, 2 if Errors (failure)
        // http://sourceforge.net/p/tidy/mailman/tidy-develop/thread/[email protected]/
        String htmltidy_string = runInteractiveProgram(XMLTIDY, gs3formatstr_notags);//removeSurroundingTags(gs3formatstr));

        if(process_exitValue >= 2) {
        System.err.println("@@@ Process exit value: " + process_exitValue);
        errorMsg = Dictionary.get("FormatConversionDialog.Tidy_Failed") 
            + " " + Dictionary.get("FormatConversionDialog.XML_Still_Invalid");
        } else {
        errorMsg = "";

        gs3formatstr_notags = htmltidy_string;

        gs3formatstr_notags = removeHTMLTags(i, gs3formatstr_notags, startsWithTableCell);
        gs3formatstr = addSurroundingTags(gs3formatstr_notags); 
        
        // Having applied html-tidy, setGS3Format() will return true if it finally parsed
        }
    }
    
    // For now, assume HTML Tidy has worked and that the gs3format has now been parsed successfully

    boolean parsed = setGS3Format(i, gs3formatstr); // will parse the gs3formatstr into a DOM object
    if(!parsed && errorMsg.equals("")) {
        errorMsg = Dictionary.get("FormatConversionDialog.Tidy_Done")
        + " " + Dictionary.get("FormatConversionDialog.XML_Still_Invalid");
    }   
    return errorMsg;    
    //return gs3formatstr;
    }

    // HTML tidy adds entire HTML tags around a single format statement. This method removes it.
    private String removeHTMLTags(int i, String gs3formatstr_notags, boolean startsWithTD) {

    // if it's a VList classifier <gsf:template match="documentNode|classifierNode"/>
    // and the gs2 format statement starts with a <td>, 
    // then remove up to and including the outermost <tr>, 
    // else remove up to and including the <body> tag
    
    String removeOuterTag = "body>";
    Element parent = (Element)getParentNode(i); // gets parent of GS2format: <gsf:template>
    if(parent.hasAttribute("match") && (!parent.hasAttribute("mode") || !parent.getAttribute("mode").equals("horizontal"))) {
        if(startsWithTD) {
        removeOuterTag = "tr>"; // remove the outermost <tr> cell HTML tidy added around the tablecell
        }
    }

    // <unwanted>
    // <removeThisOuterTag>
    // lines we want
    // </removeThisOuterTag>
    // <unwanted>
    
    int end = gs3formatstr_notags.indexOf(removeOuterTag);
    if(end != -1) {
        gs3formatstr_notags = gs3formatstr_notags.substring(end+removeOuterTag.length());
    }
    int start = gs3formatstr_notags.lastIndexOf("</"+removeOuterTag); //closing tag
    if(start != -1) {
        gs3formatstr_notags = gs3formatstr_notags.substring(0, start);
    }

    //System.err.println("@@@@ " + startsWithTD + " - TAG: " + removeOuterTag + " - AFTER REMOVING TAGS:\n" + gs3formatstr_notags);

    return gs3formatstr_notags; 
    }


    //**************** ACCESS FUNCTIONS ***************//

    // gs2 format text is the text string that goes into <gsf:format-gs2/>: <gsf:format-gs2>text</gsf:format-gs2>
    private void setGS2Format(int i, String text) {
    XMLTools.setElementTextValue(getGS2Format(i), text);
    }
    
    // gs3FormatStr represents DOM, and must be parsed and appended as sibling to the gs2format element
    // as <gsf:gs3-root/>. If it fails to parse, nest it in a CDATA element of <gsf:gs3-root/>
    private boolean setGS3Format(int i, String gs3formatstr) {

    Document ownerDoc = getGS2Format(i).getOwnerDocument();
    Node gs3format = null;
    boolean parse_success = false;

    
    String validationMsg = XMLTools.parseDOM(gs3formatstr);
    if(!validationMsg.startsWith(XMLTools.WELLFORMED)) {        
        // silently add the gs3formatstr in a CDATA block into the root

        gs3formatstr = removeSurroundingTags(gs3formatstr);
        gs3format = ownerDoc.createElement(GSF_GS3_ROOT_TAG);       
        Node cdataSection = ownerDoc.createCDATASection(gs3formatstr);
        gs3format.appendChild(cdataSection);
        parse_success = false;
    }
    else {

        // parse DOM into XML
        Document doc = XMLTools.getDOM(gs3formatstr);
        Element root = doc.getDocumentElement();
        gs3format = ownerDoc.importNode(root, true); // an element
        parse_success = true;

        //  System.err.println("@@@@ ROOT:\n" + XMLTools.xmlNodeToString(root));
    }

    // add gs3format element as sibling to gs2format element
    Element oldgs3format = getGS3Format(i);
    if(oldgs3format == null) {      
        getParentNode(i).appendChild(gs3format);
        //      System.err.println("@@@ APPEND");
    } else {
        // replace the existing
        getParentNode(i).replaceChild(gs3format, oldgs3format);
        //      System.err.println("@@@ REPLACE");
    }

    //http://stackoverflow.com/questions/12524727/remove-empty-nodes-from-a-xml-recursively

    return parse_success;
    }

    private Node getParentNode(int i) {
    return gsf_format_gs2_list.item(i).getParentNode();
    }

    private Element getGS2Format(int i) {
    return (Element)gsf_format_gs2_list.item(i);
    }

    private String getGS2FormatString(int i) {
    return XMLTools.getElementTextValue(getGS2Format(i));
    }

    private Element getGS3Format(int i) {
    Element parent  = (Element)getParentNode(i);
    NodeList nl = parent.getElementsByTagName(GSF_GS3_ROOT_TAG);
    if(nl.getLength() > 0) {
        return (Element)nl.item(0);
    }
    return null;
    }

    private String getGS3FormatString(int i) {  
    Element gs3format = getGS3Format(i);
    if(gs3format == null) {
        return "";
    } else {
        
        // if any child is a CData section, return its text as-is. There will be no indenting
        NodeList children = gs3format.getChildNodes();
        for(int j = 0 ; j < children.getLength(); j++) {
        if(children.item(j).getNodeType() == Node.CDATA_SECTION_NODE) {         
            return children.item(j).getNodeValue(); // content of CDataSection
        }
        }
        
        // else we have proper nodes, return indented string
        StringBuffer sb = new StringBuffer();
        XMLTools.xmlNodeToString(sb, gs3format, true, "  ", 0);
        return sb.toString();
    }
    }


    private String getLabel(int i) {
    String label = "";

    // Given XML of the form:
    // <browse|search>
    //   <format>
    //     <gsf:template match="documentNode|classifierNode" [mode=horizontal]>
    //       <gsf-format:gs2 />
    //       <gs3format/>
    //   </format>
    // </browse|search>

    // Want to return the label: "browse|search > documentNode|classifierNode [> horizontal]"

    Element parent = (Element)getParentNode(i); // gets parent of GS2format: <gsf:template>
    label = parent.getAttribute("match"); //e.g. documentNode, classifierNode

    if(parent.hasAttribute("mode")) { // e.g. classifierNode mode=horizontal
        label = label + " > " + parent.getAttribute("mode");
    }

    Element ancestor = (Element) parent.getParentNode().getParentNode(); // ancestor: <browse>|<search>
    label = ancestor.getTagName() + " > " + label;

    return label;
    }

    // gs2 format statements have spaces instead of newlines. For display, replace with newlines
    private String makeLines(String gs2formatstr) {
    return gs2formatstr.replaceAll(">\\s+<", ">\n<");
    }
   
    private String singleLine(String gs2formatstr) {
    return gs2formatstr.replace(">\n<", "> <"); // put the spaces back
    }

    private String removeSurroundingTags(String xml)
    {
    //return xml.replace("<"+GSF_GS3_ROOT_TAG+" xmlns:gsf=\"http://www.greenstone.org/greenstone3/schema/ConfigFormat\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n", "").replace("\n</"+GSF_GS3_ROOT_TAG+">", "");//.trim(); 
    return xml.replaceAll("<"+GSF_GS3_ROOT_TAG+" xmlns:gsf=(\"|\')http://www.greenstone.org/greenstone3/schema/ConfigFormat(\"|\') xmlns:xsl=(\"|\')http://www.w3.org/1999/XSL/Transform(\"|\')>\n?", "").replaceAll("\n?</"+GSF_GS3_ROOT_TAG+">", "");//.trim();
    }

    private String addSurroundingTags(String gs3formatstr) {
    return "<"+GSF_GS3_ROOT_TAG+" xmlns:gsf='http://www.greenstone.org/greenstone3/schema/ConfigFormat' xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>"+gs3formatstr+"</"+GSF_GS3_ROOT_TAG+">";
    }

    //*************************FUNCTIONS THAT INTERACT WITH WIDGETS************************//
    // increment() loads the next values into the dialog
    private boolean increment() {
    current_index++;
    section_label.setText ( getLabel(current_index) );
    gs2_textarea.setText( makeLines(getGS2FormatString(current_index)) );
    gs3_textarea.setText( removeSurroundingTags(getGS3FormatString(current_index)) );
    statusbar.setText("");

    // as we're on a new screen of dialog, need to clear all undo/redo history
    undoManager.discardAllEdits();
    undo_button.setEnabled(false);
    redo_button.setEnabled(false);

    int len = gsf_format_gs2_list.getLength();
    count_label.setText((current_index+1) + " / " + len);
    if((current_index+1) == len) {
        return false;
    } else {
        return true;
    }   
    }


    private void setStatus(String msg) {
    statusbar.setText(msg);
    }
    private void setErrorStatus(String msg) {
    statusbar.setBackground(Color.red);
    statusbar.setText(msg);
    }


    public void dispose() {
    //System.err.println("@@@@ DIALOG CLOSING!");
    if(dlgResult != OpenCollectionDialog.CANCEL_OPTION) {
        // Need to remove the <gsf:gs3-root/> siblings of all <gsf:format-gs2/>
        // Then, get the children of each <gsf:gs3-root/> and add these as siblings of <gsf:format-gs2/>
        
        
        int len = gsf_format_gs2_list.getLength();
        for(int k=len-1; k >= 0; k--) {
        Element parent = (Element)getParentNode(k);     
        
        //parent.normalize();

        NodeList children = parent.getChildNodes();

        // now have to loop/remove nodes in reverse order, since the following loop 
        // modifies the very nodelist we're looping over by removing nodes from it

        int numChildren = children.getLength()-1;

        for(int i=numChildren; i >= 0; i--) {
        //for(int i = 0; i < children.getLength(); i++) {
            Node child = children.item(i);

            if(child.getNodeName().equals(GSF_FORMAT_GS2_TAG)) { 
            // if we're dealing with gs2-format-stmts, put their textnode contents in CData sections
            // http://www.w3schools.com/xml/xml_cdata.asp
            // This avoids having to look at html-encoded gs2-format tags in the Format pane

            Element gs2format = (Element)child;
            String gs2formatstr = XMLTools.getElementTextValue(gs2format);
            gs2formatstr = Codec.transform(gs2formatstr, Codec.ESCAPEDHTML_TO_UNESCAPED);
            
            Node textNode = XMLTools.getNodeTextNode(gs2format);
            Node cdataSection = gs2format.getOwnerDocument().createCDATASection(gs2formatstr); 
            gs2format.replaceChild(cdataSection, textNode);
            } 
            else if(child.getNodeName().equals(GSF_GS3_ROOT_TAG)) { 

            // remove GS3 node and append its children to the parent in its place
            // the <gsf:gs3-root /> elements wouldn't be in the xml_file_doc DOM tree
            // unless they were valid XML, so don't need to check for validity here
            
            Node gs3root = child;
            NodeList gs3_format_lines = gs3root.getChildNodes();

            for(int j = 0; j < gs3_format_lines.getLength(); j++) {
                Node duplicate = gs3_format_lines.item(j).cloneNode(true);
                parent.appendChild(duplicate);              
            }
            gs3root = parent.removeChild(gs3root);
            gs3root = null; // finished processing

            } // else - skip all nodes other than <gsf:format-gs2/> and <gsf:gs3-root/>
        }
        }
 
        Element root = xml_file_doc.getDocumentElement();
        //System.err.println("### XML file to write out:\n" + XMLTools.xmlNodeToString(root));
        
        // Finally, write out the collection xml file
        String[] nonEscapingTagNames = { StaticStrings.FORMAT_STR, StaticStrings.DISPLAYITEM_STR };
        XMLTools.writeXMLFile(collect_cfg_file, xml_file_doc, nonEscapingTagNames);
        
    }
    super.dispose();
    
    }

    //******************INNER CLASSES including LISTENERS and STREAMGOBBLERS****************//

    /**
     * A textarea with the line number next to each line of the text
     */
    public class NumberedJTextArea extends RSyntaxTextArea /* JTextArea */
    {
    public void paintComponent(Graphics g)
    {
        Insets insets = getInsets();
        Rectangle rectangle = g.getClipBounds();
        g.setColor(Color.white);
        g.fillRect(rectangle.x, rectangle.y, rectangle.width, rectangle.height);
        
        super.paintComponent(g);
        
        if (rectangle.x < insets.left)
        {
            FontMetrics font_metrics = g.getFontMetrics();
            int font_height = font_metrics.getHeight();
            int y = font_metrics.getAscent() + insets.top;
            int line_number_start_point = ((rectangle.y + insets.top) / font_height) + 1;
            if (y < rectangle.y)
            {
                y = line_number_start_point * font_height - (font_height - font_metrics.getAscent());
            }
            int y_axis_end_point = y + rectangle.height + font_height;
            int x_axis_start_point = insets.left;
            x_axis_start_point -= getFontMetrics(getFont()).stringWidth(Math.max(getRows(), getLineCount() + 1) + " ");
            if (!this.getText().trim().equals(""))
            {
                g.setColor(Color.DARK_GRAY);
            }
            else
            {
                g.setColor(Color.white);
            }
            int length = ("" + Math.max(getRows(), getLineCount() + 1)).length();
            while (y < y_axis_end_point)
            {
                g.drawString(line_number_start_point + "  ", x_axis_start_point, y);
                y += font_height;
                line_number_start_point++;
            }
        }
    }
    
    
    public Insets getInsets()
    {
        Insets insets = super.getInsets(new Insets(0, 0, 0, 0));
        insets.left += getFontMetrics(getFont()).stringWidth(Math.max(getRows(), getLineCount() + 1) + " ");
        return insets;
    }
    }

    // windowClosing() is called when the user presses the top-right close button the dialog
    // this means the user wanted to cancel out of the entire Format Conversion Wizard.
    private class WindowClosingListener extends WindowAdapter {
    public void windowClosing(WindowEvent e) {      
        dlgResult = OpenCollectionDialog.CANCEL_OPTION;
    }
    }

    private class ReconvertListener implements ActionListener {
    public void actionPerformed(ActionEvent e) {
        String gs2formatstr = singleLine(gs2_textarea.getText());
        String errorMsg = processFormatStatement(current_index, gs2formatstr);
        gs3_textarea.setText( removeSurroundingTags(getGS3FormatString(current_index)) );
        if(!errorMsg.equals("")) {
        setErrorStatus(errorMsg);
        } else {
        statusbar.setText("");
        }
    }
    }

    
    private class NextButtonListener implements ActionListener {
    public void actionPerformed(ActionEvent e) {
        //statusbar.setText("");

        // check if the GS3 format statement is valid XML before storing. If not, let user to decide 
        // whether they want to re-edit it or if they want to keep it as-is, in which case it needs
        // to be stored as CDATA, which will make it an inactive format statement. 
        // See http://www.w3schools.com/xml/xml_cdata.asp
        // setGS3Format() already stores invalidXML as CDATA.
        
        // Check if the GS3 format statement is valid XML before storing. If not, let the
        // user to decide whether they want to re-edit it or store it as-is and continue        

        // user okay-ed the lines currently displayed, store them
        setGS2Format( current_index, singleLine(gs2_textarea.getText()) );
        boolean parse_success = setGS3Format( current_index, addSurroundingTags(gs3_textarea.getText()) );

        if(!parse_success) { // invalid XML, warn the user
        String message = Dictionary.get("FormatConversionDialog.Invalid_XML") + " " + Dictionary.get("FormatConversionDialog.Cancel_Or_Continue_Next");
        int user_choice = JOptionPane.showConfirmDialog(FormatConversionDialog.this, message, WARNING_TITLE, JOptionPane.OK_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE);

        if(user_choice == JOptionPane.CANCEL_OPTION) {
            return; // do nothing on this NextButton press. Don't increment. Let user re-adjust invalid XML for GS3 statement.
        }
        }
        
        if(increment()) {
        repaint();
        } else {
        next_button.setEnabled(false);
        getRootPane().setDefaultButton(accept_all_button); 

        }
        
    }
    }

    private class CancelButtonListener implements ActionListener {
    public void actionPerformed(ActionEvent e) {
         dlgResult = OpenCollectionDialog.CANCEL_OPTION;
         FormatConversionDialog.this.dispose(); // close dialog
    }
    }


    private class AcceptAllButtonListener implements ActionListener {
    public void actionPerformed(ActionEvent e) {
        //statusbar.setText("");

        // user okay-ed the lines, store them
        setGS2Format(current_index, gs2_textarea.getText());
        String gs3formatstr = gs3_textarea.getText();
        boolean parse_success = setGS3Format(current_index, addSurroundingTags(gs3formatstr));
        String message = "";

        if(!parse_success) { // invalid XML for current format statement, warn the user
        setErrorStatus(Dictionary.get("FormatConversionDialog.Invalid_XML"));

        message = Dictionary.get("FormatConversionDialog.Invalid_XML") + " " + Dictionary.get("FormatConversionDialog.Cancel_Or_Continue_Next");
        }

        // Even if the current GS3 format statement is valid XML, the user could have pressed 
        // Accept All at the very start of the FormatConversion dialog too. Check all the 
        // subsequent format statements, and if any have invalid XML, warn user.
        for(int i = current_index+1; parse_success && i < gsf_format_gs2_list.getLength(); i++) {       
        gs3formatstr = getGS3FormatString(i);
        String validationMsg = XMLTools.parseDOM(gs3formatstr);
        if(!validationMsg.startsWith(XMLTools.WELLFORMED)) {
            parse_success = false;
            message = Dictionary.get("FormatConversionDialog.Cancel_Or_Accept_All");
        }
        }

        if(!parse_success) {
        int user_choice = JOptionPane.showConfirmDialog(FormatConversionDialog.this, message, WARNING_TITLE, JOptionPane.OK_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE);
        if(user_choice == JOptionPane.CANCEL_OPTION) {
            return; // Don't close the dialog. Let the user continue looking at this or subsequent GS3 format statements.
        }
        }

        // If we're here, then either the format statements parsed, or the user accepted them anyway
        FormatConversionDialog.this.dispose(); // close dialog
    }
    }

    private class XMLTidyButtonListener implements ActionListener {
    // run HTML tidy taking input from stdin
    // http://www.w3.org/People/Raggett/tidy/
    public void actionPerformed(ActionEvent e) {
        String gs3formatstr_notags = gs3_textarea.getText();

        // Flag to determine which tags that HTML tidy adds need to be removed
        boolean startsWithTableCell = (gs3formatstr_notags.trim().toLowerCase().startsWith("<td")) ? true : false;
        // run HTML tidy on the GS3 format statement
        String htmltidy_string = runInteractiveProgram(XMLTIDY, gs3formatstr_notags);

        if(process_exitValue >= 2) {
        System.err.println("@@@ Process exit value: " + process_exitValue);
        setErrorStatus(Dictionary.get("FormatConversionDialog.Tidy_Failed"));
        } else {
        gs3formatstr_notags = htmltidy_string;
        
        // HTML tidy adds extra HTML markup around the formatstring, so will need to remove it:
        gs3formatstr_notags = removeHTMLTags(current_index, gs3formatstr_notags, startsWithTableCell);
        
        // put the XML tags around the gs3 format string again so we can convert it to a DOM object
        String gs3formatstr = addSurroundingTags(gs3formatstr_notags);
        
        boolean parse_success = setGS3Format(current_index, gs3formatstr); // converts to DOM object
        
        // Get indented GS3 format string from DOM object and display it in the text area
        gs3_textarea.setText( removeSurroundingTags(getGS3FormatString(current_index)) );

        if(parse_success) {
            statusbar.setText(Dictionary.get("FormatConversionDialog.Tidy_Done"));
        } else {
            setErrorStatus(Dictionary.get("FormatConversionDialog.Tidy_Done") 
                   + " " + Dictionary.get("FormatConversionDialog.Error_GS3_Format") 
                   + " " + Dictionary.get("FormatConversionDialog.Invalid_XML"));
        }
        }
    }
    }

    // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
    class ReadStreamGobbler extends Thread
    {
    InputStream is = null;
    StringBuffer outputstr = new StringBuffer();
    boolean split_newlines = false;
    
    
    public ReadStreamGobbler(InputStream is)
    {
        this.is = is;
        split_newlines = false;
    }

    public ReadStreamGobbler(InputStream is, boolean split_newlines)
    {
        this.is = is;
        this.split_newlines = split_newlines;
    }
    
    public void run()
    {
        BufferedReader br = null;
        try {
        br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
        String line=null;
        while ( (line = br.readLine()) != null) {
            //System.out.println("@@@ GOT LINE: " + line);
            outputstr.append(line);
            if(split_newlines) {
            outputstr.append("\n");
            }

        }
        } catch (IOException ioe) {
        ioe.printStackTrace();  
        } finally {
        closeResource(br);
        }
    }
    
    public String getOutput() { 
        return outputstr.toString(); // implicit toString() call anyway. //return outputstr; 
    }
    }
    
    class SendStreamGobbler extends Thread
    {
    OutputStream os = null;
    String inputstr = "";

    public SendStreamGobbler(OutputStream os, String inputstr)
    {
        this.os = os;
        this.inputstr = inputstr;
    }
    
    public void run()
    {
        BufferedWriter osw = null;
        try {
        osw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
        //System.out.println("@@@ SENDING LINE: " + inputstr);
        osw.write(inputstr, 0, inputstr.length());
        osw.newLine();//osw.write("\n");
        osw.flush();

        // Don't explicitly send EOF when using StreamGobblers as below, 
        // as the EOF char is echoed to output.
        // Flushing the write handle and/or closing the resource seems 
        // to already send EOF silently.

        /*if(Utility.isWindows()) {
            osw.write("\032"); // octal for Ctrl-Z, EOF on Windows
        } else { // EOF on Linux/Mac is Ctrl-D
            osw.write("\004"); // octal for Ctrl-D, see http://www.unix-manuals.com/refs/misc/ascii-table.html
        }
        osw.flush();
        */
        } catch (IOException ioe) {
        ioe.printStackTrace();  
        } finally {
        closeResource(osw);
        }
    }   
    }

    private class UndoListener implements ActionListener
    {   
    public void actionPerformed(ActionEvent event)
    {
        try {
        if (undoManager.canUndo()) {                        
            redo_button.setEnabled(true);
            undoManager.undo();                 
        }
        
        if (!undoManager.canUndo()) {
            undo_button.setEnabled(false);
        } else {
            undo_button.setEnabled(true);
        }
        
        } catch (Exception e) {
        System.err.println("Exception trying to undo: " + e.getMessage());
        }
    }
    }
    
    private class RedoListener implements ActionListener
    {
    public void actionPerformed(ActionEvent evt)
    {
        try {
        if (undoManager.canRedo()) {
            undo_button.setEnabled(true); // the difference with Format4gs3Manager, and no DocumentListener
            undoManager.redo();
        }
        
        if (!undoManager.canRedo()) {
            redo_button.setEnabled(false);
        } else {
            redo_button.setEnabled(true);
        }
        
        } catch (Exception e) {
        System.err.println("Exception trying to redo: " + e.getMessage());
        }
    }
    }

    private class CustomUndoableEditListener implements UndoableEditListener {
    public void undoableEditHappened(UndoableEditEvent evt)
    {
        
        undoManager.addEdit(evt.getEdit());
        redo_button.setEnabled(false);
        undo_button.setEnabled(true);
    }
    }

}