source: main/trunk/gli/src/org/greenstone/gatherer/cdm/CollectionConfigXMLReadWrite.java@ 34242

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Methods to read collectionConfig.xml files into internal XML form, and write 
 * them back out again.
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.cdm;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;

import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.metadata.MetadataElement;
import org.greenstone.gatherer.metadata.MetadataTools;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.util.XMLTools;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class CollectionConfigXMLReadWrite
{

    static final private String PLUGOUT_ELEMENT = "plugout";//used by building flax collections

    // a list of all known top level elements
  static final private String known_element_names_array[] = { StaticStrings.SECURITY_STR, StaticStrings.METADATALIST_STR, StaticStrings.DISPLAYITEMLIST_STR, StaticStrings.FORMAT_STR, StaticStrings.SEARCH_STR, StaticStrings.INFODB_STR, StaticStrings.BROWSE_STR, StaticStrings.IMPORT_STR, 
  /*StaticStrings.IMPORT_OPTION_STR, StaticStrings.BUILD_OPTION_STR,*/
  StaticStrings.DISPLAY_STR, StaticStrings.REPLACELISTREF_STR, StaticStrings.REPLACELIST_STR, StaticStrings.SERVICE_RACK_LIST_ELEMENT };
    static final private Set known_element_names = new HashSet(Arrays.asList(known_element_names_array));

    /**
     * *************************************************************************
     * ******************************* The code in this file is used for
     * greenstone 3 collection configuration, i.e., read ColletionConfig.xml
     * into the internal DOM tree, and convert the internal DOM tree back to
     * CollectionConfig.xml.
     * 
     * Methods named 'doXXXX' are for convert collectionConfig.xml into the
     * internal configuration xml structure; Methods named 'convertXXXX' are for
     * convert the internal configuration xml structure back to
     * collectionConfig.xml.
     ************************************************************************************************************ */

    /**
     * Arguments: metadataListNode->the 'displayItemList' element in
     * collectionConfig.xml name_value->the value of the 'name' attribute of
     * 'index' element; att_value->the value of the 'name' attribute of
     * 'displayItem' element return: an ArrayList of the contructed
     * 'CollectionMetadata' elements
     */
    static private ArrayList doDisplayItemList(Document to, Node displayListNode, String att_value, String name_value)
    {
        Element toElement = to.getDocumentElement();
        ArrayList display_item_list = new ArrayList();
        ArrayList item_list = XMLTools.getNamedElementList((Element) displayListNode, StaticStrings.DISPLAYITEM_STR, StaticStrings.NAME_ATTRIBUTE, att_value);
        if (item_list == null)
        {
            return null;
        }

        for (int i = 0; i < item_list.size(); i++)
        {
            Element item = (Element) item_list.get(i);
            boolean isCollDescr = name_value.equals(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR);

            // Sadly, XMLTools.getNodeText(item) returns empty for any HTML in displayItem
            // so we have to do it the hard way
            String text = isCollDescr ? preserveHTMLInDescriptionDisplayItem(item)
                : XMLTools.getNodeText(item); // else as before for all other displayItems

            
            //If there is nothing to display, don't bother creating the element
            // Either lang or key should be set. If key set, no textnode.
            // if lang set, should have textnode. Check if it's meaningful
            if (item.hasAttribute(StaticStrings.LANG_STR) && text.equals(""))
            {
                continue;
            }       

            // get the value in 'lang=langcode' or 'key=keyname'
            // We can have either a key attribute (with optional dictionary attribute, else dictionary is implicit)
            // OR we can have a lang attr, with the nodetext containing the actual display string.
            String lang = item.getAttribute(StaticStrings.LANG_STR);
            String key = item.getAttribute(StaticStrings.KEY_ATTRIBUTE);

            Element e = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
            e.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
            e.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_value);

            if(!lang.equals("")) {
                e.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, lang);
                text = text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
                XMLTools.setNodeText(e, text);
                
            }

            if(!key.equals("")) {
                e.setAttribute(StaticStrings.KEY_ATTRIBUTE, key);
                String dictionary = item.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
                if(!dictionary.equals("")) {
                e.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary);
                }
            }

            display_item_list.add(e);
        }
        return display_item_list;
    }

    static private ArrayList doMetadataList(Document to, Node metadataListNode, String ele_name, String att_value)
    {
        Element toElement = to.getDocumentElement();
        ArrayList metadata_list = new ArrayList();

        ArrayList item_list = XMLTools.getNamedElementList((Element) metadataListNode, StaticStrings.METADATA_STR, StaticStrings.NAME_ATTRIBUTE, att_value);
        if (item_list == null)
        {
            return null;
        }

        for (int i = 0; i < item_list.size(); i++)
        {
            Element item = (Element) item_list.get(i);
            String text = XMLTools.getNodeText(item);

            //If there is nothing to display, don't bother creating the element
            if (text.equals(""))
            {
                continue;
            }
            //get the value in 'lang=value'
            String lang = item.getAttribute(StaticStrings.LANG_STR);

            Element element = to.createElement(ele_name);
            element.setAttribute(StaticStrings.NAME_ATTRIBUTE, att_value);
            element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, lang);
            element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
            element.setAttribute(StaticStrings.SPECIAL_ATTRIBUTE, StaticStrings.TRUE_STR);
            XMLTools.setNodeText(element, text);

            metadata_list.add(element);
        }
        return metadata_list;
    }

    // 'to' is the internal structure
    static private void doMGIndexes(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        Element indexes_element = to.createElement(StaticStrings.INDEXES_ELEMENT);//<Indexes>
        indexes_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        indexes_element.setAttribute(StaticStrings.MGPP_ATTRIBUTE, StaticStrings.FALSE_STR);

        NodeList index_children = ((Element) searchNode).getElementsByTagName(StaticStrings.INDEX_LOW_STR);//index
        int num_nodes = index_children.getLength();

        for (int i = 0; i < num_nodes; i++)
        {
            Element e = (Element) index_children.item(i);
            String index_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String index_str_display = index_str;//used for creating collectionmetadata for this index
            Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);//<Index>

            if (index_str.indexOf(StaticStrings.COLON_CHARACTER) == -1)
            {
                // It doesn't contain ':' character
                System.err.println("Something is wrong! the index should be level:source tuplets.");
                // assume document level
                index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, StaticStrings.DOCUMENT_STR);
            }
            else
            {
                // Handling 'index' element
                index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, index_str.substring(0, index_str.indexOf(StaticStrings.COLON_CHARACTER)));

                index_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
            }
            //Each index may have a list of comma-separated strings.
            //split them into 'content' elements in the internal structure
            StringTokenizer content_tokenizer = new StringTokenizer(index_str, StaticStrings.COMMA_CHARACTER);
            //index_str = "";
            while (content_tokenizer.hasMoreTokens())
            {
                // Replace index_str to be qualified name, eg. dc.Subject and keywords insread of dc.Subject.

                Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
                String content_str = content_tokenizer.nextToken();
                // Since the contents of indexes have to be certain keywords, or metadata elements,
                //if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
                if (content_str.indexOf(StaticStrings.NS_SEP) == -1 && !(content_str.equals(StaticStrings.TEXT_STR)))
                {
                    content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;

                }

                content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, content_str);
                index_element.appendChild(content_element);
                content_element = null;
            } // while ends

            indexes_element.appendChild(index_element);

            // Handling 'displayItem' elements and Constructing 'collectionmetadata' elements
            // Use the fully qualified index names
            ArrayList collectionmetadata_list = doDisplayItemList(to, e, StaticStrings.NAME_ATTRIBUTE, index_str_display);
            appendArrayList(toElement, collectionmetadata_list);
        } //for loop ends
        appendProperly(toElement, indexes_element);

        //***//
        // create another set of <indexes> which will be used when user switches to MGPP/LUCENE
        // i.e. we build a default index set for a start

        String[] index_strs = { StaticStrings.TEXT_STR, StaticStrings.EXTRACTED_NAMESPACE + StaticStrings.TITLE_ELEMENT, StaticStrings.EXTRACTED_NAMESPACE + StaticStrings.SOURCE_ELEMENT };

        Element mgpp_indexes = to.createElement(StaticStrings.INDEXES_ELEMENT);//<Indexes>
        mgpp_indexes.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
        mgpp_indexes.setAttribute(StaticStrings.MGPP_ATTRIBUTE, StaticStrings.TRUE_STR);
        for (int i = 0; i < index_strs.length; i++)
        {
            Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);//<Index>
            Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
            content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_strs[i]);
            index_element.appendChild(content_element);
            mgpp_indexes.appendChild(index_element);

            // Contructing 'collectionmetadata' elements for 'mgpp' indexes
            Element collectionmetadata = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
            collectionmetadata.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
            collectionmetadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_strs[i]);
            collectionmetadata.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, StaticStrings.ENGLISH_LANGUAGE_STR);
            if (index_strs[i].indexOf(StaticStrings.NS_SEP) != -1)
            {
                index_strs[i] = index_strs[i].substring(index_strs[i].indexOf(StaticStrings.NS_SEP) + 1);
            }
            XMLTools.setNodeText(collectionmetadata, index_strs[i]);

            appendProperly(toElement, collectionmetadata);

        }
        appendProperly(toElement, mgpp_indexes);
    }

    //This is actually doing indexes for both mgpp and lucene
    static private void doMGPPIndexes(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        Element indexes_element = to.createElement(StaticStrings.INDEXES_ELEMENT);//<Indexes>
        indexes_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        indexes_element.setAttribute(StaticStrings.MGPP_ATTRIBUTE, StaticStrings.TRUE_STR);

        NodeList index_children = ((Element) searchNode).getElementsByTagName(StaticStrings.INDEX_LOW_STR);//index
        int num_nodes = index_children.getLength();

        for (int i = 0; i < num_nodes; i++)
        {

            Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);//<Index>
            Element e = (Element) index_children.item(i);
            String index_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String index_str_display = index_str;//for creating collectionmetadata for this index 


            String options_str = "";
            NodeList option_children = e.getElementsByTagName(StaticStrings.OPTION_STR);
            if(option_children != null) {
               for (int j = 0; j < option_children.getLength(); j++) {
                Element el = (Element) option_children.item(j);

                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                options_str = options_str + ((name_str.equals("")) ? "" : (" " + name_str));

                String value_str = el.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
                options_str = options_str + ((name_str.equals("")) ? "" : (" " + value_str));

                if (name_str.equals("") && !value_str.equals(""))
                {
                    continue; //invalid condition
                }

                Element option_element = null;
                option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
                option_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

                if (!name_str.equals(""))
                {
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }
                if (value_str != null && !value_str.equals(""))
                {
                    XMLTools.setNodeText(option_element, value_str);
                }
                index_element.appendChild(option_element);
               }
            }

            // Handling 'index' element
            // Double check to make sure it's not colon separated style index.
            if (index_str.indexOf(StaticStrings.COLON_CHARACTER) != -1)
            {
                System.err.println("Something is wrong! the index should NOT be level:source tuplets style.");
                index_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);

            }
            //Each index may have a list of comma-separated strings.
            //split them into 'content' elements in the internal structure
            StringTokenizer content_tokenizer = new StringTokenizer(index_str, StaticStrings.COMMA_CHARACTER);
            while (content_tokenizer.hasMoreTokens())
            {
                // Replace index_str to be qualified name, eg. dc.Subject and keywords insread of dc.Subject.             

                Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
                String content_str = content_tokenizer.nextToken();
                // Since the contents of indexes have to be certain keywords, or metadata elements, if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
                if (content_str.indexOf(StaticStrings.NS_SEP) == -1 && !(content_str.equals(StaticStrings.TEXT_STR) || content_str.equals(StaticStrings.ALLFIELDS_STR)))
                {
                    content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;
                }
                content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, content_str);
                index_element.appendChild(content_element);
                content_element = null;
            } //while ends

            indexes_element.appendChild(index_element);

            index_element = null;

            // Handling 'displayItem' element of this 'index' element
            // 'e' is the parent element 'index' of 'displayItem' element
            ArrayList collectionmetadata_list = doDisplayItemList(to, e, StaticStrings.NAME_ATTRIBUTE, index_str_display);
            appendArrayList(toElement, collectionmetadata_list);

        } // for loop ends
        toElement.appendChild(indexes_element);

        // create another set of <indexes> which will be used when user switches to MG
        // i.e. we build a default index set for a start
        Element mg_indexes = to.createElement(StaticStrings.INDEXES_ELEMENT);//<Indexes>
        mg_indexes.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
        mg_indexes.setAttribute(StaticStrings.MGPP_ATTRIBUTE, StaticStrings.FALSE_STR);

        //put the namespace '.ex' as prefix to the indexes
        String[] index_strs = { StaticStrings.TEXT_STR, StaticStrings.EXTRACTED_NAMESPACE + StaticStrings.TITLE_ELEMENT, StaticStrings.EXTRACTED_NAMESPACE + StaticStrings.SOURCE_ELEMENT };
        for (int i = 0; i < index_strs.length; i++)
        {
            Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);//<Index>
            index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, StaticStrings.DOCUMENT_STR);
            Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
            content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_strs[i]);
            index_element.appendChild(content_element);

            mg_indexes.appendChild(index_element);

            // Contructing 'collectionmetadata' elements for 'mg' indexes
            Element collectionmetadata = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
            collectionmetadata.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
            String temp = StaticStrings.DOCUMENT_STR.concat(StaticStrings.COLON_CHARACTER).concat(index_strs[i]);
            collectionmetadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, temp);
            collectionmetadata.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, StaticStrings.ENGLISH_LANGUAGE_STR);
            if (index_strs[i].indexOf(StaticStrings.NS_SEP) != -1)
            {
                index_strs[i] = index_strs[i].substring(index_strs[i].indexOf(StaticStrings.NS_SEP) + 1);
            }
            XMLTools.setNodeText(collectionmetadata, index_strs[i]);

            appendProperly(toElement, collectionmetadata);

        }
        toElement.appendChild(mg_indexes);

    }

  //This processes <sort> (lucene), or <sort> and <facet> (solr) subelements of <search> also <defaultSort>
  // Note that GLI at present does not allow the user to set or modify these.
  // This function, and its inverse convertSolrFacetsAndSorts, are only here to preserve such elements if they already exist in the collectionConfig.xml. At present they need to be manually added there.
  // it just copies them into storage.
    static private void doSolrFacetsAndSorts(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        Element solr_element = to.createElement(StaticStrings.SOLR_ELEMENT);//<Solr>
        solr_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        solr_element.setAttribute(StaticStrings.MGPP_ATTRIBUTE, StaticStrings.TRUE_STR);


        NodeList sort_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SORT_LOW_STR);//<sort>
        
        int num_nodes = sort_children.getLength();

        for (int i = 0; i < num_nodes; i++) {
          solr_element.appendChild(to.importNode((Element) sort_children.item(i), true));
        }
        Element sort_def = (Element)XMLTools.getChildByTagName(searchNode, StaticStrings.SORT_DEFAULT_ELEMENT);
        if (sort_def != null) {
          solr_element.appendChild(to.importNode(sort_def, true));
        }

        NodeList facet_children = ((Element) searchNode).getElementsByTagName(StaticStrings.FACET_LOW_STR);//facet
        num_nodes = facet_children.getLength();

        for (int i = 0; i < num_nodes; i++)
        {
          solr_element.appendChild(to.importNode((Element) facet_children.item(i), true));
        }

        // finished processing sort and facet elements
        toElement.appendChild(solr_element);

    
    }

    static private void doGlobalFormat(Document to, Element from)
    {
        // look for a top level format element
        Element fe = (Element) XMLTools.getChildByTagName(from, StaticStrings.FORMAT_STR);
        to.getDocumentElement().appendChild(doFormat(to, fe, StaticStrings.GLOBAL_STR));
    }

    static private void doDisplayFormat(Document to, Element from)
    {
        //display element in the xml file
        Element de = (Element) XMLTools.getChildByTagName(from, StaticStrings.DISPLAY_STR);
        if (de == null)
        {
            return;
        }
        //format element in the display element
        Element fe = (Element) XMLTools.getChildByTagName(de, StaticStrings.FORMAT_STR);

        to.getDocumentElement().appendChild(doFormat(to, fe, StaticStrings.DISPLAY_STR));
    }

    //construct 'DefaultIndex' element in the internal structure from collectionConfig.xml
    static private void doDefaultIndex(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        Element default_index_element = to.createElement(StaticStrings.INDEX_DEFAULT_ELEMENT);
        default_index_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

        Element e = (Element) XMLTools.getChildByTagName(searchNode, StaticStrings.INDEX_DEFAULT_ELEMENT_LOWERCASE);//defaultIndex
        if (e == null)
        {
            return;
        }
        String index_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);

        boolean old_index = false;
        if (index_str.indexOf(StaticStrings.COLON_CHARACTER) != -1)
        {
            //The index is 'level:source tuplets' which is for mg. Take out 'level'
            old_index = true;
            default_index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, index_str.substring(0, index_str.indexOf(StaticStrings.COLON_CHARACTER)));
            index_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
        }
        else
        {
            default_index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, "");
        }

        //Each index may have a list of comma-separated strings.
        //split them into 'content' elements in the internal structure
        StringTokenizer content_tokenizer = new StringTokenizer(index_str, StaticStrings.COMMA_CHARACTER);
        while (content_tokenizer.hasMoreTokens())
        {
            Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
            String content_str = content_tokenizer.nextToken();
            // Since the contents of indexes have to be certain keywords, or metadata elements, if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
            if (content_str.indexOf(StaticStrings.NS_SEP) == -1)
            {
                if (content_str.equals(StaticStrings.TEXT_STR) || (!old_index && content_str.equals(StaticStrings.ALLFIELDS_STR)))
                {
                    // in this case, do nothing
                }
                else
                {
                    content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;
                }
            }

            content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, content_str);
            default_index_element.appendChild(content_element);
            content_element = null;
        }
        appendProperly(toElement, default_index_element);
    }

    // For mg, this method is still called, but make it 'assigned=false'
    static private void doDefaultLevel(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        Element default_index_option = to.createElement(StaticStrings.INDEXOPTION_DEFAULT_ELEMENT);
        default_index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.LEVEL_DEFAULT_STR);

        Element e = (Element) XMLTools.getChildByTagName(searchNode, StaticStrings.LEVEL_DEFAULT_ELEMENT);
        if (e != null)
        {
            default_index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
            String level = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            default_index_option.setAttribute(StaticStrings.VALUE_ATTRIBUTE, level);
        }
        else
        {
            //In the case of mg, there's no level! build a default one using 'assigned=false value=document'
            default_index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
            default_index_option.setAttribute(StaticStrings.VALUE_ATTRIBUTE, StaticStrings.DOCUMENT_STR);
        }
        appendProperly(toElement, default_index_option);
    }

    // Transform plugins (pluginListNode) of collectionConfig.xml into the internal structure (i.e. Document to)
    static private void doPlugins(Document to, Node pluginListNode)
    {
        Element toElement = to.getDocumentElement();
        NodeList plugin_children = ((Element) pluginListNode).getElementsByTagName(StaticStrings.PLUGIN_STR);
        int plugin_nodes = plugin_children.getLength();

        if (plugin_nodes < 1)
        {
            return;
        }

        for (int i = 0; i < plugin_nodes; i++)
        {
            Element e = (Element) plugin_children.item(i);
            String str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            str = Utility.ensureNewPluginName(str);
            Element plugin_element = to.createElement(StaticStrings.PLUGIN_ELEMENT);
            plugin_element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, str);

            NodeList option_children = e.getElementsByTagName(StaticStrings.OPTION_STR);

            for (int j = 0; j < option_children.getLength(); j++)
            {
                Element el = (Element) option_children.item(j);
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                if (name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                {
                    name_str = name_str.substring(1);
                }
                String value_str = el.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
                Element option_element = null;

                if (name_str.equals("") && !value_str.equals(""))
                {
                    continue;
                }

                option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
                option_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
                if (name_str.equals(StaticStrings.RECPLUG_STR) && value_str.equals(StaticStrings.USE_METADATA_FILES_ARGUMENT))
                {
                    continue; // ignore this option
                }

                if (value_str != null)
                {
                    // Remove any speech marks appended in strings containing whitespace
                    if (value_str.startsWith(StaticStrings.SPEECH_CHARACTER) && value_str.endsWith(StaticStrings.SPEECH_CHARACTER))
                    {
                        value_str = value_str.substring(1, value_str.length() - 1);
                    }
                    if (name_str.equals(StaticStrings.METADATA_STR))
                    {
                        // The metadata argument must be the fully qualified name of a metadata element, so if it doesn't yet have a namespace, append the extracted metadata namespace.
                        String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                        value_str = "";
                        for (int k = 0; k <= values.length - 1; k++)
                        {
                            if (values[k].indexOf(StaticStrings.NS_SEP) == -1)
                            {
                                values[k] = StaticStrings.EXTRACTED_NAMESPACE + values[k];
                            }

                            if (k < values.length - 1)
                            {
                                value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;

                            }
                            else
                            {
                                value_str = value_str + values[k];
                            }
                        }
                    }
                }
                if (!name_str.equals(""))
                {
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }
                if (!value_str.equals(""))
                {
                    XMLTools.setNodeText(option_element, value_str);
                }
                plugin_element.appendChild(option_element);

            }

            appendProperly(toElement, plugin_element);
        }

    }

    //Handle classifiers
    static private void doClassifiers(Document to, Node browseNode)
    {
        Element toElement = to.getDocumentElement();
        NodeList classifier_children = ((Element) browseNode).getElementsByTagName(StaticStrings.CLASSIFIER_STR);
        int num_nodes = classifier_children.getLength();

        if (num_nodes < 1)
        {
            return;
        }

        for (int i = 0; i < num_nodes; i++)
        {
            Element e = (Element) classifier_children.item(i);
            String str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element classify_element = to.createElement(StaticStrings.CLASSIFY_ELEMENT);
            classify_element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, str);

            String options_str = "";
            NodeList option_children = e.getElementsByTagName(StaticStrings.OPTION_STR);
            for (int j = 0; j < option_children.getLength(); j++)
            {
                Element el = (Element) option_children.item(j);
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                options_str = options_str + ((name_str.equals("")) ? "" : (" " + name_str));
                if (name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                {
                    name_str = name_str.substring(1);
                }
                String value_str = el.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
                options_str = options_str + ((name_str.equals("")) ? "" : (" " + value_str));
                Element option_element = null;

                if (name_str.equals("") && !value_str.equals(""))
                {
                    continue; //invalid condition
                }

                option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
                option_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

                if (!name_str.equals(""))
                {
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }

                if (!value_str.equals("") && name_str.equals(StaticStrings.METADATA_STR))
                {
                    // The metadata argument must be the fully qualified name of a metadata element, so if it doesn't yet have a namespace, append the extracted metadata namespace.
                    String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                    value_str = "";
                    for (int k = 0; k <= values.length - 1; k++)
                    {
                        if (values[k].indexOf(StaticStrings.NS_SEP) == -1)
                        {
                            values[k] = StaticStrings.EXTRACTED_NAMESPACE + values[k];
                        }
                        else
                        {
                            MetadataElement metadata_element = MetadataTools.getMetadataElementWithName(values[k]);
                            if (metadata_element != null)
                            {
                                values[k] = metadata_element.getDisplayName();
                            }
                        }
                        if (k < values.length - 1)
                        {
                            value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;
                        }
                        else
                        {
                            value_str = value_str + values[k];
                        }
                    }
                }

                if (value_str != null && !value_str.equals(""))
                {
                    XMLTools.setNodeText(option_element, value_str);
                }
                classify_element.appendChild(option_element);

            } // for each option
            
            //format element for this classifier
            Element format = (Element) XMLTools.getChildByTagName(e, StaticStrings.FORMAT_STR);
            if (format != null)
            {
                classify_element.appendChild(doFormat(to, format, null));
            }

            // Handling 'displayItem' element of this 'classifier' element - for now, just copy in and out so they don't get deleted
            NodeList di_children = e.getElementsByTagName(StaticStrings.DISPLAYITEM_STR);
            
            XMLTools.duplicateElementList(to, classify_element, di_children, true);
                        
            appendProperly(toElement, classify_element);
        }

        // default format statement for all classifiers
        Element default_classifier_format = (Element) XMLTools.getChildByTagName(browseNode, StaticStrings.FORMAT_STR);

        to.getDocumentElement().appendChild(doFormat(to, default_classifier_format, StaticStrings.BROWSE_STR));
    }

    static private Element doFormat(Document to, Element format, String name_str)
    {
        Element format_element = to.createElement(StaticStrings.FORMAT_STR);
        if (name_str != null)
        {
            format_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
        }

        // Don't write out an empty format statement of <format/> (i.e. format has no child nodes)
        // as this will end up embedded in another format statement as <format><format/><format />
        // This doubling up of format stmts will then prevent GLI from opening the collection again.
        if (format != null && format.hasChildNodes())
        { // not an empty format statement
            String gsf_text = XMLTools.xmlNodeToString(format);

            //We don't want the <format> elements in the string
            int startFormatIndex = gsf_text.indexOf(StaticStrings.FORMAT_START_TAG) + StaticStrings.FORMAT_START_TAG.length();
            int endFormatIndex = gsf_text.lastIndexOf(StaticStrings.FORMAT_END_TAG);
            gsf_text = gsf_text.substring(startFormatIndex, endFormatIndex);

            XMLTools.setNodeText(format_element, gsf_text);
        }
        return format_element;
    }

    // Handling 'subcollection' elements in 'search' element of 'collectionConfig.xml'
    static private void doSubcollection(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        NodeList sub_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SUBCOLLECTION_STR);
        int sub_nodes = sub_children.getLength();

        // There is no subcollection
        if (sub_nodes < 1)
        {
            return;
        }

        for (int i = 0; i < sub_nodes; i++)
        {
            Element sub_child = (Element) sub_children.item(i);
            String name_str = sub_child.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String filter_str = sub_child.getAttribute(StaticStrings.FILTER_ATTRIBUTE);

            // filter_str is in the form '<! (if set)><metadata>/<metadata value>/<flag (if any)>'

            int pos = filter_str.indexOf(StaticStrings.SEPARATOR_CHARACTER);
            String meta_str = "";
            String meta_value_str = "";
            String clude_str = "";
            String flag_str = "";
            if (pos == -1)
            {

                meta_str = meta_value_str = filter_str;
                clude_str = StaticStrings.INCLUDE_STR;
            }
            else
            {
                clude_str = StaticStrings.INCLUDE_STR;
                if (filter_str.startsWith(StaticStrings.EXCLAMATION_CHARACTER))
                {
                    clude_str = StaticStrings.EXCLUDE_STR;
                    // Peel off "!"
                    filter_str = filter_str.substring(StaticStrings.EXCLAMATION_CHARACTER.length());
                }

                String[] strs = filter_str.split(StaticStrings.SEPARATOR_CHARACTER);
                if (strs[0] != null && strs[0] != "")
                {
                    meta_str = strs[0];
                }
                if (!meta_str.equals(StaticStrings.FILENAME_STR) && meta_str.indexOf(StaticStrings.NS_SEP) == -1)
                {
                    meta_str = StaticStrings.EXTRACTED_NAMESPACE + meta_str;
                }

                if (strs[1] != null && strs[1] != "")
                {
                    meta_value_str = strs[1];
                }
                if (strs.length > 2)
                {
                    //This means there has been set a flag
                    if (strs[2] != null && strs[2] != "")
                    {
                        flag_str = strs[2];
                    }
                }
            }
            Element subcollection_element = to.createElement(StaticStrings.SUBCOLLECTION_ELEMENT);
            subcollection_element.setAttribute(StaticStrings.NAME_STR, name_str);
            subcollection_element.setAttribute(StaticStrings.CONTENT_ATTRIBUTE, meta_str);
            subcollection_element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, clude_str);
            if (flag_str != "")
            {
                subcollection_element.setAttribute(StaticStrings.OPTIONS_ATTRIBUTE, flag_str);
            }
            XMLTools.setNodeText(subcollection_element, meta_value_str);

            toElement.appendChild(subcollection_element);
        }
    }

    //Handle levels (document, section). In the internal structure, the element is called 'IndexOption'
    static private void doLevel(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        NodeList level_children = ((Element) searchNode).getElementsByTagName(StaticStrings.LEVEL_ATTRIBUTE);
        int level_nodes = level_children.getLength();

        // it's mg, there's no level. So we construct a default 'indexOption' in the internal structure
        if (level_nodes < 1)
        {
            Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
            index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
            index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.LEVELS_STR);

            Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
            option_element.setAttribute(StaticStrings.NAME_STR, StaticStrings.DOCUMENT_STR);
            index_option.appendChild(option_element);

            appendProperly(toElement, index_option);

            return;
        }

        Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
        index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.LEVELS_STR);

        for (int i = 0; i < level_nodes; i++)
        {
            Element level_element = (Element) level_children.item(i);
            String level_str = level_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
            option_element.setAttribute(StaticStrings.NAME_STR, level_str);
            index_option.appendChild(option_element);

            // Contructing 'collectionmetadata' elements from the 'displayItem' of this 'level' element
            ArrayList displayItem_list = XMLTools.getNamedElementList(level_element, StaticStrings.DISPLAYITEM_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.NAME_STR);
            if (displayItem_list == null)
            {
                return;
            }
            for (int j = 0; j < displayItem_list.size(); j++)
            {
                Element item = (Element) displayItem_list.get(j);
                String text = XMLTools.getNodeText(item);

                //If there is nothing to display, don't bother creating the element
                // Either lang or key should be set. If lang set, then should have textnode.
                if (item.hasAttribute(StaticStrings.LANG_STR) && text.equals("")) {
                    continue;
                }
                
                String lang = item.getAttribute(StaticStrings.LANG_ATTRIBUTE);
                String key = item.getAttribute(StaticStrings.KEY_ATTRIBUTE);
            
                Element collectionmetadata = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
                collectionmetadata.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
                collectionmetadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, level_str);

                if(!lang.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, lang);
                    XMLTools.setNodeText(collectionmetadata, text);
                }
                
                if(!key.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.KEY_ATTRIBUTE, key);
                    String dictionary = item.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
                    if(!dictionary.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary);
                    }
                }

                appendProperly(toElement, collectionmetadata);
            }
        }
        appendProperly(toElement, index_option);
    }

    //Handle 'indexSubcollection' element of collectionConfig.xml,  which is called 'SubcollectionIndexes' in the internal structure. These contain the subcollection indexes (i.e. the filter or filter combinations), and displayed words for the filter names.
    static private void doIndexSubcollection(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        NodeList index_sub_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SUBCOLLECTION_INDEX_ELEMENT);
        int num_nodes = index_sub_children.getLength();

        // there is no subcollection index
        if (num_nodes < 1)
        {
            return;
        }

        Element subcollection_indexes = to.createElement(StaticStrings.SUBCOLLECTION_INDEXES_ELEMENT);

        for (int i = 0; i < num_nodes; i++)
        {
            Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);
            Element index_sub_child = (Element) index_sub_children.item(i);
            String name_str = index_sub_child.getAttribute(StaticStrings.NAME_ATTRIBUTE);

            // name_str is in the form of comma separated strings, each of which is a subcollection filter name
            String[] filters = name_str.split(StaticStrings.COMMA_CHARACTER);
            for (int j = 0; j < filters.length; j++)
            {

                Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
                content_element.setAttribute(StaticStrings.NAME_STR, filters[j]);
                index_element.appendChild(content_element);
            }
            subcollection_indexes.appendChild(index_element);

            // Contructing 'collectionmetadata' elements from the 'displayItem' of this 'indexSubcollection' element
            ArrayList displayItem_list = XMLTools.getNamedElementList(index_sub_child, StaticStrings.DISPLAYITEM_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.NAME_STR);
            if (displayItem_list == null)
            {
                // there is no display item for this element
                continue;
            }
            for (int j = 0; j < displayItem_list.size(); j++)
            {
                Element item = (Element) displayItem_list.get(j);
                String text = XMLTools.getNodeText(item);
                

                //If there is nothing to display, don't bother creating the element
                // Either lang or key should be set. If key set, no textnode.
                // if lang set, should have textnode. Check if it's meaningful
                if (item.hasAttribute(StaticStrings.LANG_STR) && text.equals(""))
                {
                    continue;
                }

                String lang = item.getAttribute(StaticStrings.LANG_ATTRIBUTE);
                String key = item.getAttribute(StaticStrings.KEY_ATTRIBUTE);

                Element collectionmetadata = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
                collectionmetadata.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
                collectionmetadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                if(!lang.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, lang);
                    XMLTools.setNodeText(collectionmetadata, text);
                }
                if(!key.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.KEY_ATTRIBUTE, key);
                    String dictionary = item.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
                    if(!dictionary.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary);
                    }
                }
                
                appendProperly(toElement, collectionmetadata);
            }
        }
        appendProperly(toElement, subcollection_indexes);
    }

    //Handle 'indexLanguage' element of collectionConfig.xml,  which is called 'Languages' in the internal structure. These contain the language indexes, and displayed words for those language index names.
    static private void doIndexLanguage(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        NodeList index_sub_children = ((Element) searchNode).getElementsByTagName(StaticStrings.LANGUAGE_INDEX_ELEMENT);
        int num_nodes = index_sub_children.getLength();

        // there is no subcollection index
        if (num_nodes < 1)
        {
            return;
        }

        Element language_indexes = to.createElement(StaticStrings.LANGUAGES_ELEMENT);

        for (int i = 0; i < num_nodes; i++)
        {
            Element language_element = to.createElement(StaticStrings.LANGUAGE_ELEMENT);
            Element index_sub_child = (Element) index_sub_children.item(i);
            String name_str = index_sub_child.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            language_element.setAttribute(StaticStrings.NAME_STR, name_str);
            language_indexes.appendChild(language_element);

            // Contructing 'collectionmetadata' elements from the 'displayItem' of this 'indexLanguage' element
            ArrayList displayItem_list = XMLTools.getNamedElementList(index_sub_child, StaticStrings.DISPLAYITEM_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.NAME_STR);
            if (displayItem_list == null)
            {
                // there is no display item for this element
                continue;
            }
            for (int j = 0; j < displayItem_list.size(); j++)
            {
                Element item = (Element) displayItem_list.get(j);
                String text = XMLTools.getNodeText(item);
                //If there is nothing to display, don't bother creating the element
                // Either lang or key should be set. If key set, no textnode.
                // if lang set, should have textnode. Check if it's meaningful
                if (item.hasAttribute(StaticStrings.LANG_STR) && text.equals(""))
                {
                    continue;
                }

                // get the value in 'lang=langcode' or 'key=keyname'
                // We can have either a key attribute (with optional dictionary attribute, else dictionary is implicit)
                // OR we can have a lang attr, with the nodetext containing the actual display strin.g
                String lang = item.getAttribute(StaticStrings.LANG_ATTRIBUTE);
                String key = item.getAttribute(StaticStrings.KEY_ATTRIBUTE);


                Element collectionmetadata = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
                collectionmetadata.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
                collectionmetadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);

                if(!lang.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, lang);
                    XMLTools.setNodeText(collectionmetadata, text);
                }
                if(!key.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.KEY_ATTRIBUTE, key);
                    String dictionary = item.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
                    if(!dictionary.equals("")) {
                    collectionmetadata.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary);
                    }
                }

                appendProperly(toElement, collectionmetadata);
            }
        }
        toElement.appendChild(language_indexes);
    }

    // Handling search types
    static private void doSearchType(Document to, Node searchNode)
    {
        NodeList type_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SEARCHTYPE_ELEMENT);
        int num_types = type_children.getLength();
        String searchtype_str = "";
        if (num_types < 1)
        {
            // not defined yet, add in default
            searchtype_str = "plain,simpleform,advancedform";
        }
        else
        {
            for (int i = 0; i < num_types; i++)
            {
                Node e = type_children.item(i);
                String t = ((Element) e).getAttribute(StaticStrings.NAME_ATTRIBUTE);
                if (i > 0)
                {
                    searchtype_str += ",";
                }
                searchtype_str += t;
            }
        }
        searchtype_str = searchtype_str.trim();

        // pretend its a format statement
        Element search_type_element = to.createElement(StaticStrings.FORMAT_STR);
        search_type_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, StaticStrings.SEARCHTYPE_ELEMENT);
        XMLTools.setNodeText(search_type_element, searchtype_str);
        appendProperly(to.getDocumentElement(), search_type_element);

    }

    // Handling search format statement
    static private void doSearchFormat(Document to, Node searchNode)
    {
        // THere is currently just one format element for search. HOwever, need to check for old config files which used to have <format name="searchTypes">
        NodeList format_children = ((Element) searchNode).getElementsByTagName(StaticStrings.FORMAT_STR);
        int format_nodes = format_children.getLength();
        if (format_nodes < 1)
        {
            return;
        }
        Element format = null;
        for (int i = 0; i < format_nodes; i++)
        {
            Node e = format_children.item(i);
            if (e.hasAttributes() == false)
            {
                //The format element for format statement has no attribute
                format = (Element) e;
            }
        }
        //format statement for search
        if (format != null)
        {
            (to.getDocumentElement()).appendChild(doFormat(to, format, StaticStrings.SEARCH_STR));
        }
    }

    // Handling defaultIndexLanguage and languageMetadata in collectionConfig.xml ('elementNameFrom'); in the internal structure, they are called 'DefaultLanguage' and 'LanguageMetadata' ('elementNameTo') respectively.
    // Converting from collectionConfig.xml to the internal xml structure.
    static private void doLanguageMetadata(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        String elementNameFrom = StaticStrings.LANGUAGE_METADATA_ELEMENT_STR;
        String elementNameTo = StaticStrings.LANGUAGE_METADATA_ELEMENT;
        Node from_element = XMLTools.getChildByTagName(searchNode, elementNameFrom);
        if (from_element == null)
        {
            return; // such an element not found
        }

        Element to_element = to.createElement(elementNameTo);

        String name_str = ((Element) from_element).getAttribute(StaticStrings.NAME_ATTRIBUTE);
        if (name_str.indexOf(StaticStrings.NS_SEP) == -1)
        {
            name_str = StaticStrings.EXTRACTED_NAMESPACE + name_str;
        }
        to_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
        to_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

        toElement.appendChild(to_element);
    }

    static private void doReplaceListRef(Document to, Element from)
    {
        Element toElement = to.getDocumentElement();

        NodeList replace_elements = from.getElementsByTagName(StaticStrings.REPLACELISTREF_STR);
        XMLTools.duplicateElementList(to, toElement, replace_elements, true);
    }

    //  int num_elems = replace_elements.getLength();
    //  if (num_elems < 1)
    //  {
    //      return;
    //  }
    //  for (int i = 0; i < num_elems; i++)
    //  {
    //      Element to_element = XMLTools.duplicateElement(to, (Element) replace_elements.item(i), true);
    //      toElement.appendChild(to_element);
    //  }
    // }

    static private void convertReplaceListRef(Document from, Document to)
    {
        Element toElement = to.getDocumentElement();

        NodeList replace_elements = from.getDocumentElement().getElementsByTagName(StaticStrings.REPLACELISTREF_STR);
        XMLTools.duplicateElementList(to, toElement, replace_elements, true);
    }

    // int num_elems = replace_elements.getLength();
    // if (num_elems < 1)
    // {
    //  return;
    // }
    // for (int i = 0; i < num_elems; i++)
    // {
    //  Element to_element = XMLTools.duplicateElement(to, (Element) replace_elements.item(i), true);
    //  toElement.appendChild(to_element);
    // }
    //  }

    /*
     * replacelist currently not editable in GLI, just copy it in and back out
     * again
     */
    static private void doReplaceList(Document to, Element from)
    {
        Element toElement = to.getDocumentElement();

        Node rl_element = XMLTools.getChildByTagName(from, StaticStrings.REPLACELIST_STR);
        if (rl_element == null)
        {
            return; // such an element not found
        }

        Element to_element = XMLTools.duplicateElement(to, (Element) rl_element, true);
        toElement.appendChild(to_element);
    }

    static private void convertReplaceList(Document from, Document to)
    {
        Element toElement = to.getDocumentElement();

        Node rl_element = XMLTools.getChildByTagName(from.getDocumentElement(), StaticStrings.REPLACELIST_STR);
        if (rl_element == null)
        {
            return; // such an element not found
        }

        Element to_element = XMLTools.duplicateElement(to, (Element) rl_element, true);
        toElement.appendChild(to_element);
    }

    /**
     * serviceRackList is currently not editable in GLI - just copy it in from
     * config file and write it out again.
     */
    static private void doServiceRackList(Document to, Element from)
    {
        Element toElement = to.getDocumentElement();

        Node srl_element = XMLTools.getChildByTagName(from, StaticStrings.SERVICE_RACK_LIST_ELEMENT);
        if (srl_element == null)
        {
            return; // such an element not found
        }

        Element to_element = XMLTools.duplicateElement(to, (Element) srl_element, true);
        toElement.appendChild(to_element);
    }

    static private void convertServiceRackList(Document from, Document to)
    {
        Element toElement = to.getDocumentElement();

        Node srl_element = XMLTools.getChildByTagName(from.getDocumentElement(), StaticStrings.SERVICE_RACK_LIST_ELEMENT);
        if (srl_element == null)
        {
            return; // such an element not found
        }

        Element to_element = XMLTools.duplicateElement(to, (Element) srl_element, true);
        toElement.appendChild(to_element);
    }

    static private void doDefaultIndexLanguage(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        String elementNameFrom = StaticStrings.LANGUAGE_DEFAULT_INDEX_ELEMENT;
        String elementNameTo = StaticStrings.LANGUAGE_DEFAULT_ELEMENT;
        Node from_element = XMLTools.getChildByTagName(searchNode, elementNameFrom);
        if (from_element == null)
        {
            return; // such an element not found
        }

        Element to_element = to.createElement(elementNameTo);

        String name_str = ((Element) from_element).getAttribute(StaticStrings.NAME_ATTRIBUTE);
        to_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
        to_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

        toElement.appendChild(to_element);
    }

    //Handle 'indexOption' (i.e. casefold, stem etc). In the internal structure, the element is called 'IndexOption'
    static private void doIndexOption(Document to, Node searchNode)
    {
        Element toElement = to.getDocumentElement();
        //Node index_option_node = XMLTools.getChildByTagName(searchNode, StaticStrings.INDEXOPTION_STR);
        //if (index_option_node == null)
        //{
        //  return;
        //}
        //NodeList option_children = ((Element) index_option_node).getElementsByTagName(StaticStrings.OPTION_STR);
        NodeList option_children = ((Element) searchNode).getElementsByTagName(StaticStrings.INDEXOPTION_STR);
        int num_options = option_children.getLength();

        // for lucene, there is no 'indexOption'. We build a default 'indexOption' and 'assigned=false' in case the user switches to mg or mgpp
        if (num_options < 1)
        {
            Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
            index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
            index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.INDEXOPTIONS_STR);
            String[] option_str = { StaticStrings.CASEFOLD_OPTION_STR, StaticStrings.STEM_OPTION_STR };
            for (int i = 0; i < option_str.length; i++)
            {
                Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
                option_element.setAttribute(StaticStrings.NAME_STR, option_str[i]);
                index_option.appendChild(option_element);
            }
            appendProperly(toElement, index_option);
            return;
        }

        Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
        index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.INDEXOPTIONS_STR);

        for (int i = 0; i < num_options; i++)
        {
            String option_str = ((Element) option_children.item(i)).getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
            option_element.setAttribute(StaticStrings.NAME_STR, option_str);
            index_option.appendChild(option_element);
        }
        appendProperly(toElement, index_option);
    }

    static private Element doBuildType(Document to, String att_value)
    {

        //construct 'BuildType' element
        Element element = to.createElement(StaticStrings.BUILDTYPE_ELEMENT);
        element.setAttribute(StaticStrings.NAME_ATTRIBUTE, StaticStrings.BUILDTYPE_STR);
        element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, StaticStrings.ENGLISH_LANGUAGE_STR);
        element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        element.setAttribute(StaticStrings.SPECIAL_ATTRIBUTE, StaticStrings.TRUE_STR);

        XMLTools.setNodeText(element, att_value);

        return element;
    }

    static private Element doDatabaseType(Document to, String att_value)
    {

        //construct 'DatabaseType' element
        Element element = to.createElement(StaticStrings.DATABASETYPE_ELEMENT);
        element.setAttribute(StaticStrings.NAME_ATTRIBUTE, StaticStrings.DATABASETYPE_STR);
        element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, StaticStrings.ENGLISH_LANGUAGE_STR);
        element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        element.setAttribute(StaticStrings.SPECIAL_ATTRIBUTE, StaticStrings.TRUE_STR);

        XMLTools.setNodeText(element, att_value);

        return element;
    }

    // Convert 'description', 'smallicon' etc.
    static private void convertDisplayItemList(Document from, Document to)
    {
        Element displayItemList = to.createElement(StaticStrings.DISPLAYITEMLIST_STR);
        Element destination = to.getDocumentElement();

        // certain special collectionmeta elements should have different names
        // as displayItems in the collectionConfig.xml than they do in memory
        Map attributeMap = new HashMap(4);
        attributeMap.put(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR, StaticStrings.DESCRIPTION_STR);
        attributeMap.put(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR, StaticStrings.NAME_STR);
        attributeMap.put(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR, StaticStrings.SMALLICON_STR);
        attributeMap.put(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTION_STR, StaticStrings.ICON_STR);

        NodeList e_list = from.getDocumentElement().getElementsByTagName(StaticStrings.COLLECTIONMETADATA_ELEMENT);
        // if such elements don't exist, don't bother
        if (e_list != null)
        {

            for (int j = 0; j < e_list.getLength(); j++)
            {
                Element e = (Element) e_list.item(j);
                if (e.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                {
                    continue;
                }
                String text = XMLTools.getNodeText(e);
                String lang = e.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE);
                String key = e.getAttribute(StaticStrings.KEY_ATTRIBUTE);
                String dictionary = e.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);

                String name_value = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                String name_mapping = (String) attributeMap.get(name_value);
                if (name_mapping != null)
                {
                    name_value = name_mapping;
                }

                Element displayItem = constructElement(StaticStrings.DISPLAYITEM_STR, name_value, StaticStrings.LANG_STR, lang, text, StaticStrings.KEY_ATTRIBUTE, key, StaticStrings.DICTIONARY_ATTRIBUTE, dictionary, to);
                //Element displayItem = constructDisplayItem(e, to, name_value);
                displayItemList.appendChild(displayItem);
            }

        }
        destination.appendChild(displayItemList);
    }

    // This method creates a DisplayItem element of the type of 'to' by using the ingredients from the element 'e'
    static private Element constructDisplayItem(Element e, Document to, String name)
    {
        String lang_string = e.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE);
        String key_string = e.getAttribute(StaticStrings.KEY_ATTRIBUTE);
        String dictionary_string = e.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
        String text = XMLTools.getNodeText(e);

        Element displayItem = to.createElement(StaticStrings.DISPLAYITEM_STR);
        displayItem.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);

        if(!lang_string.equals("")) {
            displayItem.setAttribute(StaticStrings.LANG_STR, lang_string);
            XMLTools.setNodeText(displayItem, text);
        }
        if(!key_string.equals("")) {
            displayItem.setAttribute(StaticStrings.KEY_ATTRIBUTE, key_string);
            if(!dictionary_string.equals("")) {
            displayItem.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary_string);
            }
        }
        return displayItem;
    }

    static private Element constructDisplayItem(Element e, Document to)
    {   
        return constructDisplayItem(e, to, StaticStrings.NAME_ATTRIBUTE);
    }

  static private void convertSecurity(Document from, Document to)
    {
      Node security = XMLTools.getChildByTagNameIndexed(from.getDocumentElement(), StaticStrings.SECURITY_STR, 0);
      if (security != null)
        {
          Element to_element = XMLTools.duplicateElement(to, (Element) security, true);
          to.getDocumentElement().appendChild(to_element);
        
        }
    }
    static private void convertMetadataList(Document from, Document to)
    {
        Element metadataList = to.createElement(StaticStrings.METADATALIST_STR);
        Element destination = to.getDocumentElement();

        String[] ele_names = { StaticStrings.COLLECTIONMETADATA_CREATOR_ELEMENT, StaticStrings.COLLECTIONMETADATA_MAINTAINER_ELEMENT, StaticStrings.COLLECTIONMETADATA_PUBLIC_ELEMENT };
        String[] att_names = { StaticStrings.COLLECTIONMETADATA_CREATOR_STR, StaticStrings.COLLECTIONMETADATA_MAINTAINER_STR, StaticStrings.COLLECTIONMETADATA_PUBLIC_STR };
        for (int i = 0; i < ele_names.length; i++)
        {
            Element e = XMLTools.getNamedElement(from.getDocumentElement(), ele_names[i], StaticStrings.NAME_ATTRIBUTE, att_names[i]);
            if (e == null)
            {
                continue;
            }
            String text = XMLTools.getNodeText(e);
            Element metadata = to.createElement(StaticStrings.METADATA_STR);
            metadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, att_names[i]);
            metadata.setAttribute(StaticStrings.LANG_STR, StaticStrings.ENGLISH_LANGUAGE_STR);
            XMLTools.setNodeText(metadata, text);
            metadataList.appendChild(metadata);
        }

        destination.appendChild(metadataList);
    }

    // This method creates an element with the name 'element_name' of the type of 'to' by using the other three strings
    static private Element constructElement(String element_name, String name_value, String lang_att, String lang_value, String text, String key_att, String key, String dict_att, String dictionary, Document to)
    {
        Element e = to.createElement(element_name);
        e.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        e.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_value);

        if(!lang_value.equals("")) {
            e.setAttribute(lang_att, lang_value);
            XMLTools.setNodeText(e, text);
        }       
        if(!key.equals("")) {
            e.setAttribute(key_att, key);
            if(!dictionary.equals("")) {
            e.setAttribute(dict_att, dictionary);
            }
        }

        return e;
    }

    // Convert classify in the internal(i.e. Document from) to collectionconfig.xml (i.e. Document to)
    static private void convertClassifier(Document from, Document to)
    {
        Element browse_element = to.createElement(StaticStrings.BROWSE_STR);
        NodeList children = from.getDocumentElement().getElementsByTagName(StaticStrings.CLASSIFY_ELEMENT);

        int num_children = (children == null) ? 0 : children.getLength();

        if (num_children == 0)
        {
            return;
        }

        for (int i = 0; i < num_children; i++)
        {

            Element child = (Element) children.item(i);
            if (child.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }
            String str = child.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
            Element classifier_element = to.createElement(StaticStrings.CLASSIFIER_STR);
            classifier_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, str);

            NodeList option_children = child.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
            for (int j = 0; j < option_children.getLength(); j++)
            {
                Element el = (Element) option_children.item(j);
                if (el.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                {
                    continue;
                }
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                String value_str = XMLTools.getNodeText(el);

                if (name_str == null && value_str == null)
                {
                    continue;
                }
                Element option_element = to.createElement(StaticStrings.OPTION_STR);
                if (name_str != null && name_str.equals(StaticStrings.METADATA_STR))
                {

                    // The metadata argument is the fully qualified name of a metadata element, so if it contains a namespace, remove the extracted metadata namespace as the build process doesn't know about it.
                    String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                    value_str = "";
                    for (int k = 0; k <= values.length - 1; k++)
                    {
                        if (values[k].startsWith(StaticStrings.EXTRACTED_NAMESPACE) && values[k].indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
                        {
                            values[k] = values[k].substring(StaticStrings.EXTRACTED_NAMESPACE.length());
                        }
                        else
                        {
                            MetadataElement metadata_element = MetadataTools.getMetadataElementWithDisplayName(values[k]);
                            if (metadata_element != null)
                            {
                                values[k] = metadata_element.getFullName();
                            }
                        }
                        if (k < values.length - 1)
                        {
                            value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;
                        }
                        else
                        {
                            value_str = value_str + values[k];
                        }
                    }
                }

                if (!name_str.equals(""))
                {
                    if (!name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                    {
                        name_str = StaticStrings.MINUS_CHARACTER + name_str;
                    }
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }

                if (!value_str.equals(""))
                {
                    option_element.setAttribute(StaticStrings.VALUE_ATTRIBUTE, value_str);
                }

                classifier_element.appendChild(option_element);
            }

            //format element for this classifier
            Element e = (Element) XMLTools.getChildByTagName(child, StaticStrings.FORMAT_STR);

            if (e != null)
            {
                classifier_element.appendChild(convertFormat(to, e));
            }

            // Handling 'displayItem' element of this 'classifier' element - for now, just copy in and out so they don't get deleted
            NodeList di_children = child.getElementsByTagName(StaticStrings.DISPLAYITEM_STR);
            
            XMLTools.duplicateElementList(to, classifier_element, di_children, true);
            browse_element.appendChild(classifier_element);
        }

        //convert default classifier format
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.BROWSE_STR);
        browse_element.appendChild(convertFormat(to, e));

        to.getDocumentElement().appendChild(browse_element);
    }

    static private Element convertFormat(Document to, Element e)
    {
        String format_str = XMLTools.getNodeText(e);
        Element format = to.createElement(StaticStrings.FORMAT_STR);
        //XMLTools.copyAllChildren (format, e);
        XMLTools.setNodeText(format, format_str);
        return format;
    }

    //convert format statement for search
    static private void convertSearchFormat(Document from, Document to)
    {
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.SEARCH_STR);

        search.appendChild(convertFormat(to, e));

    }

    //convert format statement for display of the documents
    static private void convertDisplayFormat(Document from, Document to)
    {
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.DISPLAY_STR);
        if (e == null)
        {
            return;
        }
        Element display = to.createElement(StaticStrings.DISPLAY_STR);
        display.appendChild(convertFormat(to, e));
        to.getDocumentElement().appendChild(display);
    }

    // convert global format statement
    static private void convertGlobalFormat(Document from, Document to)
    {
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.GLOBAL_STR);

        to.getDocumentElement().appendChild(convertFormat(to, e));

    }

    // Convert plugins in the internal(i.e. Document from) to collectionconfig.xml (i.e. Document to)
    static private void convertPlugins(Document from, Document to)
    {
        Element import_element = to.createElement(StaticStrings.IMPORT_STR);
        Element plugin_list_element = to.createElement(StaticStrings.PLUGINLIST_STR);

        NodeList children = from.getDocumentElement().getElementsByTagName(StaticStrings.PLUGIN_ELEMENT);
        int num_children = (children == null) ? 0 : children.getLength();
        if (num_children == 0)
        {
            return;
        }

        for (int i = 0; i < num_children; i++)
        {

            Element child = (Element) children.item(i);
            if (child.getAttribute(StaticStrings.SEPARATOR_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
            {
                continue;
            }
            if (child.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }

            String str = child.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
            Element plugin_element = to.createElement(StaticStrings.PLUGIN_STR);
            plugin_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, str);

            NodeList option_children = child.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
            for (int j = 0; j < option_children.getLength(); j++)
            {
                Element el = (Element) option_children.item(j);
                if (!el.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
                {
                    continue;
                }
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                String value_str = XMLTools.getNodeText(el);

                if (name_str == null && value_str == null)
                {
                    continue;
                }
                Element option_element = to.createElement(StaticStrings.OPTION_STR);
                if (name_str != null && name_str.equals(StaticStrings.METADATA_STR))
                {

                    // The metadata argument is the fully qualified name of a metadata element, so if it contains a namespace, remove the extracted metadata namespace as the build process doesn't know about it, but ONLY if it is not embedded metadata (e.g. ex.dc.*)
                    String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                    value_str = "";
                    for (int k = 0; k <= values.length - 1; k++)
                    {
                        if (values[k].startsWith(StaticStrings.EXTRACTED_NAMESPACE) && values[k].indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
                        {
                            values[k] = values[k].substring(StaticStrings.EXTRACTED_NAMESPACE.length());
                        }

                        if (k < values.length - 1)
                        {
                            value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;
                        }
                        else
                        {
                            value_str = value_str + values[k];
                        }
                    }
                }

                if (!name_str.equals(""))
                {
                    if (!name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                    {
                        name_str = StaticStrings.MINUS_CHARACTER + name_str;
                    }
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }

                if (!value_str.equals(""))
                {
                    option_element.setAttribute(StaticStrings.VALUE_ATTRIBUTE, value_str);
                }

                plugin_element.appendChild(option_element);
            }//for loop ends

            plugin_list_element.appendChild(plugin_element);
        }//for loop ends

        import_element.appendChild(plugin_list_element);

        //do the plugout element (used by building flax collections)
        Node plugout = XMLTools.getChildByTagNameIndexed(from.getDocumentElement(), PLUGOUT_ELEMENT, 0);
        if (plugout != null)
        {
            Element to_element = XMLTools.duplicateElement(to, (Element) plugout, true);
            import_element.appendChild(to_element);
        }

        to.getDocumentElement().appendChild(import_element);
    }

    //Handle 'searchType' of collectionConfig.xml. In the internal structure, its also called 'searchType', eg. plain, form
    static private void convertSearchType(Document from, Document to)
    {
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.SEARCHTYPE_ELEMENT);//searchType

        if (e == null)
        {
            return;
        }
        String searchtype_str = XMLTools.getNodeText(e).trim();
        //Get the 'search' element from 'to'
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

        String[] types = searchtype_str.split(",");
        for (int i = 0; i < types.length; i++)
        {
            Element search_type_element = to.createElement(StaticStrings.SEARCHTYPE_ELEMENT);
            search_type_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, types[i]);
            search.appendChild(search_type_element);
        }
    }

    static private void convertBuildType(Document from, Document to)
    {
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.BUILDTYPE_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.BUILDTYPE_STR);
        if (e == null)
        {
            return;
        }
        String indexer = XMLTools.getNodeText(e);
        Element search = to.createElement(StaticStrings.SEARCH_STR);
        search.setAttribute(StaticStrings.TYPE_ATTRIBUTE, indexer);
        to.getDocumentElement().appendChild(search);
    }

    static private void convertDatabaseType(Document from, Document to)
    {
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.DATABASETYPE_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.DATABASETYPE_STR);
        if (e == null)
        {
            return;
        }
        String db = XMLTools.getNodeText(e);
        Element dbtype = to.createElement(StaticStrings.INFODB_STR);
        dbtype.setAttribute(StaticStrings.TYPE_ATTRIBUTE, db);
        to.getDocumentElement().appendChild(dbtype);
    }

    static private void convertDefaultIndex(Document from, Document to, Element search)
    {
        Element source = from.getDocumentElement();

        Element default_index_element = (Element) XMLTools.getChildByTagName(source, StaticStrings.INDEX_DEFAULT_ELEMENT);
        if (default_index_element == null)
        {
            return;
        }

        String indexer = search.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
        String level_str = default_index_element.getAttribute(StaticStrings.LEVEL_ATTRIBUTE);
        // Debugging purposes
        if (level_str.equals("") && indexer.equals(StaticStrings.MG_STR))
        {
            System.out.println("Bug: DefaultIndex should have its level attribute not empty.");
        }

        NodeList content_elements = default_index_element.getElementsByTagName(StaticStrings.CONTENT_ELEMENT);
        int content_elements_length = content_elements.getLength();

        // Don't output anything if no indexes are set
        if (content_elements_length == 0)
        {
            return;//
        }

        String index_str = "";

        if (indexer.equals(StaticStrings.MG_STR))
        {
            //combine level with indexes
            index_str = level_str + StaticStrings.COLON_CHARACTER;
        }
        else
        { //for mgpp/lucene, just take index
            //do nothing
        }

        for (int k = 0; k < content_elements_length; k++)
        {
            Element content_element = (Element) content_elements.item(k);
            if (content_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }
            String name_str = content_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);

            if (name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && name_str.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
            {
                name_str = name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
            }

            index_str = index_str + name_str;

            // Make it comma separated string
            if (k < content_elements_length - 1)
            {
                index_str = index_str + StaticStrings.COMMA_CHARACTER;
            }
            content_element = null;
        }//for loop ends

        Element default_index = to.createElement(StaticStrings.INDEX_DEFAULT_ELEMENT_LOWERCASE);
        default_index.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_str);
        search.appendChild(default_index);

    }

    static private void convertSubcollection(Document from, Document to)
    {
        Element source = from.getDocumentElement();
        //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

        // Get the Subcollection element from the internal structure
        NodeList subcollection_elements = source.getElementsByTagName(StaticStrings.SUBCOLLECTION_ELEMENT);
        if (subcollection_elements == null)
        {
            return;
        }
        int subcollection_elements_length = subcollection_elements.getLength();

        if (subcollection_elements_length == 0)
        { // no
            return;
        }

        for (int j = 0; j < subcollection_elements_length; j++)
        {

            Element e = (Element) subcollection_elements.item(j);
            if (e.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }
            String content = e.getAttribute(StaticStrings.CONTENT_ATTRIBUTE);
            String name = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String options = e.getAttribute(StaticStrings.OPTIONS_ATTRIBUTE);
            String type = e.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
            String text = XMLTools.getNodeText(e);

            String filter = "";
            if (type.equals(StaticStrings.EXCLUDE_STR))
            {
                filter = StaticStrings.EXCLAMATION_CHARACTER;
            }

            if (content.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && content.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
            {
                content = content.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
            }
            filter = filter + content + StaticStrings.SEPARATOR_CHARACTER + text;
            if (options != null && options != "")
            {
                filter = filter + StaticStrings.SEPARATOR_CHARACTER + options;
            }
            Element subcollection = to.createElement(StaticStrings.SUBCOLLECTION_STR);
            subcollection.setAttribute(StaticStrings.FILTER_ATTRIBUTE, filter);
            subcollection.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);

            search.appendChild(subcollection);
        }
    }

    static private void convertSubcollectionIndexes(Document from, Document to)
    {
        Element source = from.getDocumentElement();
        //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

        // Get the SubcollectionIndexes element from the internal structure
        Element subcollection_indexes = (Element) XMLTools.getChildByTagName(source, StaticStrings.SUBCOLLECTION_INDEXES_ELEMENT);
        if (subcollection_indexes == null)
        {
            return;
        }
        NodeList index_elements = subcollection_indexes.getElementsByTagName(StaticStrings.INDEX_ELEMENT);
        int index_elements_length = index_elements.getLength();

        if (index_elements_length == 0)
        { // no indexes
            return;
        }

        for (int j = 0; j < index_elements_length; j++)
        {
            Element index_element = (Element) index_elements.item(j);
            if (index_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }

            Element index = to.createElement(StaticStrings.SUBCOLLECTION_INDEX_ELEMENT);

            String index_value = "";

            NodeList content_elements = index_element.getElementsByTagName(StaticStrings.CONTENT_ELEMENT);
            int content_elements_length = content_elements.getLength();

            for (int k = 0; k < content_elements_length; k++)
            {
                Element content_element = (Element) content_elements.item(k);
                if (content_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                {
                    continue;
                }
                String name_str = content_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                index_value += name_str;
                // Make it comma separated string
                if (k < content_elements_length - 1)
                {
                    index_value += StaticStrings.COMMA_CHARACTER;
                }
                content_element = null;
            }//for loop ends

            index.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_value);

            // Now constructing 'displayItem' element for this 'indexSubcollection' element
            // from the collectionmetadata element
            ArrayList collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.COLLECTIONMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, index_value);

            if (collectionmetadata_list != null)
            {

                for (int k = 0; k < collectionmetadata_list.size(); k++)
                {
                    Element collectionmetadata = (Element) collectionmetadata_list.get(k);
                    if (collectionmetadata.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                    {
                        continue;
                    }
                    Element displayItem = constructDisplayItem(collectionmetadata, to);
                    index.appendChild(displayItem);
                }
            }

            search.appendChild(index);

        } //for loop ends
    }

    static private void convertLanguages(Document from, Document to)
    {
        Element source = from.getDocumentElement();
        //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

        // Get the Languages element from the internal structure
        Element languages = (Element) XMLTools.getChildByTagName(source, StaticStrings.LANGUAGES_ELEMENT);
        if (languages == null)
        {
            return;
        }
        NodeList language_elements = languages.getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
        int language_elements_length = language_elements.getLength();

        if (language_elements_length == 0)
        {
            return;
        }

        for (int j = 0; j < language_elements_length; j++)
        {
            Element element = (Element) language_elements.item(j);
            if (element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }

            // Create indexLanguage element
            Element index_language = to.createElement(StaticStrings.LANGUAGE_INDEX_ELEMENT);

            String name_str = element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            index_language.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);

            // Now constructing 'displayItem' element for this 'indexLanguage' element
            // from the collectionmetadata element
            ArrayList collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.COLLECTIONMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, name_str);

            if (collectionmetadata_list != null)
            {

                for (int k = 0; k < collectionmetadata_list.size(); k++)
                {
                    Element collectionmetadata = (Element) collectionmetadata_list.get(k);
                    if (collectionmetadata.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                    {
                        continue;
                    }
                    Element displayItem = constructDisplayItem(collectionmetadata, to);
                    index_language.appendChild(displayItem);
                }
            }

            search.appendChild(index_language);

        } //for loop ends

        // Convert DefaultLanguage
        // Get the DefaultLanguage element from the internal structure
        Element default_language = (Element) XMLTools.getChildByTagName(source, StaticStrings.LANGUAGE_DEFAULT_ELEMENT);
        if (default_language != null)
        {
            String lang_name = default_language.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element default_index_language = to.createElement(StaticStrings.LANGUAGE_DEFAULT_INDEX_ELEMENT);
            default_index_language.setAttribute(StaticStrings.NAME_ATTRIBUTE, lang_name);
            search.appendChild(default_index_language);
        }
        // Convert LanguageMetadata
        // Get the LanguageMetadata element from the internal structure
        Element language_metadata = (Element) XMLTools.getChildByTagName(source, StaticStrings.LANGUAGE_METADATA_ELEMENT);
        if (language_metadata != null)
        {
            String meta_name = language_metadata.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element language_meta = to.createElement(StaticStrings.LANGUAGE_METADATA_ELEMENT_STR);
            if (meta_name.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && meta_name.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
            {
                meta_name = meta_name.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
            }
            language_meta.setAttribute(StaticStrings.NAME_ATTRIBUTE, meta_name);
            search.appendChild(language_meta);
        }
    }

    //convert indexes and their displayItems, which go in 'search' element in collectionConfig.xml
    //parameter 'to' is the document to be saved as collectionConfig.xml
    //parameter 'from' is the internal xml structure
    static private void convertIndex(Document from, Document to)
    {
        Element source = from.getDocumentElement();
        //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

        //THere are two sets of indexes elements, find the one which is assigned 'true'
        Element indexes = XMLTools.getNamedElement(source, StaticStrings.INDEXES_ELEMENT, StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        if (indexes == null)
        {
            return;
        }
        NodeList index_elements = indexes.getElementsByTagName(StaticStrings.INDEX_ELEMENT);
        int index_elements_length = index_elements.getLength();

        if (index_elements_length == 0)
        { // no indexes
            return;
        }

        //find out it's mg or mgpp/lucene
        String mg = search.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
        boolean mg_indexer = false;
        if (mg.equals(StaticStrings.MG_STR))
        {
            mg_indexer = true;//it's mg, then the level is set as attribute of
        }
        if (mg_indexer == false)
        {
            // It's mgpp. Construct 'level' and 'defaultLevel' elements separately.
            convertLevels(from, to, search);
        }

        for (int j = 0; j < index_elements_length; j++)
        {
            Element index_element = (Element) index_elements.item(j);
            if (index_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
                continue;
            }

            Element index_ele = to.createElement(StaticStrings.INDEX_LOW_STR);//index

            // Used for creating displayItem for this element 'index_ele' further below
            // full_index_names contain 'ex.'
            String full_index_name = "";
            String level_str = "";

            StringBuffer index_value = new StringBuffer();
            if (mg_indexer == true)
            {
                // For mg indexer, there is a 'level' attribute in the index element of the internal structure
                // But mgpp/lucene don't
                level_str = index_element.getAttribute(StaticStrings.LEVEL_ATTRIBUTE);
                if (level_str.length() > 0)
                {
                    index_value.append(level_str).append(StaticStrings.COLON_CHARACTER);
                    //index_value = index_value.StaticStrings.COLON_CHARACTER;
                }
            }

            NodeList content_elements = index_element.getElementsByTagName(StaticStrings.CONTENT_ELEMENT);
            int content_elements_length = content_elements.getLength();

            for (int k = 0; k < content_elements_length; k++)
            {
                Element content_element = (Element) content_elements.item(k);
                if (content_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                {
                    continue;
                }
                String name_str = content_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);

                full_index_name = full_index_name + name_str;
                if (k < content_elements_length - 1)
                {
                    full_index_name = full_index_name + StaticStrings.COMMA_CHARACTER;
                }

                if (name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && name_str.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
                {
                    name_str = name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
                }

                index_value.append(name_str);
                name_str = null;
                // Make it comma separated string
                if (k < content_elements_length - 1)
                {
                    index_value.append(StaticStrings.COMMA_CHARACTER);
                }
                content_element = null;
            }//for loop ends

            String temp_str = index_value.toString();
            index_ele.setAttribute(StaticStrings.NAME_ATTRIBUTE, temp_str);

            // Now constructing 'displayItem' element for this 'index_ele' element
            // The index names in the collectionmetadata elements in the internal structure are not the names that
            // are used in the content elements (i.e. ex.Source or dc.Subject and keywords), but the names that are
            // in the configuration files (i.e. Source or dc.Subject)
            ArrayList collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.COLLECTIONMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, temp_str);

            if (collectionmetadata_list == null)
            {
                //try the full name, i.e. with 'ex.'
                if (mg_indexer == true)
                {
                    // but first append level info if we are mg
                    full_index_name = level_str + StaticStrings.COLON_CHARACTER + full_index_name;
                }
                collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.COLLECTIONMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, full_index_name);
            }

            if (collectionmetadata_list != null)
            {

                for (int k = 0; k < collectionmetadata_list.size(); k++)
                {
                    Element collectionmetadata = (Element) collectionmetadata_list.get(k);
                    if (collectionmetadata.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                    {
                        continue;
                    }
                    Element displayItem = constructDisplayItem(collectionmetadata, to);

                    index_ele.appendChild(displayItem);
                }
            }

            // deal with any <option name='' value=''> children of this index
            // e.g. <option name='solrfieldtype' value='text_es'>
            NodeList option_children = index_element.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
            for (int k = 0; k < option_children.getLength(); k++)
            {
                Element el = (Element) option_children.item(k);
                if (el.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                {
                    continue;
                }
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                String value_str = XMLTools.getNodeText(el);

                if (name_str == null && value_str == null)
                {
                    continue;
                }
                Element option_element = to.createElement(StaticStrings.OPTION_STR);
                if (!name_str.equals("")) {
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }
                if (!value_str.equals("")) { // or no value attribute without name attribute?
                    option_element.setAttribute(StaticStrings.VALUE_ATTRIBUTE, value_str);      
                }
                index_ele.appendChild(option_element);
            }

            search.appendChild(index_ele);

        } //for loop ends

        //Convert default index
        convertDefaultIndex(from, to, search);
        convertIndexOptions(from, to, search);
    }

        //convert <facet> and <sort> and <defaultSort> elements 
        //which go in 'search' element in collectionConfig.xml
    //parameter 'to' is the document to be saved as collectionConfig.xml
    //parameter 'from' is the internal xml structure
  // for lucene and solr collections
  // just a straight copy at present
    // Note that GLI at present does not allow the user to set or modify the solr-specific elements 
    // <sort> and <facet>. This function, and its inverse doSolrFacetsAndSorts, is only here to preserve such
    // elements if they already exist in the collectionConfig.xml. At present they need to be manually added there.
    static private void convertSolrFacetsAndSorts(Document from, Document to)
    {
        Element source = from.getDocumentElement();
        //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
        Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

        //There are two sets of <Solr> elements, find the one which is assigned 'true'   ---  is this true???
        Element solr = XMLTools.getNamedElement(source, StaticStrings.SOLR_ELEMENT, StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        if (solr == null)
        {
          return;
        }
        NodeList sort_elements = solr.getElementsByTagName(StaticStrings.SORT_LOW_STR);
        int sort_elements_length = sort_elements.getLength();

        for (int j = 0; j < sort_elements_length; j++)
        {
          Element sort_ele = (Element) sort_elements.item(j);
          search.appendChild(to.importNode(sort_ele, true));
        }
        Element sort_def = (Element)XMLTools.getChildByTagName(solr, StaticStrings.SORT_DEFAULT_ELEMENT);
        if (sort_def != null) {
          search.appendChild(to.importNode(sort_def, true));
        }
        NodeList facet_elements = solr.getElementsByTagName(StaticStrings.FACET_LOW_STR); 
        int facet_elements_length = facet_elements.getLength();
        for (int j = 0; j < facet_elements_length; j++)
          {
            Element facet_ele = (Element) facet_elements.item(j);
            search.appendChild(to.importNode(facet_ele, true));
          }

    
    }


    // Convert levels for mgpp/lucene. This method is called by converIndex() when mgpp indexer is detected.
    static private void convertLevels(Document from, Document to, Element search)
    {
        Element source = from.getDocumentElement();
        Element index_option = XMLTools.getNamedElement(source, StaticStrings.INDEXOPTIONS_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.LEVELS_STR);
        if (index_option == null)
        {
            return;
        }
        //Debugging purposes
        if (index_option.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
        {
            DebugStream.println("For mgpp, there should be an IndexOption element for levels which is assigned 'true': possible bug.");
        }

        NodeList option_elements = index_option.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
        int num_elements = option_elements.getLength();

        // Don't output anything if no indexes are set
        if (num_elements == 0)
        {
            return;//
        }

        for (int k = 0; k < num_elements; k++)
        {
            Element e = (Element) option_elements.item(k);
            String name_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element level_element = to.createElement(StaticStrings.LEVEL_ELEMENT);
            level_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);

            //Now construct displayItem for this level element from collectionmetadata
            ArrayList collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.COLLECTIONMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, name_str);

            if (collectionmetadata_list != null)
            {

                for (int j = 0; j < collectionmetadata_list.size(); j++)
                {
                    Element collectionmetadata = (Element) collectionmetadata_list.get(j);

                    Element displayItem = constructDisplayItem(collectionmetadata, to);
                    level_element.appendChild(displayItem);
                }
            }
            search.appendChild(level_element);
        }

        //Convert default level
        Element default_index_option = XMLTools.getNamedElement(source, StaticStrings.INDEXOPTION_DEFAULT_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.LEVEL_DEFAULT_STR);
        if (default_index_option == null)
        {
            return;
        }
        Element default_level = to.createElement(StaticStrings.LEVEL_DEFAULT_ELEMENT);
        String default_level_str = default_index_option.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
        default_level.setAttribute(StaticStrings.NAME_ATTRIBUTE, default_level_str);
        search.appendChild(default_level);

    }

    // Convert indexoptions for mg/mgpp/lucene. This method is called by convertIndex().
    static private void convertIndexOptions(Document from, Document to, Element search)
    {
        Element source = from.getDocumentElement();
        Element index_option = XMLTools.getNamedElement(source, StaticStrings.INDEXOPTIONS_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.INDEXOPTIONS_STR);
        if (index_option == null)
        {
            return;
        }
        //Debugging purposes
        if (index_option.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
        {
            DebugStream.println("There should be an IndexOption element which is assigned 'true': possible bug.");

            // for lucene and solr collections, don't write out stemming, casefolding and accentfolding indexOptions
            // since they are not meant to be editable: stemming and casefolding are always on in lucene
            String buildtype_value = search.getAttribute(StaticStrings.TYPE_ATTRIBUTE); //might be mg|mgpp|lucene|solr
            if(buildtype_value.equalsIgnoreCase("solr") || buildtype_value.equalsIgnoreCase("lucene")) {
                return;
            }
        } 
        //Element indexOptionEl = to.createElement(StaticStrings.INDEXOPTION_STR);
        NodeList option_elements = index_option.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
        int num_elements = option_elements.getLength();
        // Don't output anything if no index
        if (num_elements == 0)
        {
            return;//
        }
        //search.appendChild(indexOptionEl);

        for (int k = 0; k < num_elements; k++)
        {
            Element e = (Element) option_elements.item(k);
            String name_att = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            Element optionEl = to.createElement(StaticStrings.INDEXOPTION_STR);
            optionEl.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_att);
            // default value?? on/off
            //indexOptionEl.appendChild(optionEl);
            search.appendChild(optionEl);
        }

    }

    // handle top level elements which GLI knows nothing about
    // we store them internally in a Unknown element for easy access when
    // we write them out.
    static private void doUnknownElements(Document to, Element from)
    {
        Element toElement = to.getDocumentElement();
        Element unknownElement = to.createElement(StaticStrings.UNKNOWN_ELEMENT);
        toElement.appendChild(unknownElement);

        Node child = from.getFirstChild();
        while (child != null)
        {
            if (child.getNodeType() == Node.ELEMENT_NODE && !known_element_names.contains(child.getNodeName()))
            {
                unknownElement.appendChild(XMLTools.duplicateElement(to, (Element) child, true));
            }
            child = child.getNextSibling();
        }

    }

    // just copy all children of Unknown element into output doc.
    static private void convertUnknownElements(Document from, Document to)
    {

        Element toElement = to.getDocumentElement();
        Node unknown_element = XMLTools.getChildByTagName(from.getDocumentElement(), StaticStrings.UNKNOWN_ELEMENT);

        Node child = unknown_element.getFirstChild();
        while (child != null)
        {
            Element to_element = XMLTools.duplicateElement(to, (Element) child, true);
            toElement.appendChild(to_element);

            child = child.getNextSibling();
        }

    }

    // Append the element son to the element mother in the appropriate position.
    static public void appendProperly(Element mother, Element son)
    {
        if (son == null)
            return;

        Node reference_node = findInsertionPoint(mother, son);
        if (reference_node != null)
        {
            mother.insertBefore(son, reference_node);
        }
        else
        {
            mother.appendChild(son);
        }
    }

    /**
     * Find the best insertion position for the given DOM Element
     * 'target_element' in the DOM Element 'document_element'. This should try
     * to match command tag, and if found should then try to group by name or
     * type (eg CollectionMeta), or append to end is no such grouping exists (eg
     * Plugins). Failing a command match it will check against the command order
     * for the best insertion location.
     * 
     * @param target_element
     *            the command Element to be inserted
     * @return the Element which the given command should be inserted before, or
     *         null to append to end of list
     */
    static public Node findInsertionPoint(Element document_element, Element target_element)
    {
        ///ystem.err.println("Find insertion point: " + target_element.getNodeName());
        String target_element_name = target_element.getNodeName();

        // Try to find commands with the same tag.
        NodeList matching_elements = document_element.getElementsByTagName(target_element_name);
        // If we found matching elements, then we have our most likely insertion location, so check within for groupings
        if (matching_elements.getLength() != 0)
        {
            ///ystem.err.println("Found matching elements.");
            // Only CollectionMeta are grouped.
            if (target_element_name.equals(StaticStrings.COLLECTIONMETADATA_ELEMENT))
            {
                ///ystem.err.println("Dealing with collection metadata");
                // Special case: CollectionMeta can be added at either the start or end of a collection configuration file. However the start position is reserved for special metadata, so if no non-special metadata can be found we must append to the end.
                // So if the command to be added is special add it immediately after any other special command
                if (target_element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
                {
                    int index = 0;
                    Element matched_element = (Element) matching_elements.item(index);
                    Element sibling_element = (Element) matched_element.getNextSibling();
                    while (sibling_element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
                    {
                        index++;
                        matched_element = (Element) matching_elements.item(index);
                        sibling_element = (Element) matched_element.getNextSibling();
                    }
                    if (sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT))
                    {
                        Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
                        document_element.insertBefore(newline_element, sibling_element);
                    }
                    return sibling_element;
                }
                // Otherwise try to find a matching 'name' and add after the last one in that group.
                else
                {
                    int index = 0;
                    target_element_name = target_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                    boolean found = false;
                    // Skip all of the special metadata
                    Element matched_element = (Element) matching_elements.item(index);
                    while (matched_element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
                    {
                        index++;
                        matched_element = (Element) matching_elements.item(index);
                    }
                    // Begin search
                    while (!found && matched_element != null)
                    {
                        if (matched_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equals(target_element_name))
                        {
                            found = true;
                        }
                        else
                        {
                            index++;
                            matched_element = (Element) matching_elements.item(index);
                        }
                    }
                    // If we found a match, we need to continue checking until we find the last name match.
                    if (found)
                    {
                        index++;
                        Element previous_sibling = matched_element;
                        Element sibling_element = (Element) matching_elements.item(index);
                        while (sibling_element != null && sibling_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equals(target_element_name))
                        {
                            previous_sibling = sibling_element;
                            index++;
                            sibling_element = (Element) matching_elements.item(index);
                        }
                        // Previous sibling now holds the command immediately before where we want to add, so find its next sibling and add to that. In this one case we can ignore new lines!
                        return previous_sibling.getNextSibling();
                    }
                    // If not found we just add after last metadata element
                    else
                    {
                        Element last_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
                        return last_element.getNextSibling();
                    }
                }

            }
            else
            {
                ///ystem.err.println("Not dealing with collection meta.");
                Element matched_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
                // One final quick test. If the matched element is immediately followed by a NewLine command, then we insert another NewLine after the matched command, then return the NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
                Node sibling_element = matched_element.getNextSibling();
                if (sibling_element != null && sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT))
                {
                    Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
                    document_element.insertBefore(newline_element, sibling_element);
                }
                return sibling_element; // Note that this may be null
            }
        }
        ///ystem.err.println("No matching elements found.");
        // Locate where this command is in the ordering
        int command_index = -1;
        for (int i = 0; command_index == -1 && i < CollectionConfiguration.COMMAND_ORDER.length; i++)
        {
            if (CollectionConfiguration.COMMAND_ORDER[i].equals(target_element_name))
            {
                command_index = i;
            }
        }
        ///ystem.err.println("Command index is: " + command_index);
        // Now move forward, checking for existing elements in each of the preceeding command orders.
        int preceeding_index = command_index - 1;
        ///ystem.err.println("Searching before the target command.");
        while (preceeding_index >= 0)
        {
            matching_elements = document_element.getElementsByTagName(CollectionConfiguration.COMMAND_ORDER[preceeding_index]);
            // If we've found a match
            if (matching_elements.getLength() > 0)
            {
                // We add after the last element
                Element matched_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
                // One final quick test. If the matched element is immediately followed by a NewLine command, then we insert another NewLine after the matched command, then return the NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
                Node sibling_element = matched_element.getNextSibling();
                if (sibling_element != null && sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT))
                {
                    Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
                    document_element.insertBefore(newline_element, sibling_element);
                }
                return sibling_element; // Note that this may be null
            }
            preceeding_index--;
        }
        // If all that fails, we now move backwards through the commands
        int susceeding_index = command_index + 1;
        ///ystem.err.println("Searching after the target command.");
        while (susceeding_index < CollectionConfiguration.COMMAND_ORDER.length)
        {
            matching_elements = document_element.getElementsByTagName(CollectionConfiguration.COMMAND_ORDER[susceeding_index]);
            // If we've found a match
            if (matching_elements.getLength() > 0)
            {
                // We add before the first element
                Element matched_element = (Element) matching_elements.item(0);
                // One final quick test. If the matched element is immediately preceeded by a NewLine command, then we insert another NewLine before the matched command, then return this new NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
                Node sibling_element = matched_element.getPreviousSibling();
                if (sibling_element != null && sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT))
                {
                    Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
                    document_element.insertBefore(newline_element, sibling_element);
                }
                return sibling_element; // Note that this may be null
            }
            susceeding_index++;
        }
        // Well. Apparently there are no other commands in this collection configuration. So append away...
        return null;
    }

    // From collectionConfig.xml to internal structure:add 'ex.' namespace (if none).
    // From internal structure to collectionConfig.xml:always peel off 'ex.' namespace (if any), except for format statement
    //This method parses 'xml_file_doc' into 'dOc'
    static public void parse(File xml_file, Document dOc)
    {

        Document xml_file_doc = XMLTools.parseXMLFile(xml_file);
        Element fromElement = xml_file_doc.getDocumentElement();
        Element toElement = dOc.getDocumentElement();

        // security element. For now, we just save as is, as you can't edit this in GLI.
        Node securityNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.SECURITY_STR,0);
        // just save for now
        if (securityNode != null) {
          Element to_element = XMLTools.duplicateElement(dOc, (Element) securityNode, true);
          toElement.appendChild(to_element);
        }
        // It's deliberately set that 'creator', 'maintainer', and 'public' are only in English (as they are just names).
        // So the following ArrayList have only one element.
        Node metadataListNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.METADATALIST_STR, 0);
        if (metadataListNode != null)
        {
            ArrayList creator = doMetadataList(dOc, metadataListNode, StaticStrings.COLLECTIONMETADATA_CREATOR_ELEMENT, StaticStrings.COLLECTIONMETADATA_CREATOR_STR);
            ArrayList maintainer = doMetadataList(dOc, metadataListNode, StaticStrings.COLLECTIONMETADATA_MAINTAINER_ELEMENT, StaticStrings.COLLECTIONMETADATA_MAINTAINER_STR);
            ArrayList is_public = doMetadataList(dOc, metadataListNode, StaticStrings.COLLECTIONMETADATA_PUBLIC_ELEMENT, StaticStrings.COLLECTIONMETADATA_PUBLIC_STR);

            appendArrayList(toElement, creator);
            appendArrayList(toElement, maintainer);
            appendArrayList(toElement, is_public);
        }

        Node databaseNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.INFODB_STR, 0);
        String databasetype_value = "gdbm";
        if (databaseNode != null)
        {
            databasetype_value = ((Element) databaseNode).getAttribute(StaticStrings.TYPE_ATTRIBUTE);//might be gdbm|jdbm|sqlite OR not yet set (in which case it should default to gdbm)
        }

        Element databasetype = doDatabaseType(dOc, databasetype_value);
        appendProperly(toElement, databasetype);

        Node searchNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.SEARCH_STR, 0);
        String buildtype_value = ((Element) searchNode).getAttribute(StaticStrings.TYPE_ATTRIBUTE);//might be mg|mgpp|lucene
        Element buildtype = doBuildType(dOc, buildtype_value);
        appendProperly(toElement, buildtype);

        Node importNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.IMPORT_STR, 0);
        if (importNode == null)
        {
            System.out.println("There is no content in the 'import' block.");
        }
        if (importNode != null)
        {
            //do plugin list nodes
            Node pluginListNode = XMLTools.getChildByTagNameIndexed((Element) importNode, StaticStrings.PLUGINLIST_STR, 0);
            if (pluginListNode == null)
            {
                System.out.println("There is no pluginlist set.");
            }
            if (pluginListNode != null)
            {

                doPlugins(dOc, pluginListNode);
            }

            //do the plugout element (used by building flax collections)
            Node plugout = XMLTools.getChildByTagNameIndexed((Element) importNode, PLUGOUT_ELEMENT, 0);
            if (plugout != null)
            {
                Element to_element = XMLTools.duplicateElement(dOc, (Element) plugout, true);
                toElement.appendChild(to_element);
            }
        }

        Node browseNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.BROWSE_STR, 0);
        if (browseNode != null)
        {
            if (browseNode == null)
            {
                System.out.println("There is no classifier.");
            }
            doClassifiers(dOc, browseNode);
        }

        Node displayItemListNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.DISPLAYITEMLIST_STR, 0);
        if (displayItemListNode != null)
        {
            ArrayList description = doDisplayItemList(dOc, displayItemListNode, StaticStrings.DESCRIPTION_STR, StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR);
            ArrayList smallicon = doDisplayItemList(dOc, displayItemListNode, StaticStrings.SMALLICON_STR, StaticStrings.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR);
            ArrayList icon = doDisplayItemList(dOc, displayItemListNode, StaticStrings.ICON_STR, StaticStrings.COLLECTIONMETADATA_ICONCOLLECTION_STR);
            ArrayList name = doDisplayItemList(dOc, displayItemListNode, StaticStrings.NAME_STR, StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR);

            appendArrayList(toElement, description);
            appendArrayList(toElement, smallicon);
            appendArrayList(toElement, icon);
            appendArrayList(toElement, name);
        }

        if (buildtype_value.equalsIgnoreCase("mg"))
        {
            doMGIndexes(dOc, searchNode);
        }
        else
        {
            doMGPPIndexes(dOc, searchNode);
        }

        if(buildtype_value.equalsIgnoreCase("solr") || buildtype_value.equalsIgnoreCase("lucene")) {
            doSolrFacetsAndSorts(dOc, searchNode); // <facet><displayItem /></facet> and <sort><displayItem /></sort>
            // lucene will only have sort elements
        }

        doDefaultIndex(dOc, searchNode);
        doDefaultLevel(dOc, searchNode);
        doLevel(dOc, searchNode);
        doIndexOption(dOc, searchNode);
        doSubcollection(dOc, searchNode);
        doIndexSubcollection(dOc, searchNode);
        doIndexLanguage(dOc, searchNode);
        doDefaultIndexLanguage(dOc, searchNode);
        doLanguageMetadata(dOc, searchNode);
        doSearchType(dOc, searchNode);
        doSearchFormat(dOc, searchNode);
        doGlobalFormat(dOc, fromElement);
        doDisplayFormat(dOc, fromElement);
        doReplaceListRef(dOc, fromElement);
        doReplaceList(dOc, fromElement);
        doServiceRackList(dOc, fromElement);
        doUnknownElements(dOc, fromElement);
        // the official displayItems in the displayItemList element have already been handled above
        // and created as collectionmetadata elements in the dOc object
        // Now we add in all the *other* (remaining) displayItems as collectionmeta elements
        NodeList collectionMetadataList = dOc.getDocumentElement().getElementsByTagName(StaticStrings.COLLECTIONMETADATA_ELEMENT);
        Set setOfUniqueColMetaNames = new HashSet();
        for (int i = 0; i < collectionMetadataList.getLength(); i++)
        {
            Element colMeta = (Element) collectionMetadataList.item(i);
            String name = colMeta.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            setOfUniqueColMetaNames.add(name);
        }

        if (displayItemListNode != null)
        {
            NodeList nl = ((Element) displayItemListNode).getElementsByTagName(StaticStrings.DISPLAYITEM_STR);

            // make a list of all unique attribute names that are specifically not
            // description, smallicon, icon and name, since these are already processed above
            Set setOfUniqueDisplayItemNames = new LinkedHashSet();
            for (int i = 0; i < nl.getLength(); i++)
            {
                Element displayItem = (Element) nl.item(i);
                String name = displayItem.getAttribute(StaticStrings.NAME_ATTRIBUTE);

                if (name.equals(""))
                    continue; // no name attribute
                if (setOfUniqueColMetaNames.contains(name))
                    continue;

                if (name.equals(StaticStrings.DESCRIPTION_STR))
                    continue;
                if (name.equals(StaticStrings.SMALLICON_STR))
                    continue;
                if (name.equals(StaticStrings.ICON_STR))
                    continue;
                if (name.equals(StaticStrings.NAME_STR))
                    continue;
                // don't add displayItems that are handled by the indexers, etc. E.g. document:ex.Title 
                if (name.indexOf(":") != -1)
                    continue;

                // otherwise
                setOfUniqueDisplayItemNames.add(name); // Set will ensure no duplicate names
            }

            Iterator i = setOfUniqueDisplayItemNames.iterator();
            while (i.hasNext())
            {
                String displayItemName = (String) i.next();

                ArrayList custom_displayItem = doDisplayItemList(dOc, displayItemListNode, displayItemName, displayItemName);
                appendArrayList(toElement, custom_displayItem);
            }
        }

    }

    /**
     * For now it only makes sense to allow a collection's about page description to contain HTML.
     * Collection titles may go into a pre tag or text only context or into HTML headings,
     * where preserving any HTML int coll title would become meaningless or even a hindrance.
     * TODO: if users used HTML pre tags, they'll have wanted whitespace preserved.
     * TODO There's probably a better way to do this, but I'm unable to think of it. Messy but works.
     */
    static private String preserveHTMLInDescriptionDisplayItem(Element collDescription) {
    // First get the <displayItem> as text
    String text = XMLTools.elementToString(collDescription, true);

    // We only want the contents, not the xml processing instruction or the
    // <displayItem></displayItem> or self-closing(!) <displayItem/> when no text for colldescr
    // Remove xml processing instr too by removing <displayItem until first > after that
    // this will also handle self-closing tag case
    String lookFor = "<displayItem";
    int start = text.indexOf(lookFor);
    if(start != -1) {
        start += lookFor.length();
        start = text.indexOf(">", start);
        text = text.substring(start+1).trim(); // trim to ensure no empty lines to deceive us
        // especially after a self-closing tag
    }

    // If we have a positive number of lines remaining, it means it wasn't a self-closing
    // tag. Remove the closing tag
    String[] lines = text.split("\\r?\\n");
    text = "";              
    if(lines.length > 1) {
        for (int j = 0; j < lines.length-1; j++) { // skip last line: closing tag
        text += lines[j].trim() + "\n"; // Easiest solution:
                                   // trim white space introduced by the one extra level of
                                   // indentation intoduced when enclosing <dispItem> tags removed
        }
    }

    text = text.replaceAll("&amp;", "&");// feels stupid, when we're going to change it back again soon
               // but it's the last bit needed to get GLI to preserve html coll descriptions
               // while displaying the html as html (without char entities) in the config file editor
    
    return text;//.trim(); // don't want trailing whitespace within displayItem
    }
    
    static public String generateStringVersion(Document doc)
    {
        return XMLTools.xmlNodeToString(doc);
    }

    static public void save(File collect_config_xml_file, Document doc)
    {
        Document collection_config_xml_document = convertInternalToCollectionConfig(doc);
        String[] nonEscapingTagNames = { StaticStrings.FORMAT_STR, StaticStrings.DISPLAYITEM_STR };
        XMLTools.writeXMLFile(collect_config_xml_file, collection_config_xml_document, nonEscapingTagNames);
    }

    //Convert the internal XML DOM tree (dOc) into that of collectionConfig.xml (skeleton)
    static private Document convertInternalToCollectionConfig(Document dOc)
    {
        //first parse an empty skeleton of xml config file
        //The aim is to convert the internal structure into this skeleton
        // This skeleton just has the CollectionConfig element and nothing else
        Document skeleton = XMLTools.parseXMLFile("xml/CollectionConfig.xml", true);
        //Element internal = dOc.getDocumentElement();
        convertSecurity(dOc, skeleton);
        convertMetadataList(dOc, skeleton);
        convertDisplayItemList(dOc, skeleton);
        convertGlobalFormat(dOc, skeleton);
        convertBuildType(dOc, skeleton);
        convertDatabaseType(dOc, skeleton);
        convertIndex(dOc, skeleton);
        convertSolrFacetsAndSorts(dOc, skeleton);
        convertPlugins(dOc, skeleton);//also do the plugout element
        convertClassifier(dOc, skeleton);
        convertSubcollectionIndexes(dOc, skeleton);
        convertLanguages(dOc, skeleton);
        convertSubcollection(dOc, skeleton);
        convertSearchType(dOc, skeleton);
        convertSearchFormat(dOc, skeleton);
        convertDisplayFormat(dOc, skeleton);
        convertReplaceListRef(dOc, skeleton);
        convertReplaceList(dOc, skeleton);
        convertServiceRackList(dOc, skeleton);
        convertUnknownElements(dOc, skeleton); // try to catch everything GLI doesn't know about

        return skeleton;
    }

    // Append the elements, which are of Element type, in 'list' to Element 'to'
    static private void appendArrayList(Element to, ArrayList list)
    {
        if (list == null)
            return;

        for (int i = 0; i < list.size(); i++)
        {
            appendProperly(to, (Element) list.get(i));
        }
    }

}