source: main/trunk/gli/src/org/greenstone/gatherer/cdm/CollectionConfigXMLReadWrite.java@ 36305

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Methods to read collectionConfig.xml files into internal XML form, and write 
 * them back out again.
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.cdm;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;

import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.metadata.MetadataElement;
import org.greenstone.gatherer.metadata.MetadataTools;
import org.greenstone.gatherer.util.Codec;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.util.XMLTools;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class CollectionConfigXMLReadWrite
{

    // a list of all known top level elements; importOption and buildOPtion are commented out here as they are currently handled by unknown code
    // /*StaticStrings.IMPORT_OPTION_STR, StaticStrings.BUILD_OPTION_STR,*/
    static final private String known_element_names_array[] = { StaticStrings.SECURITY_STR, StaticStrings.METADATALIST_STR, StaticStrings.DISPLAYITEMLIST_STR, StaticStrings.FORMAT_STR, StaticStrings.SEARCH_STR, StaticStrings.INFODB_STR, StaticStrings.BROWSE_STR, StaticStrings.IMPORT_STR, StaticStrings.DISPLAY_STR, StaticStrings.REPLACELISTREF_STR, StaticStrings.REPLACELIST_STR, StaticStrings.SERVICE_RACK_LIST_ELEMENT };
    static final private Set known_element_names = new HashSet(Arrays.asList(known_element_names_array));

    /**
     * *************************************************************************
     * ******************************* The code in this file is used for
     * greenstone 3 collection configuration, i.e., read ColletionConfig.xml
     * into the internal DOM tree, and convert the internal DOM tree back to
     * CollectionConfig.xml.
     * 
     * The main methods are save - save internal DOM out to config file, and 
     * parse - read in the config file into the internal DOM tree.
     *
     * Methods named 'doXXXX' are for converting collectionConfig.xml into 
     * the internal configuration xml structure; 
     * Methods named 'convertXXXX' are for
     * converting the internal configuration xml structure back to
     * collectionConfig.xml.
     ************************************************************************************************************ */

    /** Generates a String version of the internal XML document */
    static public String generateStringVersion(Document doc)
    {
    return XMLTools.xmlNodeToString(doc);
    }


    // From collectionConfig.xml to internal structure:add 'ex.' namespace (if none).
    // From internal structure to collectionConfig.xml:always peel off 'ex.' namespace (if any), except for format statement

    //This method parses 'xml_file_doc' into 'dOc'
    static public void parse(File xml_file, Document dOc)
    {

    Document xml_file_doc = XMLTools.parseXMLFile(xml_file);
    Element fromElement = xml_file_doc.getDocumentElement();
    Element toElement = dOc.getDocumentElement();

    // security element.
    doSecurity(dOc, fromElement);

    // metadataList - creator, maintainer, public, plus any custom metadata
    doMetadataList(dOc, fromElement);

    // top level display items
    doDisplayItemList(dOc, fromElement);

    // global format
    doGlobalFormat(dOc, fromElement);
    // infodb type
    doDatabaseType(dOc, fromElement);

    // searching. <search> index, level, subcolls etc
    Node searchNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.SEARCH_STR, 0);
    // return buildtype here as we use it later
    String buildtype = doBuildType(dOc, (Element)searchNode);
    if (buildtype.equalsIgnoreCase("mg")) {
        
        doIndexes(dOc, searchNode, true);
    }
    else {
        
        doIndexes(dOc, searchNode, false);
    }

    if(buildtype.equalsIgnoreCase("solr") || buildtype.equalsIgnoreCase("lucene")) {
        doSorts(dOc, searchNode);
        doDefaultSort(dOc, searchNode);
        if (buildtype.equalsIgnoreCase("solr")) {
        doFacets(dOc, searchNode);
        }
        // lucene will only have sort elements
    }

    doDefaultIndex(dOc, searchNode);
    doDefaultLevel(dOc, searchNode);
    doLevel(dOc, searchNode);
    doIndexOption(dOc, searchNode);
    doSubcollection(dOc, searchNode);
    doIndexSubcollection(dOc, searchNode);
    doIndexLanguage(dOc, searchNode);
    doDefaultIndexLanguage(dOc, searchNode);
    doLanguageMetadata(dOc, searchNode);
    doSearchType(dOc, searchNode);
    doSearchFormat(dOc, searchNode);

    // importing
    Node importNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.IMPORT_STR, 0);
    if (importNode == null) {
        
        System.out.println("There is no 'import' element.");
    } else {

        // plugins + plugout
        doPluginsAndPlugout(dOc, importNode);

    }

    // import and build options are handled by unknown code at present,
    // as can't be used in gli

    // browsing

    Node browseNode = XMLTools.getChildByTagNameIndexed(fromElement, StaticStrings.BROWSE_STR, 0);
    if (browseNode == null) {
        
        System.out.println("There is no browse element.");
    } else {
        doClassifiers(dOc, browseNode);
    }

    // <display> - currently just has a format element?

    doDisplayFormat(dOc, fromElement);

    // lesser top level elements
    doReplaceListRef(dOc, fromElement);
    doReplaceList(dOc, fromElement);
    doServiceRackList(dOc, fromElement);

    // handle everything else
    doUnknownElements(dOc, fromElement);

    }

    /** Saves the internal XML document out to the collectionConfig file */
    static public void save(File collect_config_xml_file, Document doc)
    {
    Document collection_config_xml_document = convertInternalToCollectionConfig(doc);
    String[] nonEscapingTagNames = { StaticStrings.FORMAT_STR, StaticStrings.DISPLAYITEM_STR };
    XMLTools.writeXMLFile(collect_config_xml_file, collection_config_xml_document, nonEscapingTagNames);
    }


    //Convert the internal XML DOM tree (dOc) into that of collectionConfig.xml (skeleton)
    static private Document convertInternalToCollectionConfig(Document dOc)
    {
    //first parse an empty skeleton of xml config file
    //The aim is to convert the internal structure into this skeleton
    // This skeleton just has the CollectionConfig element and nothing else
    Document skeleton = XMLTools.parseXMLFile("xml/CollectionConfig.xml", true);
    convertSecurity(dOc, skeleton);
    convertMetadataList(dOc, skeleton);
    convertDisplayItemList(dOc, skeleton);
    convertGlobalFormat(dOc, skeleton);
    convertBuildType(dOc, skeleton); // creates the search elem
    Element search = (Element) XMLTools.getChildByTagName(skeleton.getDocumentElement(), StaticStrings.SEARCH_STR);
    String buildtype = search.getAttribute(StaticStrings.TYPE_ATTRIBUTE);

    if (buildtype.equals(StaticStrings.MG_STR)) {
        // for mg, levels are included in index definition
        convertIndexes(dOc, skeleton, search, true);
        //Convert default index
        convertDefaultIndex(dOc, skeleton, search, true);

    } else {
        convertIndexes(dOc, skeleton, search, false);
        //Convert default index
        convertDefaultIndex(dOc, skeleton, search, false);
        // do the levels
        convertLevels(dOc, skeleton, search);
        convertDefaultLevel(dOc, skeleton, search);
            
    }
    // sortfields
    if (buildtype.equals(StaticStrings.LUCENE_STR) || buildtype.equals(StaticStrings.SOLR_STR)) {
        convertSorts(dOc, skeleton, search);
        convertDefaultSort(dOc, skeleton, search);
            
    }
    // facet fields
    if (buildtype.equals(StaticStrings.SOLR_STR)) {
        convertFacets(dOc, skeleton, search);
            
    }

    convertIndexOptions(dOc, skeleton, search);
    convertSubcollectionIndexes(dOc, skeleton, search);
    convertLanguages(dOc, skeleton, search);
    convertSubcollection(dOc, skeleton, search);
    convertSearchType(dOc, skeleton, search);
    convertSearchFormat(dOc, skeleton, search);

    convertDatabaseType(dOc, skeleton);
    convertPluginsAndPlugout(dOc, skeleton);// creates import/pluginList/plugin* + plugout if present (flax)
    convertClassifier(dOc, skeleton); //creates browse/classifier*
    convertDisplayFormat(dOc, skeleton);
    convertReplaceListRef(dOc, skeleton);
    convertReplaceList(dOc, skeleton);
    convertServiceRackList(dOc, skeleton);
    convertUnknownElements(dOc, skeleton); // try to catch everything GLI doesn't know about

    return skeleton;
    }
    ///////////////////////////////
    // Security
    //////////////////////////////

    /** Copy security element from config file to internal DOM */
    static private void doSecurity(Document to, Element from) {
    Node securityNode = XMLTools.getChildByTagNameIndexed(from, StaticStrings.SECURITY_STR,0);
    if (securityNode != null) {
        Element new_security = XMLTools.duplicateElement(to, (Element) securityNode, true);
        to.getDocumentElement().appendChild(new_security);
    }
    }

    /** Write out security element from internal DOM to config file */
    static private void convertSecurity(Document from, Document to)
    {
    Node security = XMLTools.getChildByTagNameIndexed(from.getDocumentElement(), StaticStrings.SECURITY_STR, 0);
    if (security != null)
        {
        Element to_element = XMLTools.duplicateElement(to, (Element) security, true);
        to.getDocumentElement().appendChild(to_element);
        
        }
    }


    ///////////////////////////////////////////////////////
    // metadataList - includes creator, maintainer, public
    ///////////////////////////////////////////////////////

    /** handle metadataList from config file. creator, maintainer, public get their own special elements in internal DOM, the rest just get copied to a metadataList, unused by GLI */
    static private void doMetadataList(Document to, Element from)
    {
    Element toElement = to.getDocumentElement();
    Element new_metadataList = to.createElement(StaticStrings.METADATALIST_STR);

    Node metadataListNode = XMLTools.getChildByTagNameIndexed(from, StaticStrings.METADATALIST_STR, 0);
    if (metadataListNode == null) {
        return;
    }
    NodeList metas = ((Element)metadataListNode).getElementsByTagName(StaticStrings.METADATA_STR);
    int num_children = (metas == null) ? 0: metas.getLength();
    if (num_children == 0) {
        return;
    }
    boolean has_meta = false;
    for (int i=0; i < num_children; i++) {
        Element m = (Element) metas.item(i);

        String name = m.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        if (name.equals(StaticStrings.COLLECTIONMETADATA_CREATOR_STR)) {
        // creator
        doSpecialMetadata(to, m, StaticStrings.COLLECTIONMETADATA_CREATOR_ELEMENT);
        } else if (name.equals(StaticStrings.COLLECTIONMETADATA_MAINTAINER_STR)) {
        // maintainer
        doSpecialMetadata(to, m, StaticStrings.COLLECTIONMETADATA_MAINTAINER_ELEMENT);
        } else if (name.equals(StaticStrings.COLLECTIONMETADATA_PUBLIC_STR)) {
        // public
        doSpecialMetadata(to, m, StaticStrings.COLLECTIONMETADATA_PUBLIC_ELEMENT);
        } else {
        // custom metadata - we just store it in the metadataList
        Element new_m = XMLTools.duplicateElement(to, m, true);
        new_metadataList.appendChild(new_m);
        has_meta = true;
        }
    }
    if (has_meta) {
        toElement.appendChild(new_metadataList);
    }
    }

    /** convert a special metadata element into internal DOM element */
    static private void doSpecialMetadata(Document to, Element meta, String new_element_name) {
    String text = XMLTools.getNodeText(meta);

    //If there is nothing to display, don't bother creating the element
    if (text.equals("")) return;
        
    Element new_element = to.createElement(new_element_name);
    new_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, meta.getAttribute(StaticStrings.NAME_ATTRIBUTE));
    new_element.setAttribute(StaticStrings.LANG_ATTRIBUTE, meta.getAttribute(StaticStrings.LANG_ATTRIBUTE));
    new_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    new_element.setAttribute(StaticStrings.SPECIAL_ATTRIBUTE, StaticStrings.TRUE_STR);
    XMLTools.setNodeText(new_element, text);
    to.getDocumentElement().appendChild(new_element);
    } // doMetadataList

    /** output metadataList to config file. Comprises any custom metadata stored in metadataList in internal DOM, plus creator, maintainer, public elements converted back to metadataList */
    static private void convertMetadataList(Document from, Document to)
    {
    // if we have existing metadataList (not used by GLI, just copied over)
    // then we use that otherwise we create a new one
    Node existing_metadataList = XMLTools.getChildByTagNameIndexed(from.getDocumentElement(), StaticStrings.METADATALIST_STR, 0);
    Element metadataList;
    if (existing_metadataList != null) {
        metadataList = XMLTools.duplicateElement(to, (Element)existing_metadataList, true);
    } else {
        metadataList = to.createElement(StaticStrings.METADATALIST_STR);
    }

    to.getDocumentElement().appendChild(metadataList);

    // creator, maintainer, public are stored differently in internal DOM, but
    // go into the metadataList for the config file

    String[] ele_names = { StaticStrings.COLLECTIONMETADATA_CREATOR_ELEMENT, StaticStrings.COLLECTIONMETADATA_MAINTAINER_ELEMENT, StaticStrings.COLLECTIONMETADATA_PUBLIC_ELEMENT };
    String[] att_names = { StaticStrings.COLLECTIONMETADATA_CREATOR_STR, StaticStrings.COLLECTIONMETADATA_MAINTAINER_STR, StaticStrings.COLLECTIONMETADATA_PUBLIC_STR };
    for (int i = 0; i < ele_names.length; i++) {
        
        Element e = XMLTools.getNamedElement(from.getDocumentElement(), ele_names[i], StaticStrings.NAME_ATTRIBUTE, att_names[i]);
        if (e == null)
        {
            continue;
        }
        String text = XMLTools.getNodeText(e);
        Element metadata = to.createElement(StaticStrings.METADATA_STR);
        metadata.setAttribute(StaticStrings.NAME_ATTRIBUTE, att_names[i]);
        metadata.setAttribute(StaticStrings.LANG_STR, StaticStrings.ENGLISH_LANGUAGE_STR);
        XMLTools.setNodeText(metadata, text);
        metadataList.appendChild(metadata);
    }
        
    }// convertMetadataList


    ////////////////////////////////////////////////
    // display items (top level)
    ///////////////////////////////////////////////

    /** processes the top level displayItemList, and adds them all as
    CollectionMeta to the internal doc. */
    static private void doDisplayItemList(Document to, Element from) {
    Element toElement = to.getDocumentElement();
    Node displayItemListNode = XMLTools.getChildByTagNameIndexed(from, StaticStrings.DISPLAYITEMLIST_STR, 0);
    if (displayItemListNode == null) {
        return;
    }
    NodeList displayItems = ((Element)displayItemListNode).getElementsByTagName(StaticStrings.DISPLAYITEM_STR);
    for (int i=0; i<displayItems.getLength(); i++) {
        Element item = (Element) displayItems.item(i);
        String name = item.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        String internal_name = name; // unless mentioned below
        if (name.equals(StaticStrings.NAME_STR)) {
        internal_name = StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR;
        } else if (name.equals(StaticStrings.DESCRIPTION_STR)) {
        internal_name = StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR;
        } else if (name.equals(StaticStrings.ICON_STR)) {
        internal_name = StaticStrings.COLLECTIONMETADATA_ICONCOLLECTION_STR;
        } else if (name.equals(StaticStrings.SMALLICON_STR)) {
        internal_name = StaticStrings.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR;
        }

        Element coll_meta = processSingleDisplayItem(to, item, internal_name, null);
        if (coll_meta != null) {
        toElement.appendChild(coll_meta);
        } 
    }


    }


    // create a displayItemList for config file, from CollectionMetadata
    static private void convertDisplayItemList(Document from, Document to)
    {
    Element displayItemList = to.createElement(StaticStrings.DISPLAYITEMLIST_STR);
    Element destination = to.getDocumentElement();

    // certain special collectionmeta elements should have different names
    // as displayItems in the collectionConfig.xml than they do in memory
    Map attributeMap = new HashMap(4);
    attributeMap.put(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR, StaticStrings.DESCRIPTION_STR);
    attributeMap.put(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR, StaticStrings.NAME_STR);
    attributeMap.put(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR, StaticStrings.SMALLICON_STR);
    attributeMap.put(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTION_STR, StaticStrings.ICON_STR);

    NodeList e_list = from.getDocumentElement().getElementsByTagName(StaticStrings.COLLECTIONMETADATA_ELEMENT);
    // if such elements don't exist, don't bother
    if (e_list != null) {
        

        for (int j = 0; j < e_list.getLength(); j++) {
            
        Element e = (Element) e_list.item(j);
        if (e.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) {
                
            continue;
        }

        String name_value = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        String name_mapping = (String) attributeMap.get(name_value);
        if (name_mapping != null) {             
            name_value = name_mapping;
        }
        Element displayItem  = constructDisplayItem(e, to, name_value);
                
        displayItemList.appendChild(displayItem);
        }

    }
    destination.appendChild(displayItemList);
    }

    //////////////////////////////////////////////
    // global (top level) format
    /////////////////////////////////////////////

    static private void doGlobalFormat(Document to, Element from)
    {
    // look for a top level format element
    Element fe = (Element) XMLTools.getChildByTagName(from, StaticStrings.FORMAT_STR);
    to.getDocumentElement().appendChild(doFormat(to, fe, StaticStrings.GLOBAL_STR));
    }

    // convert global format statement
    static private void convertGlobalFormat(Document from, Document to)
    {
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.GLOBAL_STR);

    to.getDocumentElement().appendChild(convertFormat(to, e));

    }


    ////////////////////////////////////////////////
    // database type = infodb type
    ////////////////////////////////////////////////

    static private void doDatabaseType(Document to, Element from) {

    Node databaseNode = XMLTools.getChildByTagNameIndexed(from, StaticStrings.INFODB_STR, 0);
    String databasetype_value = "gdbm";
    if (databaseNode != null) {
        
        databasetype_value = ((Element) databaseNode).getAttribute(StaticStrings.TYPE_ATTRIBUTE);//might be gdbm|jdbm|sqlite OR not yet set (in which case it should default to gdbm)
    }
    
    Element element = to.createElement(StaticStrings.DATABASETYPE_ELEMENT);
    element.setAttribute(StaticStrings.NAME_ATTRIBUTE, StaticStrings.DATABASETYPE_STR);
    element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, StaticStrings.ENGLISH_LANGUAGE_STR);
    element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    element.setAttribute(StaticStrings.SPECIAL_ATTRIBUTE, StaticStrings.TRUE_STR);
    
    XMLTools.setNodeText(element, databasetype_value);

    appendProperly(to.getDocumentElement(), element);

    } 
    static private void convertDatabaseType(Document from, Document to)
    {
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.DATABASETYPE_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.DATABASETYPE_STR);
    if (e == null) {
        
        return;
    }
    String db = XMLTools.getNodeText(e);
    Element dbtype = to.createElement(StaticStrings.INFODB_STR);
    dbtype.setAttribute(StaticStrings.TYPE_ATTRIBUTE, db);
    to.getDocumentElement().appendChild(dbtype);
    }

    // search *************************************************************

    ///////////////////////////////////////////////////
    // build type
    //////////////////////////////////////////////////

    static private String doBuildType(Document to, Element searchNode)
    {

    String buildtype = ((Element) searchNode).getAttribute(StaticStrings.TYPE_ATTRIBUTE);//might be mg|mgpp|lucene|solr



    //construct 'BuildType' element
    Element element = to.createElement(StaticStrings.BUILDTYPE_ELEMENT);
    element.setAttribute(StaticStrings.NAME_ATTRIBUTE, StaticStrings.BUILDTYPE_STR);
    element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, StaticStrings.ENGLISH_LANGUAGE_STR);
    element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    element.setAttribute(StaticStrings.SPECIAL_ATTRIBUTE, StaticStrings.TRUE_STR);

    XMLTools.setNodeText(element, buildtype);
    appendProperly(to.getDocumentElement(), element);
    return buildtype;
    }

    /** This uses the buildtype element in internal DOM, and creates the search element in the config file output, with type set to the indexer type */
    static private void convertBuildType(Document from, Document to)
    {
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.BUILDTYPE_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.BUILDTYPE_STR);
    if (e == null)
        {
        return;
        }
    String indexer = XMLTools.getNodeText(e);
    Element search = to.createElement(StaticStrings.SEARCH_STR);
    search.setAttribute(StaticStrings.TYPE_ATTRIBUTE, indexer);
    to.getDocumentElement().appendChild(search);
    }




    /////////////////////////////////////////////
    // indexes
    ////////////////////////////////////////////

    /**This converts indexes and their displayItems - set is_mg_index to true if dealing with mg, as that has slightly different format to the rest. */
    static private void doIndexes(Document to, Node searchNode, boolean is_mg_index)
    {
    doBaseSearchPartInternal(to, searchNode, StaticStrings.INDEXES_ELEMENT, StaticStrings.INDEX_ELEMENT, StaticStrings.INDEX_LOW_STR, SearchMeta.TYPE_INDEX, is_mg_index);
    Element toElement = to.getDocumentElement();
    Element indexes_element = (Element) XMLTools.getChildByTagName(toElement, StaticStrings.INDEXES_ELEMENT);
    indexes_element.setAttribute(StaticStrings.MGPP_ATTRIBUTE, (is_mg_index? StaticStrings.FALSE_STR:StaticStrings.TRUE_STR));


    boolean creating_mg_index = !is_mg_index;
    // create another set of <indexes> which will be used when user switches to/from MG
    // i.e. we build a default index set for a start
    Element other_indexes = to.createElement(StaticStrings.INDEXES_ELEMENT);//<Indexes>
    other_indexes.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
    other_indexes.setAttribute(StaticStrings.MGPP_ATTRIBUTE, (creating_mg_index? StaticStrings.FALSE_STR:StaticStrings.TRUE_STR)); 

    //put the namespace '.ex' as prefix to the indexes
    String[] index_strs = { StaticStrings.TEXT_STR, StaticStrings.EXTRACTED_NAMESPACE + StaticStrings.TITLE_ELEMENT, StaticStrings.EXTRACTED_NAMESPACE + StaticStrings.SOURCE_ELEMENT };
    for (int i = 0; i < index_strs.length; i++) {
        
        Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);//<Index>
        if (creating_mg_index) {
        index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, StaticStrings.DOCUMENT_STR);
        }
        Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
        content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_strs[i]);
        index_element.appendChild(content_element);

        other_indexes.appendChild(index_element);

    } // foreach fake index
    toElement.appendChild(other_indexes);

    } // doIndexes


    //construct 'DefaultIndex' element in the internal structure from collectionConfig.xml
    // TODO

    static private void doDefaultIndex(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    Element default_index_element = to.createElement(StaticStrings.INDEX_DEFAULT_ELEMENT);
    default_index_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

    Element e = (Element) XMLTools.getChildByTagName(searchNode, StaticStrings.INDEX_DEFAULT_ELEMENT_LOWERCASE);//defaultIndex
    if (e == null) {
        
        return;
    }
    String index_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);

    boolean old_index = false;
    if (index_str.indexOf(StaticStrings.COLON_CHARACTER) != -1) {
        
        //The index is 'level:source tuplets' which is for mg. Take out 'level'
        old_index = true;
        default_index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, index_str.substring(0, index_str.indexOf(StaticStrings.COLON_CHARACTER)));
        index_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
    }
    else {
        
        default_index_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, "");
    }

    //Each index may have a list of comma-separated strings.
    //split them into 'content' elements in the internal structure
    StringTokenizer content_tokenizer = new StringTokenizer(index_str, StaticStrings.COMMA_CHARACTER);
    while (content_tokenizer.hasMoreTokens()) {
        
        Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
        String content_str = content_tokenizer.nextToken();
        // Since the contents of indexes have to be certain keywords, or metadata elements, if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
        if (content_str.indexOf(StaticStrings.NS_SEP) == -1) {
            
        if (content_str.equals(StaticStrings.TEXT_STR) || (!old_index && content_str.equals(StaticStrings.ALLFIELDS_STR))) {
                
            // in this case, do nothing
        }
        else {
            
            content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;
        }
        }

        content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, content_str);
        default_index_element.appendChild(content_element);
        content_element = null;
    }
    appendProperly(toElement, default_index_element);
    } //doDefaultIndex

    static private void convertIndexes(Document from, Document to, Element search, boolean is_mg_index) {

    convertBaseSearchPartInternal(from, to, search, StaticStrings.INDEXES_ELEMENT, StaticStrings.INDEX_ELEMENT, StaticStrings.INDEX_LOW_STR, SearchMeta.TYPE_INDEX, is_mg_index);
    }
    static private void convertDefaultIndex(Document from, Document to, Element search, boolean is_mg_index) {
    convertBaseDefaultSearchPartInternal(from, to, search, StaticStrings.INDEX_DEFAULT_ELEMENT, StaticStrings.INDEX_DEFAULT_ELEMENT_LOWERCASE, is_mg_index);
    }
    


    ////////////////////////////////////////////////////////
    // sortfields and facets
    ////////////////////////////////////////////////////////

    static private void doSorts(Document to, Node searchNode) {
    doBaseSearchPartInternal(to, searchNode, StaticStrings.SORTS_ELEMENT, StaticStrings.SORT_ELEMENT, StaticStrings.SORT_LOW_STR, SearchMeta.TYPE_SORT, false);
    }
    static private void convertSorts(Document from, Document to, Element search) {
    convertBaseSearchPartInternal(from, to, search, StaticStrings.SORTS_ELEMENT, StaticStrings.SORT_ELEMENT, StaticStrings.SORT_LOW_STR, SearchMeta.TYPE_SORT, false);
    }

    // TODO
    static private void doDefaultSort(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    
    Element from_e = (Element) XMLTools.getChildByTagName(searchNode, StaticStrings.SORT_DEFAULT_ELEMENT);
    if (from_e == null) {
        return; // there is no default
    }
    Element default_sort = to.createElement(StaticStrings.SORT_DEFAULT_ELEMENT);
    
    default_sort.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    String name = from_e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
    default_sort.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
    appendProperly(toElement, default_sort);
    }
    static private void convertDefaultSort(Document from, Document to, Element search) {
    // todo - should this be combined with index? or write own code?
    convertBaseDefaultSearchPartInternal(from, to, search, StaticStrings.SORT_DEFAULT_ELEMENT, StaticStrings.SORT_DEFAULT_ELEMENT, false);
    }
    static private void doFacets(Document to, Node searchNode) {
    doBaseSearchPartInternal(to, searchNode, StaticStrings.FACETS_ELEMENT, StaticStrings.FACET_ELEMENT, StaticStrings.FACET_LOW_STR, SearchMeta.TYPE_FACET, false);
    }
    static private void convertFacets(Document from, Document to, Element search) {
    convertBaseSearchPartInternal(from, to, search, StaticStrings.FACETS_ELEMENT, StaticStrings.FACET_ELEMENT, StaticStrings.FACET_LOW_STR, SearchMeta.TYPE_FACET, false);
    }
    //////////////////////////////////////////////////
    // levels
    /////////////////////////////////////////////////


    //Handle levels (document, section). In the internal structure, the element is called 'IndexOption'
    static private void doLevel(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    NodeList level_children = ((Element) searchNode).getElementsByTagName(StaticStrings.LEVEL_ATTRIBUTE);
    int level_nodes = level_children.getLength();

    // it's mg, there's no level. So we construct a default 'indexOption' in the internal structure
    if (level_nodes < 1)
        {
        Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
        index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
        index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.LEVELS_STR);

        Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
        option_element.setAttribute(StaticStrings.NAME_STR, StaticStrings.DOCUMENT_STR);
        index_option.appendChild(option_element);

        appendProperly(toElement, index_option);

        return;
        }

    Element index_options = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
    index_options.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    index_options.setAttribute(StaticStrings.NAME_STR, StaticStrings.LEVELS_STR);

    for (int i = 0; i < level_nodes; i++)
        {
        Element level_element = (Element) level_children.item(i);
        String level_str = level_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
        option_element.setAttribute(StaticStrings.NAME_STR, level_str);
        index_options.appendChild(option_element);

        // Contructing 'searchmetadata' elements from the 'displayItem' of this 'level' element
        doSearchDisplayItems(to, level_element, level_str, SearchMeta.TYPE_LEVEL);
        //ArrayList searchmetadata_list = doDisplayItemListForType(to, level_element, StaticStrings.NAME_STR, level_str, SearchMeta.TYPE_LEVEL);
        //appendArrayList(toElement, searchmetadata_list);
        }
    appendProperly(toElement, index_options);
    } // doLevel

    // For mg, this method is still called, but make it 'assigned=false'
    static private void doDefaultLevel(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    Element default_index_option = to.createElement(StaticStrings.INDEXOPTION_DEFAULT_ELEMENT);
    default_index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.LEVEL_DEFAULT_STR);

    Element e = (Element) XMLTools.getChildByTagName(searchNode, StaticStrings.LEVEL_DEFAULT_ELEMENT);
    if (e != null) {
        
        default_index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
        String level = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        default_index_option.setAttribute(StaticStrings.VALUE_ATTRIBUTE, level);
    }
    else {
        
        //In the case of mg, there's no level! build a default one using 'assigned=false value=document'
        default_index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
        default_index_option.setAttribute(StaticStrings.VALUE_ATTRIBUTE, StaticStrings.DOCUMENT_STR);
    }
    appendProperly(toElement, default_index_option);
    }

    // Convert levels for mgpp/lucene. This method is called by converIndex() when mgpp indexer is detected.
    static private void convertLevels(Document from, Document to, Element search)
    {
    Element source = from.getDocumentElement();
    Element index_option = XMLTools.getNamedElement(source, StaticStrings.INDEXOPTIONS_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.LEVELS_STR);
    if (index_option == null)
        {
        return;
        }
    //Debugging purposes
    if (index_option.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
        {
        DebugStream.println("For mgpp, there should be an IndexOption element for levels which is assigned 'true': possible bug.");
        }

    NodeList option_elements = index_option.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
    int num_elements = option_elements.getLength();

    // Don't output anything if no indexes are set
    if (num_elements == 0)
        {
        return;//
        }

    for (int k = 0; k < num_elements; k++)
        {
        Element e = (Element) option_elements.item(k);
        String name_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element level_element = to.createElement(StaticStrings.LEVEL_ELEMENT);
        level_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);

        //Now construct displayItem for this level element from searchmetadata
        // ** here
        ArrayList searchmetadata_list = getMatchingSearchMetaElements(source, name_str, SearchMeta.TYPE_LEVEL);
        //ArrayList collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.SEARCHMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, name_str);

        if (searchmetadata_list != null)
            {

            for (int j = 0; j < searchmetadata_list.size(); j++)
                {
                Element searchmetadata = (Element) searchmetadata_list.get(j);

                Element displayItem = constructDisplayItem(searchmetadata, to);
                level_element.appendChild(displayItem);
                }
            }
        search.appendChild(level_element);
        }
    }
    static private void convertDefaultLevel(Document from, Document to, Element search) {
    Element source = from.getDocumentElement();
    Element default_index_option = XMLTools.getNamedElement(source, StaticStrings.INDEXOPTION_DEFAULT_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.LEVEL_DEFAULT_STR);
    if (default_index_option == null)
        {
        return;
        }
    Element default_level = to.createElement(StaticStrings.LEVEL_DEFAULT_ELEMENT);
    String default_level_str = default_index_option.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
    default_level.setAttribute(StaticStrings.NAME_ATTRIBUTE, default_level_str);
    search.appendChild(default_level);
    
    }

    ////////////////////////////////////////////////
    // index options
    /////////////////////////////////////////////////

    //Handle 'indexOption' (i.e. casefold, stem etc). In the internal structure, the element is called 'IndexOption'
    static private void doIndexOption(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    //Node index_option_node = XMLTools.getChildByTagName(searchNode, StaticStrings.INDEXOPTION_STR);
    //if (index_option_node == null)
    //{
    //  return;
    //}
    //NodeList option_children = ((Element) index_option_node).getElementsByTagName(StaticStrings.OPTION_STR);
    NodeList option_children = ((Element) searchNode).getElementsByTagName(StaticStrings.INDEXOPTION_STR);
    int num_options = option_children.getLength();

    // for lucene, there is no 'indexOption'. We build a default 'indexOption' and 'assigned=false' in case the user switches to mg or mgpp
    if (num_options < 1)
        {
        Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
        index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
        index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.INDEXOPTIONS_STR);
        String[] option_str = { StaticStrings.CASEFOLD_OPTION_STR, StaticStrings.STEM_OPTION_STR };
        for (int i = 0; i < option_str.length; i++)
            {
            Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
            option_element.setAttribute(StaticStrings.NAME_STR, option_str[i]);
            index_option.appendChild(option_element);
            }
        appendProperly(toElement, index_option);
        return;
        }

    Element index_option = to.createElement(StaticStrings.INDEXOPTIONS_ELEMENT);
    index_option.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    index_option.setAttribute(StaticStrings.NAME_STR, StaticStrings.INDEXOPTIONS_STR);

    for (int i = 0; i < num_options; i++)
        {
        String option_str = ((Element) option_children.item(i)).getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
        option_element.setAttribute(StaticStrings.NAME_STR, option_str);
        index_option.appendChild(option_element);
        }
    appendProperly(toElement, index_option);
    }


    // Convert indexoptions for mg/mgpp/lucene. This method is called by convertIndex().
    static private void convertIndexOptions(Document from, Document to, Element search)
    {
    Element source = from.getDocumentElement();
    Element index_option = XMLTools.getNamedElement(source, StaticStrings.INDEXOPTIONS_ELEMENT, StaticStrings.NAME_ATTRIBUTE, StaticStrings.INDEXOPTIONS_STR);
    if (index_option == null)
        {
        return;
        }
    //Debugging purposes
    if (index_option.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) {
        
        DebugStream.println("There should be an IndexOption element which is assigned 'true': possible bug.");

        // for lucene and solr collections, don't write out stemming, casefolding and accentfolding indexOptions
        // since they are not meant to be editable: stemming and casefolding are always on in lucene
        String buildtype_value = search.getAttribute(StaticStrings.TYPE_ATTRIBUTE); //might be mg|mgpp|lucene|solr
        if(buildtype_value.equalsIgnoreCase("solr") || buildtype_value.equalsIgnoreCase("lucene")) {
        return;
        }
    } 
    //Element indexOptionEl = to.createElement(StaticStrings.INDEXOPTION_STR);
    NodeList option_elements = index_option.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
    int num_elements = option_elements.getLength();
    // Don't output anything if no index
    if (num_elements == 0)
        {
        return;//
        }
    //search.appendChild(indexOptionEl);

    for (int k = 0; k < num_elements; k++)
        {
        Element e = (Element) option_elements.item(k);
        String name_att = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element optionEl = to.createElement(StaticStrings.INDEXOPTION_STR);
        optionEl.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_att);
        // default value?? on/off
        //indexOptionEl.appendChild(optionEl);
        search.appendChild(optionEl);
        }

    }

    //////////////////////////////////////////
    // subcollections
    //////////////////////////////////////////

    // Handling 'subcollection' elements in 'search' element of 'collectionConfig.xml'
    static private void doSubcollection(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    NodeList sub_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SUBCOLLECTION_STR);
    int sub_nodes = sub_children.getLength();

    // There is no subcollection
    if (sub_nodes < 1)
        {
        return;
        }

    for (int i = 0; i < sub_nodes; i++)
        {
        Element sub_child = (Element) sub_children.item(i);
        String name_str = sub_child.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        String filter_str = sub_child.getAttribute(StaticStrings.FILTER_ATTRIBUTE);

        // filter_str is in the form '<! (if set)><metadata>/<metadata value>/<flag (if any)>'

        int pos = filter_str.indexOf(StaticStrings.SEPARATOR_CHARACTER);
        String meta_str = "";
        String meta_value_str = "";
        String clude_str = "";
        String flag_str = "";
        if (pos == -1)
            {

            meta_str = meta_value_str = filter_str;
            clude_str = StaticStrings.INCLUDE_STR;
            }
        else
            {
            clude_str = StaticStrings.INCLUDE_STR;
            if (filter_str.startsWith(StaticStrings.EXCLAMATION_CHARACTER))
                {
                clude_str = StaticStrings.EXCLUDE_STR;
                // Peel off "!"
                filter_str = filter_str.substring(StaticStrings.EXCLAMATION_CHARACTER.length());
                }

            String[] strs = filter_str.split(StaticStrings.SEPARATOR_CHARACTER);
            if (strs[0] != null && strs[0] != "")
                {
                meta_str = strs[0];
                }
            if (!meta_str.equals(StaticStrings.FILENAME_STR) && meta_str.indexOf(StaticStrings.NS_SEP) == -1)
                {
                meta_str = StaticStrings.EXTRACTED_NAMESPACE + meta_str;
                }

            if (strs[1] != null && strs[1] != "")
                {
                meta_value_str = strs[1];
                }
            if (strs.length > 2)
                {
                //This means there has been set a flag
                if (strs[2] != null && strs[2] != "")
                    {
                    flag_str = strs[2];
                    }
                }
            }
        Element subcollection_element = to.createElement(StaticStrings.SUBCOLLECTION_ELEMENT);
        subcollection_element.setAttribute(StaticStrings.NAME_STR, name_str);
        subcollection_element.setAttribute(StaticStrings.CONTENT_ATTRIBUTE, meta_str);
        subcollection_element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, clude_str);
        if (flag_str != "")
            {
            subcollection_element.setAttribute(StaticStrings.OPTIONS_ATTRIBUTE, flag_str);
            }
        XMLTools.setNodeText(subcollection_element, meta_value_str);

        toElement.appendChild(subcollection_element);
        }
    } // doSubCollection


    static private void convertSubcollection(Document from, Document to, Element search)
    {
    Element source = from.getDocumentElement();
    //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
    //Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

    // Get the Subcollection element from the internal structure
    NodeList subcollection_elements = source.getElementsByTagName(StaticStrings.SUBCOLLECTION_ELEMENT);
    if (subcollection_elements == null)
        {
        return;
        }
    int subcollection_elements_length = subcollection_elements.getLength();

    if (subcollection_elements_length == 0)
        { // no
        return;
        }

    for (int j = 0; j < subcollection_elements_length; j++)
        {

        Element e = (Element) subcollection_elements.item(j);
        if (e.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
            continue;
            }
        String content = e.getAttribute(StaticStrings.CONTENT_ATTRIBUTE);
        String name = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        String options = e.getAttribute(StaticStrings.OPTIONS_ATTRIBUTE);
        String type = e.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
        String text = XMLTools.getNodeText(e);

        String filter = "";
        if (type.equals(StaticStrings.EXCLUDE_STR))
            {
            filter = StaticStrings.EXCLAMATION_CHARACTER;
            }

        if (content.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && content.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
            {
            content = content.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
            }
        filter = filter + content + StaticStrings.SEPARATOR_CHARACTER + text;
        if (options != null && options != "")
            {
            filter = filter + StaticStrings.SEPARATOR_CHARACTER + options;
            }
        Element subcollection = to.createElement(StaticStrings.SUBCOLLECTION_STR);
        subcollection.setAttribute(StaticStrings.FILTER_ATTRIBUTE, filter);
        subcollection.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);

        search.appendChild(subcollection);
        }
    }

    //Handle 'indexSubcollection' element of collectionConfig.xml,  which is called 'SubcollectionIndexes' in the internal structure. These contain the subcollection indexes (i.e. the filter or filter combinations), and displayed words for the filter names.
    static private void doIndexSubcollection(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    NodeList index_sub_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SUBCOLLECTION_INDEX_ELEMENT);
    int num_nodes = index_sub_children.getLength();

    // there is no subcollection index
    if (num_nodes < 1)
        {
        return;
        }

    Element subcollection_indexes = to.createElement(StaticStrings.SUBCOLLECTION_INDEXES_ELEMENT);

    for (int i = 0; i < num_nodes; i++)
        {
        Element index_element = to.createElement(StaticStrings.INDEX_ELEMENT);
        Element index_sub_child = (Element) index_sub_children.item(i);
        String name_str = index_sub_child.getAttribute(StaticStrings.NAME_ATTRIBUTE);

        // name_str is in the form of comma separated strings, each of which is a subcollection filter name
        String[] filters = name_str.split(StaticStrings.COMMA_CHARACTER);
        for (int j = 0; j < filters.length; j++)
            {

            Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
            content_element.setAttribute(StaticStrings.NAME_STR, filters[j]);
            index_element.appendChild(content_element);
            }
        subcollection_indexes.appendChild(index_element);

        // Contructing 'searchmetadata' elements from the 'displayItem' of this 'indexSubcollection' element
        doSearchDisplayItems(to, index_sub_child, name_str, SearchMeta.TYPE_PARTITION);
        //ArrayList searchmetadata_list = doDisplayItemListForType(to, index_sub_child, StaticStrings.NAME_STR, name_str, SearchMeta.TYPE_PARTITION); // todo should be name_str???
        //appendArrayList(toElement, searchmetadata_list);
        }
    appendProperly(toElement, subcollection_indexes);
    } // doIndexSubCollection

    static private void convertSubcollectionIndexes(Document from, Document to, Element search)
    {
    Element source = from.getDocumentElement();
    //Get the 'search' element from 'to' which has already been created in 'convertBuildType'
    //Element search = (Element) XMLTools.getChildByTagName(to.getDocumentElement(), StaticStrings.SEARCH_STR);

    // Get the SubcollectionIndexes element from the internal structure
    Element subcollection_indexes = (Element) XMLTools.getChildByTagName(source, StaticStrings.SUBCOLLECTION_INDEXES_ELEMENT);
    if (subcollection_indexes == null)
        {
        return;
        }
    NodeList index_elements = subcollection_indexes.getElementsByTagName(StaticStrings.INDEX_ELEMENT);
    int index_elements_length = index_elements.getLength();

    if (index_elements_length == 0)
        { // no indexes
        return;
        }

    for (int j = 0; j < index_elements_length; j++)
        {
        Element index_element = (Element) index_elements.item(j);
        if (index_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
            continue;
            }

        Element index = to.createElement(StaticStrings.SUBCOLLECTION_INDEX_ELEMENT);

        String index_value = "";

        NodeList content_elements = index_element.getElementsByTagName(StaticStrings.CONTENT_ELEMENT);
        int content_elements_length = content_elements.getLength();

        for (int k = 0; k < content_elements_length; k++)
            {
            Element content_element = (Element) content_elements.item(k);
            if (content_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                {
                continue;
                }
            String name_str = content_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            index_value += name_str;
            // Make it comma separated string
            if (k < content_elements_length - 1)
                {
                index_value += StaticStrings.COMMA_CHARACTER;
                }
            content_element = null;
            }//for loop ends

        index.setAttribute(StaticStrings.NAME_ATTRIBUTE, index_value);

        // Now constructing 'displayItem' element for this 'indexSubcollection' element
        // from the searchmetadata element
        // *** here
        ArrayList searchmetadata_list = getMatchingSearchMetaElements(source, index_value, SearchMeta.TYPE_PARTITION);
        //ArrayList collectionmetadata_list = XMLTools.getNamedElementList(source, StaticStrings.COLLECTIONMETADATA_ELEMENT, StaticStrings.NAME_ATTRIBUTE, index_value);

        if (searchmetadata_list != null)
            {

            for (int k = 0; k < searchmetadata_list.size(); k++)
                {
                Element searchmetadata = (Element) searchmetadata_list.get(k);
                if (searchmetadata.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
                    {
                    continue;
                    }
                Element displayItem = constructDisplayItem(searchmetadata, to);
                index.appendChild(displayItem);
                }
            }

        search.appendChild(index);

        } //for loop ends
    }

    ///////////////////////////////////////////////
    // language partitions
    //////////////////////////////////////////////

    //Handle 'indexLanguage' element of collectionConfig.xml,  which is called 'Languages' in the internal structure. These contain the language indexes, and displayed words for those language index names.
    static private void doIndexLanguage(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    NodeList index_sub_children = ((Element) searchNode).getElementsByTagName(StaticStrings.LANGUAGE_INDEX_ELEMENT);
    int num_nodes = index_sub_children.getLength();

    // there is no subcollection index
    if (num_nodes < 1)
        {
        return;
        }

    Element language_indexes = to.createElement(StaticStrings.LANGUAGES_ELEMENT);

    for (int i = 0; i < num_nodes; i++)
        {
        Element language_element = to.createElement(StaticStrings.LANGUAGE_ELEMENT);
        Element index_sub_child = (Element) index_sub_children.item(i);
        String name_str = index_sub_child.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        language_element.setAttribute(StaticStrings.NAME_STR, name_str);
        language_indexes.appendChild(language_element);

        // Contructing 'searchmetadata' elements from the 'displayItem' of this 'indexLanguage' element
        doSearchDisplayItems(to, index_sub_child, name_str, SearchMeta.TYPE_LANGUAGE);
        }
    toElement.appendChild(language_indexes);
    } //doIndexLanguage


    static private void doDefaultIndexLanguage(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    String elementNameFrom = StaticStrings.LANGUAGE_DEFAULT_INDEX_ELEMENT;
    String elementNameTo = StaticStrings.LANGUAGE_DEFAULT_ELEMENT;
    Node from_element = XMLTools.getChildByTagName(searchNode, elementNameFrom);
    if (from_element == null) return;

    Element to_element = to.createElement(elementNameTo);

    String name_str = ((Element) from_element).getAttribute(StaticStrings.NAME_ATTRIBUTE);
    to_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
    to_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

    toElement.appendChild(to_element);
    }

    // Handling <languageMetadata> in collectionConfig.xml ('elementNameFrom'); in the internal structure, it is called 'LanguageMetadata' ('elementNameTo').
    // Converting from collectionConfig.xml to the internal xml structure.
    static private void doLanguageMetadata(Document to, Node searchNode)
    {
    Element toElement = to.getDocumentElement();
    String elementNameFrom = StaticStrings.LANGUAGE_METADATA_ELEMENT_STR;
    String elementNameTo = StaticStrings.LANGUAGE_METADATA_ELEMENT;
    Node from_element = XMLTools.getChildByTagName(searchNode, elementNameFrom);
    if (from_element == null) return;

    Element to_element = to.createElement(elementNameTo);

    String name_str = ((Element) from_element).getAttribute(StaticStrings.NAME_ATTRIBUTE);
    if (name_str.indexOf(StaticStrings.NS_SEP) == -1) {
        
        name_str = StaticStrings.EXTRACTED_NAMESPACE + name_str;
    }
    to_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
    to_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

    toElement.appendChild(to_element);
    }

    /** language partitions - convert from internal DOM to config file, 
    does <language>, <defaultLanguage>, <languageMetadata> plus the <displayItem>s inside <language>
    */
    static private void convertLanguages(Document from, Document to, Element search)
    {
    Element source = from.getDocumentElement();

    // Get the Languages element from the internal structure
    Element languages = (Element) XMLTools.getChildByTagName(source, StaticStrings.LANGUAGES_ELEMENT);
    if (languages == null) return;

    NodeList language_elements = languages.getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
    int language_elements_length = language_elements.getLength();

    if (language_elements_length == 0) return;

    for (int j = 0; j < language_elements_length; j++) {
        
        Element element = (Element) language_elements.item(j);
        if (element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

        // Create indexLanguage element
        Element index_language = to.createElement(StaticStrings.LANGUAGE_INDEX_ELEMENT);

        String name_str = element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        index_language.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);

        // Now constructing 'displayItem' element for this 'indexLanguage' element
        // from the searchmetadata element

        ArrayList searchmetadata_list = getMatchingSearchMetaElements(source, name_str, SearchMeta.TYPE_LANGUAGE);

        if (searchmetadata_list != null) {
        for (int k = 0; k < searchmetadata_list.size(); k++) {
                
            Element searchmetadata = (Element) searchmetadata_list.get(k);
            if (searchmetadata.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;
            
            Element displayItem = constructDisplayItem(searchmetadata, to);
            index_language.appendChild(displayItem);
        }
        }

        search.appendChild(index_language);

    } //for loop ends

    // Convert DefaultLanguage
    // Get the DefaultLanguage element from the internal structure
    Element default_language = (Element) XMLTools.getChildByTagName(source, StaticStrings.LANGUAGE_DEFAULT_ELEMENT);
    if (default_language != null) {
        
        String lang_name = default_language.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element default_index_language = to.createElement(StaticStrings.LANGUAGE_DEFAULT_INDEX_ELEMENT);
        default_index_language.setAttribute(StaticStrings.NAME_ATTRIBUTE, lang_name);
        search.appendChild(default_index_language);
    }
    // Convert LanguageMetadata
    // Get the LanguageMetadata element from the internal structure
    Element language_metadata = (Element) XMLTools.getChildByTagName(source, StaticStrings.LANGUAGE_METADATA_ELEMENT);
    if (language_metadata != null) {
        
        String meta_name = language_metadata.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element language_meta = to.createElement(StaticStrings.LANGUAGE_METADATA_ELEMENT_STR);
        if (meta_name.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && meta_name.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1) {
        
        meta_name = meta_name.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
        }
        language_meta.setAttribute(StaticStrings.NAME_ATTRIBUTE, meta_name);
        search.appendChild(language_meta);
    }
    }

    ////////////////////////////////////////////////
    // search types
    //////////////////////////////////////////////////

    // Handling search types
    static private void doSearchType(Document to, Node searchNode)
    {
    NodeList type_children = ((Element) searchNode).getElementsByTagName(StaticStrings.SEARCHTYPE_ELEMENT);
    int num_types = type_children.getLength();
    String searchtype_str = "";
    if (num_types < 1) {
        
        // not defined yet, add in default
        searchtype_str = "plain,simpleform,advancedform";
    }
    else {
        
        for (int i = 0; i < num_types; i++) {
            
        Node e = type_children.item(i);
        String t = ((Element) e).getAttribute(StaticStrings.NAME_ATTRIBUTE);
        if (i > 0) {
                
            searchtype_str += ",";
        }
        searchtype_str += t;
        }
    }
    searchtype_str = searchtype_str.trim();

    // pretend its a format statement
    Element search_type_element = to.createElement(StaticStrings.FORMAT_STR);
    search_type_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, StaticStrings.SEARCHTYPE_ELEMENT);
    XMLTools.setNodeText(search_type_element, searchtype_str);
    appendProperly(to.getDocumentElement(), search_type_element);

    }


    //Handle 'searchType' of collectionConfig.xml. In the internal structure, its also called 'searchType', eg. plain, form
    static private void convertSearchType(Document from, Document to, Element search)
    {
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.SEARCHTYPE_ELEMENT);//searchType

    if (e == null) return;

    String searchtype_str = XMLTools.getNodeText(e).trim();

    String[] types = searchtype_str.split(",");
    for (int i = 0; i < types.length; i++) {
        
        Element search_type_element = to.createElement(StaticStrings.SEARCHTYPE_ELEMENT);
        search_type_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, types[i]);
        search.appendChild(search_type_element);
    }
    }

    /////////////////////////////////////////////
    // search format
    ///////////////////////////////////////////////


    // Handling search format statement
    static private void doSearchFormat(Document to, Node searchNode)
    {
    // THere is currently just one format element for search. HOwever, need to check for old config files which used to have <format name="searchTypes">
    NodeList format_children = ((Element) searchNode).getElementsByTagName(StaticStrings.FORMAT_STR);
    int format_nodes = format_children.getLength();
    if (format_nodes < 1) return;

    Element format = null;
    for (int i = 0; i < format_nodes; i++) {
        
        Node e = format_children.item(i);
        if (e.hasAttributes() == false) {
            
        //The format element for format statement has no attribute
        format = (Element) e;
        }
    }
    //format statement for search
    if (format != null) {
        
        (to.getDocumentElement()).appendChild(doFormat(to, format, StaticStrings.SEARCH_STR));
    }
    }

    //convert format statement for search
    static private void convertSearchFormat(Document from, Document to, Element search)
    {
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.SEARCH_STR);

    search.appendChild(convertFormat(to, e));

    }

    ///////////////////////////////////////////
    // plugins
    ///////////////////////////////////////////

    // Transform plugins (pluginListNode) of collectionConfig.xml into the internal structure (i.e. Document to)
    static private void doPluginsAndPlugout(Document to, Node importNode)
    {
    Element toElement = to.getDocumentElement();
    Node pluginListNode = XMLTools.getChildByTagNameIndexed((Element) importNode, StaticStrings.PLUGINLIST_STR, 0);
    if (pluginListNode == null) {
            
        System.out.println("There is no pluginlist set.");
            
    } else {
        NodeList plugin_children = ((Element) pluginListNode).getElementsByTagName(StaticStrings.PLUGIN_STR);
        int plugin_nodes = plugin_children.getLength();

        if (plugin_nodes >= 1) {
        for (int i = 0; i < plugin_nodes; i++)
            {
            Element e = (Element) plugin_children.item(i);
            String str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            str = Utility.ensureNewPluginName(str);
            Element plugin_element = to.createElement(StaticStrings.PLUGIN_ELEMENT);
            plugin_element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, str);

            NodeList option_children = e.getElementsByTagName(StaticStrings.OPTION_STR);

            for (int j = 0; j < option_children.getLength(); j++)
                {
                Element el = (Element) option_children.item(j);
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                if (name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                    {
                    name_str = name_str.substring(1);
                    }
                String value_str = el.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
                Element option_element = null;

                if (name_str.equals("") && !value_str.equals(""))
                    {
                    continue;
                    }

                option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
                option_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
                if (name_str.equals(StaticStrings.RECPLUG_STR) && value_str.equals(StaticStrings.USE_METADATA_FILES_ARGUMENT))
                    {
                    continue; // ignore this option
                    }

                if (value_str != null)
                    {
                    // Remove any speech marks appended in strings containing whitespace
                    if (value_str.startsWith(StaticStrings.SPEECH_CHARACTER) && value_str.endsWith(StaticStrings.SPEECH_CHARACTER))
                        {
                        value_str = value_str.substring(1, value_str.length() - 1);
                        }
                    if (name_str.equals(StaticStrings.METADATA_STR))
                        {
                        // The metadata argument must be the fully qualified name of a metadata element, so if it doesn't yet have a namespace, append the extracted metadata namespace.
                        String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                        value_str = "";
                        for (int k = 0; k <= values.length - 1; k++)
                            {
                            if (values[k].indexOf(StaticStrings.NS_SEP) == -1)
                                {
                                values[k] = StaticStrings.EXTRACTED_NAMESPACE + values[k];
                                }

                            if (k < values.length - 1)
                                {
                                value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;

                                }
                            else
                                {
                                value_str = value_str + values[k];
                                }
                            }
                        }
                    }
                if (!name_str.equals(""))
                    {
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                    }
                if (!value_str.equals(""))
                    {
                    XMLTools.setNodeText(option_element, value_str);
                    }
                plugin_element.appendChild(option_element);

                }

            appendProperly(toElement, plugin_element);
            }
        } // if we had plugin nodes
    }// else plguins not null
    
    // do the plugout element if there is one (flax)
    Node plugout = XMLTools.getChildByTagNameIndexed((Element) importNode, StaticStrings.PLUGOUT_ELEMENT, 0);
    if (plugout != null)
        {
        Element to_element = XMLTools.duplicateElement(to, (Element) plugout, true);
        toElement.appendChild(to_element);
        }
        
        
    }


    // Convert plugins in the internal(i.e. Document from) to collectionconfig.xml (i.e. Document to)
    static private void convertPluginsAndPlugout(Document from, Document to)
    {
    Element import_element = to.createElement(StaticStrings.IMPORT_STR);
    Element plugin_list_element = to.createElement(StaticStrings.PLUGINLIST_STR);

    NodeList children = from.getDocumentElement().getElementsByTagName(StaticStrings.PLUGIN_ELEMENT);
    int num_children = (children == null) ? 0 : children.getLength();
    if (num_children == 0)
        {
        return;
        }

    for (int i = 0; i < num_children; i++)
        {

        Element child = (Element) children.item(i);
        if (child.getAttribute(StaticStrings.SEPARATOR_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
            {
            continue;
            }
        if (child.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR))
            {
            continue;
            }

        String str = child.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
        Element plugin_element = to.createElement(StaticStrings.PLUGIN_STR);
        plugin_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, str);

        NodeList option_children = child.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
        for (int j = 0; j < option_children.getLength(); j++)
            {
            Element el = (Element) option_children.item(j);
            if (!el.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.TRUE_STR))
                {
                continue;
                }
            String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String value_str = XMLTools.getNodeText(el);

            if (name_str == null && value_str == null)
                {
                continue;
                }
            Element option_element = to.createElement(StaticStrings.OPTION_STR);
            if (name_str != null && name_str.equals(StaticStrings.METADATA_STR))
                {

                // The metadata argument is the fully qualified name of a metadata element, so if it contains a namespace, remove the extracted metadata namespace as the build process doesn't know about it, but ONLY if it is not embedded metadata (e.g. ex.dc.*)
                String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                value_str = "";
                for (int k = 0; k <= values.length - 1; k++)
                    {
                    if (values[k].startsWith(StaticStrings.EXTRACTED_NAMESPACE) && values[k].indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
                        {
                        values[k] = values[k].substring(StaticStrings.EXTRACTED_NAMESPACE.length());
                        }

                    if (k < values.length - 1)
                        {
                        value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;
                        }
                    else
                        {
                        value_str = value_str + values[k];
                        }
                    }
                }

            if (!name_str.equals(""))
                {
                if (!name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                    {
                    name_str = StaticStrings.MINUS_CHARACTER + name_str;
                    }
                option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }

            if (!value_str.equals(""))
                {
                option_element.setAttribute(StaticStrings.VALUE_ATTRIBUTE, value_str);
                }

            plugin_element.appendChild(option_element);
            }//for loop ends

        plugin_list_element.appendChild(plugin_element);
        }//for loop ends

    import_element.appendChild(plugin_list_element);

    //do the plugout element (used by building flax collections)
    Node plugout = XMLTools.getChildByTagNameIndexed(from.getDocumentElement(), StaticStrings.PLUGOUT_ELEMENT, 0);
    if (plugout != null)
        {
        Element to_element = XMLTools.duplicateElement(to, (Element) plugout, true);
        import_element.appendChild(to_element);
        }

    to.getDocumentElement().appendChild(import_element);
    } // convertPlugins


    // Browse**********************************************************

    ////////////////////////////////////////////////////
    // classifiers
    ////////////////////////////////////////////////////


    //Handle classifiers
    static private void doClassifiers(Document to, Node browseNode)
    {
    Element toElement = to.getDocumentElement();
    NodeList classifier_children = ((Element) browseNode).getElementsByTagName(StaticStrings.CLASSIFIER_STR);
    int num_nodes = classifier_children.getLength();

    if (num_nodes < 1)
        {
        return;
        }

    for (int i = 0; i < num_nodes; i++)
        {
        Element e = (Element) classifier_children.item(i);
        String str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        Element classify_element = to.createElement(StaticStrings.CLASSIFY_ELEMENT);
        classify_element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, str);

        String options_str = "";
        NodeList option_children = e.getElementsByTagName(StaticStrings.OPTION_STR);
        for (int j = 0; j < option_children.getLength(); j++)
            {
            Element el = (Element) option_children.item(j);
            String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            options_str = options_str + ((name_str.equals("")) ? "" : (" " + name_str));
            if (name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                {
                name_str = name_str.substring(1);
                }
            String value_str = el.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
            options_str = options_str + ((name_str.equals("")) ? "" : (" " + value_str));
            Element option_element = null;

            if (name_str.equals("") && !value_str.equals(""))
                {
                continue; //invalid condition
                }

            option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
            option_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

            if (!name_str.equals(""))
                {
                option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }

            if (!value_str.equals("") && name_str.equals(StaticStrings.METADATA_STR))
                {
                // The metadata argument must be the fully qualified name of a metadata element, so if it doesn't yet have a namespace, append the extracted metadata namespace.
                String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                value_str = "";
                for (int k = 0; k <= values.length - 1; k++)
                    {
                    if (values[k].indexOf(StaticStrings.NS_SEP) == -1)
                        {
                        values[k] = StaticStrings.EXTRACTED_NAMESPACE + values[k];
                        }
                    else
                        {
                        MetadataElement metadata_element = MetadataTools.getMetadataElementWithName(values[k]);
                        if (metadata_element != null)
                            {
                            values[k] = metadata_element.getDisplayName();
                            }
                        }
                    if (k < values.length - 1)
                        {
                        value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;
                        }
                    else
                        {
                        value_str = value_str + values[k];
                        }
                    }
                }

            if (value_str != null && !value_str.equals(""))
                {
                XMLTools.setNodeText(option_element, value_str);
                }
            classify_element.appendChild(option_element);

            } // for each option
            
        //format element for this classifier
        Element format = (Element) XMLTools.getChildByTagName(e, StaticStrings.FORMAT_STR);
        if (format != null)
            {
            classify_element.appendChild(doFormat(to, format, null));
            }

        // Handling 'displayItem' element of this 'classifier' element - for now, just copy in and out so they don't get deleted
        NodeList di_children = e.getElementsByTagName(StaticStrings.DISPLAYITEM_STR);
            
        XMLTools.duplicateElementList(to, classify_element, di_children, true);
                        
        appendProperly(toElement, classify_element);
        }

    // default format statement for all classifiers
    Element default_classifier_format = (Element) XMLTools.getChildByTagName(browseNode, StaticStrings.FORMAT_STR);

    to.getDocumentElement().appendChild(doFormat(to, default_classifier_format, StaticStrings.BROWSE_STR));
    } // doClassifiers

    // Convert classify in the internal(i.e. Document from) to collectionconfig.xml (i.e. Document to)
    static private void convertClassifier(Document from, Document to)
    {
    Element browse_element = to.createElement(StaticStrings.BROWSE_STR);
    NodeList children = from.getDocumentElement().getElementsByTagName(StaticStrings.CLASSIFY_ELEMENT);

    int num_children = (children == null) ? 0 : children.getLength();

    if (num_children == 0) return;

    for (int i = 0; i < num_children; i++)
        {

        Element child = (Element) children.item(i);
        if (child.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

        String str = child.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
        Element classifier_element = to.createElement(StaticStrings.CLASSIFIER_STR);
        classifier_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, str);

        NodeList option_children = child.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
        for (int j = 0; j < option_children.getLength(); j++)
            {
            Element el = (Element) option_children.item(j);
            if (el.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

            String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String value_str = XMLTools.getNodeText(el);

            if (name_str == null && value_str == null) continue;

            Element option_element = to.createElement(StaticStrings.OPTION_STR);
            if (name_str != null && name_str.equals(StaticStrings.METADATA_STR))
                {

                // The metadata argument is the fully qualified name of a metadata element, so if it contains a namespace, remove the extracted metadata namespace as the build process doesn't know about it.
                String[] values = value_str.split(StaticStrings.COMMA_CHARACTER);
                value_str = "";
                for (int k = 0; k <= values.length - 1; k++)
                    {
                    if (values[k].startsWith(StaticStrings.EXTRACTED_NAMESPACE) && values[k].indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1)
                        {
                        values[k] = values[k].substring(StaticStrings.EXTRACTED_NAMESPACE.length());
                        }
                    else
                        {
                        MetadataElement metadata_element = MetadataTools.getMetadataElementWithDisplayName(values[k]);
                        if (metadata_element != null)
                            {
                            values[k] = metadata_element.getFullName();
                            }
                        }
                    if (k < values.length - 1)
                        {
                        value_str = value_str + values[k] + StaticStrings.COMMA_CHARACTER;
                        }
                    else
                        {
                        value_str = value_str + values[k];
                        }
                    }
                }

            if (!name_str.equals(""))
                {
                if (!name_str.startsWith(StaticStrings.MINUS_CHARACTER))
                    {
                    name_str = StaticStrings.MINUS_CHARACTER + name_str;
                    }
                option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }

            if (!value_str.equals(""))
                {
                option_element.setAttribute(StaticStrings.VALUE_ATTRIBUTE, value_str);
                }

            classifier_element.appendChild(option_element);
            }

        //format element for this classifier
        Element e = (Element) XMLTools.getChildByTagName(child, StaticStrings.FORMAT_STR);

        if (e != null)
            {
            classifier_element.appendChild(convertFormat(to, e));
            }

        // Handling 'displayItem' element of this 'classifier' element - for now, just copy in and out so they don't get deleted
        NodeList di_children = child.getElementsByTagName(StaticStrings.DISPLAYITEM_STR);
            
        XMLTools.duplicateElementList(to, classifier_element, di_children, true);
        browse_element.appendChild(classifier_element);
        }

    //convert default classifier format
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.BROWSE_STR);
    browse_element.appendChild(convertFormat(to, e));

    to.getDocumentElement().appendChild(browse_element);
    } // convertClassifier

    // Display ********************************************************

    static private void doDisplayFormat(Document to, Element from)
    {
    //display element in the xml file
    Element de = (Element) XMLTools.getChildByTagName(from, StaticStrings.DISPLAY_STR);
    if (de == null) return;

    //format element in the display element
    Element fe = (Element) XMLTools.getChildByTagName(de, StaticStrings.FORMAT_STR);

    to.getDocumentElement().appendChild(doFormat(to, fe, StaticStrings.DISPLAY_STR));
    }

    //convert format statement for display of the documents
    static private void convertDisplayFormat(Document from, Document to)
    {
    Element e = XMLTools.getNamedElement(from.getDocumentElement(), StaticStrings.FORMAT_STR, StaticStrings.NAME_ATTRIBUTE, StaticStrings.DISPLAY_STR);
    if (e == null) return;

    Element display = to.createElement(StaticStrings.DISPLAY_STR);
    display.appendChild(convertFormat(to, e));
    to.getDocumentElement().appendChild(display);
    }



    // Other top level elements *****************************************


    ////////////////////////////////////////////////////
    // replaceList and replaceListRef
    ///////////////////////////////////////////////////

    /*
     * replacelist currently not editable in GLI, just copy it in and back out
     * again
     */
    static private void doReplaceList(Document to, Element from)
    {
    Element toElement = to.getDocumentElement();

    Node rl_element = XMLTools.getChildByTagName(from, StaticStrings.REPLACELIST_STR);
    if (rl_element == null) return;

    Element to_element = XMLTools.duplicateElement(to, (Element) rl_element, true);
    toElement.appendChild(to_element);
    }

    static private void convertReplaceList(Document from, Document to)
    {
    Element toElement = to.getDocumentElement();

    Node rl_element = XMLTools.getChildByTagName(from.getDocumentElement(), StaticStrings.REPLACELIST_STR);
    if (rl_element == null) return;

    Element to_element = XMLTools.duplicateElement(to, (Element) rl_element, true);
    toElement.appendChild(to_element);
    }

    static private void doReplaceListRef(Document to, Element from)
    {
    Element toElement = to.getDocumentElement();

    NodeList replace_elements = from.getElementsByTagName(StaticStrings.REPLACELISTREF_STR);
    XMLTools.duplicateElementList(to, toElement, replace_elements, true);
    }


    static private void convertReplaceListRef(Document from, Document to)
    {
    Element toElement = to.getDocumentElement();

    NodeList replace_elements = from.getDocumentElement().getElementsByTagName(StaticStrings.REPLACELISTREF_STR);
    XMLTools.duplicateElementList(to, toElement, replace_elements, true);
    }

    ///////////////////////////////////////////////
    // service racks
    ////////////////////////////////////////////////

    /**
     * serviceRackList is currently not editable in GLI - just copy it in from
     * config file and write it out again.
     */
    static private void doServiceRackList(Document to, Element from)
    {
    Element toElement = to.getDocumentElement();

    Node srl_element = XMLTools.getChildByTagName(from, StaticStrings.SERVICE_RACK_LIST_ELEMENT);
    if (srl_element == null) return;

    Element to_element = XMLTools.duplicateElement(to, (Element) srl_element, true);
    toElement.appendChild(to_element);
    }

    static private void convertServiceRackList(Document from, Document to)
    {
    Element toElement = to.getDocumentElement();

    Node srl_element = XMLTools.getChildByTagName(from.getDocumentElement(), StaticStrings.SERVICE_RACK_LIST_ELEMENT);
    if (srl_element == null) return;

    Element to_element = XMLTools.duplicateElement(to, (Element) srl_element, true);
    toElement.appendChild(to_element);
    }


    ////////////////////////////////////////////////////
    // Unknown elements
    //////////////////////////////////////////////////////

    // handle top level elements which GLI knows nothing about
    // we store them internally in a Unknown element for easy access when
    // we write them out.
    static private void doUnknownElements(Document to, Element from)
    {
    Element toElement = to.getDocumentElement();
    Element unknownElement = to.createElement(StaticStrings.UNKNOWN_ELEMENT);
    toElement.appendChild(unknownElement);

    Node child = from.getFirstChild();
    while (child != null)
        {
        if (child.getNodeType() == Node.ELEMENT_NODE && !known_element_names.contains(child.getNodeName()))
            {
            unknownElement.appendChild(XMLTools.duplicateElement(to, (Element) child, true));
            }
        child = child.getNextSibling();
        }

    }

    // just copy all children of Unknown element into output doc.
    static private void convertUnknownElements(Document from, Document to)
    {

    Element toElement = to.getDocumentElement();
    Node unknown_element = XMLTools.getChildByTagName(from.getDocumentElement(), StaticStrings.UNKNOWN_ELEMENT);

    Node child = unknown_element.getFirstChild();
    while (child != null)
        {
        Element to_element = XMLTools.duplicateElement(to, (Element) child, true);
        toElement.appendChild(to_element);

        child = child.getNextSibling();
        }

    }


    //****************************************************************************

    ///////////////////////////////////////////////////
    // Helper functions
    /////////////////////////////////////////////////


    // /////////////////////////////////////////////
    // base code for index/sort/facet
    ///////////////////////////////////////////////

    static private void doBaseSearchPartInternal(Document to, Node searchNode, String new_parts_str, String new_part_str, String existing_part_str, String type, boolean is_mg_index) {

    Element toElement = to.getDocumentElement();
    Element parts_element = to.createElement(new_parts_str);//<Indexes> / <Facets> / <Sorts>
    parts_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);

    NodeList part_children = ((Element) searchNode).getElementsByTagName(existing_part_str);//index/facet/sort
    int num_nodes = part_children.getLength();
    
    for (int i = 0; i < num_nodes; i++)
        {

        Element part_element = to.createElement(new_part_str);//<Index> INDEX_ELEMENT
        Element e = (Element) part_children.item(i);
        String part_str = e.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        String part_str_display = part_str;//for creating searchmetadata for this index 
        if (!is_mg_index) {
            // look for options inside the index, eg
            // <option name="solrfieldtype" value="text_ja" />
            String options_str = "";
            NodeList option_children = e.getElementsByTagName(StaticStrings.OPTION_STR);
            if(option_children != null) {
            for (int j = 0; j < option_children.getLength(); j++) {
                Element el = (Element) option_children.item(j);
            
                String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                options_str = options_str + ((name_str.equals("")) ? "" : (" " + name_str));
            
                String value_str = el.getAttribute(StaticStrings.VALUE_ATTRIBUTE);
                options_str = options_str + ((name_str.equals("")) ? "" : (" " + value_str));
            
                if (name_str.equals("") && !value_str.equals(""))
                {
                    continue; //invalid condition
                }
            
                Element option_element = null;
                option_element = to.createElement(StaticStrings.OPTION_ELEMENT);
                option_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
            
                if (!name_str.equals(""))
                {
                    option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
                }
                if (value_str != null && !value_str.equals(""))
                {
                    XMLTools.setNodeText(option_element, value_str);
                }
                part_element.appendChild(option_element);
            }
            }
        }

        // Handling 'index' element
        // Double check colon separated style
        int colon_index = part_str.indexOf(StaticStrings.COLON_CHARACTER);
        if (!is_mg_index && colon_index != -1 )   {
            System.err.println("Something is wrong! the "+existing_part_str+" should NOT be level:source tuplets style.");
            part_str = part_str.substring(colon_index + 1);
            
        }
        if (is_mg_index) {
            if (colon_index == -1) {
            
            // MG, and it doesn't contain ':' character
            System.err.println("Something is wrong! the index should be level:source tuplets.");
            // assume document level
            part_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, StaticStrings.DOCUMENT_STR);
            }
            else {
            part_element.setAttribute(StaticStrings.LEVEL_ATTRIBUTE, part_str.substring(0, colon_index));
            
            part_str = part_str.substring(part_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
            }
        }
        //Each index/sort/facet may have a list of comma-separated strings.
        //split them into 'content' elements in the internal structure
        StringTokenizer content_tokenizer = new StringTokenizer(part_str, StaticStrings.COMMA_CHARACTER);
        while (content_tokenizer.hasMoreTokens())
            {
            // Replace part_str to be qualified name, eg. dc.Subject and keywords insread of dc.Subject.             
            
            Element content_element = to.createElement(StaticStrings.CONTENT_ELEMENT);
            String content_str = content_tokenizer.nextToken();
            // Since the contents of parts have to be certain keywords, or metadata elements, if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
            if (content_str.indexOf(StaticStrings.NS_SEP) == -1 && !(content_str.equals(StaticStrings.TEXT_STR) || content_str.equals(StaticStrings.ALLFIELDS_STR) || content_str.equals(StaticStrings.NONE_STR) || content_str.equals(StaticStrings.RANK_STR)))
                {
                content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;
                }
            content_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, content_str);
            part_element.appendChild(content_element);
            content_element = null;
            } //while ends

        parts_element.appendChild(part_element);
        
        part_element = null;

        // Handling 'displayItem' element of this 'index' element
        // 'e' is the parent element 'index' of 'displayItem' element
        doSearchDisplayItems(to, e, part_str_display, type);
        
        } // for loop ends
    toElement.appendChild(parts_element);
    //return toElement;

    } // doBaseSearchPartInternal

    // this code is used for indexes, sortfields, facets
    // internal_parts_str = Indexes/Sorts/Facets - internal doc
    // internal_part_str = Index/Sort/Facet - internal doc
    // output_part_str = index/sortfield/facet - what goes in collectionConfig

    private static void convertBaseSearchPartInternal(Document from, Document to, Element search, String internal_parts_str, String internal_part_str, String output_part_str, String searchmeta_type, boolean is_mg_index)
    {
    
    Element source = from.getDocumentElement();
    Element parts= XMLTools.getNamedElement(source, internal_parts_str, StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    if (parts == null) return;

    NodeList part_elements = parts.getElementsByTagName(internal_part_str);
    int part_elements_length = part_elements.getLength();
    
    if (part_elements_length == 0) return;

    for (int j = 0; j < part_elements_length; j++) {
        
        Element part_element = (Element) part_elements.item(j);
        if (part_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;
      
        Element part_ele = to.createElement(output_part_str);//part
      
        // Used for creating displayItem for this element 'part_ele' further below
        // full_part_names contain 'ex.'
        String full_part_name = ""; //why this string and below strin buffer??
        String level_str = "";
      
        StringBuffer part_value = new StringBuffer();
        if (is_mg_index) {
        level_str = part_element.getAttribute(StaticStrings.LEVEL_ATTRIBUTE);
        part_value.append(level_str).append(StaticStrings.COLON_CHARACTER);
        full_part_name = level_str + StaticStrings.COLON_CHARACTER;
        }

        NodeList content_elements = part_element.getElementsByTagName(StaticStrings.CONTENT_ELEMENT);
        int content_elements_length = content_elements.getLength();
      
        for (int k = 0; k < content_elements_length; k++) {
            
        Element content_element = (Element) content_elements.item(k);
        if (content_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

        String name_str = content_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        
        full_part_name = full_part_name + name_str;
        if (k < content_elements_length - 1) {
                
            full_part_name = full_part_name + StaticStrings.COMMA_CHARACTER;
        }
        
        if (name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && name_str.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1) {
                
            name_str = name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
        }
        
        part_value.append(name_str);
        name_str = null;
        // Make it comma separated string
        if (k < content_elements_length - 1) {
                
            part_value.append(StaticStrings.COMMA_CHARACTER);
        }
        content_element = null;
        } //k for loop ends

        // ok to here
        String temp_str = part_value.toString();
        part_ele.setAttribute(StaticStrings.NAME_ATTRIBUTE, temp_str);
      
        // Now constructing 'displayItem' element for this 'part_ele' element
        // The part names in the searchmetadata elements in the internal structure are not the names that
        // are used in the content elements (i.e. ex.Source or dc.Subject and keywords), but the names that are
        // in the configuration files (i.e. Source or dc.Subject)
        ArrayList searchmetadata_list = getMatchingSearchMetaElements(source, temp_str, searchmeta_type); 
   
        if (searchmetadata_list == null) {
            
        //try the full name, i.e. with 'ex.'
        searchmetadata_list = getMatchingSearchMetaElements(source, full_part_name, searchmeta_type);
        }
      
        if (searchmetadata_list != null) {
            
        
        for (int k = 0; k < searchmetadata_list.size(); k++) {
                
            Element searchmetadata = (Element) searchmetadata_list.get(k);
            if (searchmetadata.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

            Element displayItem = constructDisplayItem(searchmetadata, to);
          
            part_ele.appendChild(displayItem);
        }
        }
    
        if (!is_mg_index) {
        // deal with any <option name='' value=''> children of this part
        // e.g. <option name='solrfieldtype' value='text_es'>
        NodeList option_children = part_element.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
        for (int k = 0; k < option_children.getLength(); k++) {
            
            Element el = (Element) option_children.item(k);
            if (el.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

            String name_str = el.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            String value_str = XMLTools.getNodeText(el);
          
            if (name_str == null && value_str == null) continue;
            Element option_element = to.createElement(StaticStrings.OPTION_STR);
            if (!name_str.equals("")) {
            option_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
            }
            if (!value_str.equals("")) { // or no value attribute without name attribute?
            option_element.setAttribute(StaticStrings.VALUE_ATTRIBUTE, value_str);      
            }
            part_ele.appendChild(option_element);
        } // for each option
        }
        search.appendChild(part_ele);
      
    } //for loop ends

    } // convertBaseSearchPartInternal

    // find searchMetas (which become display items) matching meta_name and meta_type
    static private ArrayList getMatchingSearchMetaElements(Element source, String meta_name, String meta_type) {
    return XMLTools.getNamedElementList(source, StaticStrings.SEARCHMETADATA_ELEMENT, new String[]{StaticStrings.NAME_ATTRIBUTE, StaticStrings.TYPE_ATTRIBUTE}, new String[]{ meta_name, meta_type});

    }

    // 
    static private void convertBaseDefaultSearchPartInternal(Document from, Document to, Element search, String internal_default_part_str, String output_default_part_str, boolean is_mg_index)
    {
    Element source = from.getDocumentElement();

    Element default_part_element = (Element) XMLTools.getChildByTagName(source, internal_default_part_str);
    if (default_part_element == null)
        {
        return;
        }

    String level_str = "";
    if (is_mg_index) {
        level_str = default_part_element.getAttribute(StaticStrings.LEVEL_ATTRIBUTE);
        // Debugging purposes
        if (level_str.equals("")) {
        
        System.out.println("Bug: DefaultIndex should have its level attribute not empty. Setting it to document");
        level_str = StaticStrings.DOCUMENT_STR;
        }
    }
    NodeList content_elements = default_part_element.getElementsByTagName(StaticStrings.CONTENT_ELEMENT);
    int content_elements_length = content_elements.getLength();

    // Don't output anything if no parts are set
    if (content_elements_length == 0) return;

    String part_str = "";

    if (is_mg_index) {
    
        //combine level with indexes
        part_str = level_str + StaticStrings.COLON_CHARACTER;
    }

    for (int k = 0; k < content_elements_length; k++) {
        
        Element content_element = (Element) content_elements.item(k);
        if (content_element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.FALSE_STR)) continue;

        String name_str = content_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);

        if (name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE) && name_str.indexOf(StaticStrings.NS_SEP, StaticStrings.EXTRACTED_NAMESPACE.length()) == -1) {
            
        name_str = name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
        }

        part_str = part_str + name_str;

        // Make it comma separated string
        if (k < content_elements_length - 1) {
            
        part_str = part_str + StaticStrings.COMMA_CHARACTER;
        }
        content_element = null;
    }//for loop ends
    
    Element default_part = to.createElement(output_default_part_str);
    default_part.setAttribute(StaticStrings.NAME_ATTRIBUTE, part_str);
    search.appendChild(default_part);

    } // convertBaseDefaultSearchPartInternal
    

    ////////////////////////////////
    // handling format statement code
    ////////////////////////////////

    static private Element doFormat(Document to, Element format, String name_str)
    {
    Element format_element = to.createElement(StaticStrings.FORMAT_STR);
    if (name_str != null) {
        
        format_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name_str);
    }

    // Don't write out an empty format statement of <format/> (i.e. format has no child nodes)
    // as this will end up embedded in another format statement as <format><format/><format />
    // This doubling up of format stmts will then prevent GLI from opening the collection again.
    if (format != null && format.hasChildNodes()) {
        // not an empty format statement
        String gsf_text = XMLTools.xmlNodeToString(format);

        //We don't want the <format> elements in the string
        int startFormatIndex = gsf_text.indexOf(StaticStrings.FORMAT_START_TAG) + StaticStrings.FORMAT_START_TAG.length();
        int endFormatIndex = gsf_text.lastIndexOf(StaticStrings.FORMAT_END_TAG);
        gsf_text = gsf_text.substring(startFormatIndex, endFormatIndex);

        XMLTools.setNodeText(format_element, gsf_text);
    }
    return format_element;
    }

    static private Element convertFormat(Document to, Element e)
    {
    String format_str = XMLTools.getNodeText(e);
    Element format = to.createElement(StaticStrings.FORMAT_STR);
    //XMLTools.copyAllChildren (format, e);
    XMLTools.setNodeText(format, format_str);
    return format;
    }

    ///////////////////////////////////////////////////////////////
    // helper code for display items
    ///////////////////////////////////////////////////////////////


    static private void doSearchDisplayItems(Document to, Element from, String name, String type) {
    Element toElement = to.getDocumentElement();
    // Node displayItemListNode = XMLTools.getChildByTagNameIndexed(from, StaticStrings.DISPLAYITEMLIST_STR, 0);
    // if (displayItemListNode == null) {
    //     return;
    // }
    NodeList displayItems = from.getElementsByTagName(StaticStrings.DISPLAYITEM_STR);
    for (int i=0; i<displayItems.getLength(); i++) {
        Element item = (Element) displayItems.item(i);
        Element search_meta = processSingleDisplayItem(to, item, name, type);
        if (search_meta == null) {
        System.err.println("search meta is null "+name+", "+type);
        }else {
        toElement.appendChild(search_meta);
        }
    }
    }

    /** convert displayItem (item) into CollectionMeta (if type is null) or SearchMeta (if type is not null). Use new_name if not null. */
    static private Element processSingleDisplayItem(Document to, Element item, String new_name, String type) {
    // special case for collection description - is allowed html
    boolean isCollDescr = new_name.equals(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR);

    // Sadly, XMLTools.getNodeText(item) returns empty for any HTML in displayItem
    // so we have to do it the hard way
    // We can't take a shortcut like format statements, which also need to keep their
    // tags intact in collConfig.xml, because we know in advance the format tag name
    // Can't guess at what html tag names the user uses, and there may be several at
    // the same level inside collDescription (e.g. multiple paragraphs)
    String text = isCollDescr ? preserveHTMLInDescriptionDisplayItem(item)
        : XMLTools.getNodeText(item); // else as before for all other displayItems
    
    //If there is nothing to display, don't bother creating the element
    // Either lang or key should be set. If key set, no text node needed.
    // if lang set, should have textnode. Check if it's meaningful
    if (item.hasAttribute(StaticStrings.LANG_STR) && text.equals("")) {
        return null;
    }
    
    // get the value in 'lang=langcode' or 'key=keyname'
    // We can have either a key attribute (with optional dictionary attribute, else dictionary is implicit)
    // OR we can have a lang attr, with the nodetext containing the actual display string.
    String lang = item.getAttribute(StaticStrings.LANG_STR);
    String key = item.getAttribute(StaticStrings.KEY_ATTRIBUTE);
    String name;
    if (new_name != null) {
        name = new_name;
    } else {
        name = item.getAttribute(StaticStrings.NAME_ATTRIBUTE);
    }


    Element e;
    if (type ==null) {
        e = to.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
    } else {
        e = to.createElement(StaticStrings.SEARCHMETADATA_ELEMENT);
    }
    e.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
    e.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
    
    if (type != null) {
        e.setAttribute(StaticStrings.TYPE_ATTRIBUTE, type);
    }
    if(!lang.equals("")) {
        e.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, lang);
        text = text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
        XMLTools.setNodeText(e, text);
        
    } else if(!key.equals("")) {
        e.setAttribute(StaticStrings.KEY_ATTRIBUTE, key);
        String dictionary = item.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
        if(!dictionary.equals("")) {
        e.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary);
        }
    }
    return e;
    }


    /** Construct s <displayItem> element from e which is a CollectionMeta or a SearchMeta
    i.e. internal DOM -> config file
    */
    static private Element constructDisplayItem(Element e, Document to, String name)
    {
    String lang_string = e.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE);
    String key_string = e.getAttribute(StaticStrings.KEY_ATTRIBUTE);
    String dictionary_string = e.getAttribute(StaticStrings.DICTIONARY_ATTRIBUTE);
    String text = XMLTools.getNodeText(e);

    Element displayItem = to.createElement(StaticStrings.DISPLAYITEM_STR);
    displayItem.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);

    if(!lang_string.equals("")) {
        displayItem.setAttribute(StaticStrings.LANG_STR, lang_string);
        if (name.equals(StaticStrings.DESCRIPTION_STR)) {
        // special case for collection description string, which is allowed html tags
        text = Codec.transform(text, Codec.REINSTATE_HTML_TAGS);
        }
        XMLTools.setNodeText(displayItem, text);
    }
    if(!key_string.equals("")) {
        displayItem.setAttribute(StaticStrings.KEY_ATTRIBUTE, key_string);
        if(!dictionary_string.equals("")) {
        displayItem.setAttribute(StaticStrings.DICTIONARY_ATTRIBUTE, dictionary_string);
        }
    }
    return displayItem;
    }

    static private Element constructDisplayItem(Element e, Document to)
    {   
    return constructDisplayItem(e, to, StaticStrings.NAME_ATTRIBUTE);
    }



    ////////////////////////////////////////////////////////////////////
    // DOM tree manipulation
    ///////////////////////////////////////////////////////////////////

    // Append the element son to the element mother in the appropriate position.
    static private void appendProperly(Element mother, Element son)
    {
    if (son == null)
        return;

    Node reference_node = findInsertionPoint(mother, son);
    if (reference_node != null) {
        mother.insertBefore(son, reference_node);
    }
    else {
        mother.appendChild(son);
    }
    }

    /**
     * Find the best insertion position for the given DOM Element
     * 'target_element' in the DOM Element 'document_element'. This should try
     * to match command tag, and if found should then try to group by name or
     * type (eg CollectionMeta), or append to end is no such grouping exists (eg
     * Plugins). Failing a command match it will check against the command order
     * for the best insertion location.
     * 
     * @param target_element
     *            the command Element to be inserted
     * @return the Element which the given command should be inserted before, or
     *         null to append to end of list
     */
    static private Node findInsertionPoint(Element document_element, Element target_element)
    {
    ///ystem.err.println("Find insertion point: " + target_element.getNodeName());
    String target_element_name = target_element.getNodeName();

    // Try to find commands with the same tag.
    NodeList matching_elements = document_element.getElementsByTagName(target_element_name);
    // If we found matching elements, then we have our most likely insertion location, so check within for groupings
    if (matching_elements.getLength() != 0) {
        
        ///ystem.err.println("Found matching elements.");
        // Only CollectionMeta are grouped.
        if (target_element_name.equals(StaticStrings.COLLECTIONMETADATA_ELEMENT)) {
        ///ystem.err.println("Dealing with collection metadata");
        // Special case: CollectionMeta can be added at either the start or end of a collection configuration file. However the start position is reserved for special metadata, so if no non-special metadata can be found we must append to the end.
        // So if the command to be added is special add it immediately after any other special command
        if (target_element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR)) {
                 
            int index = 0;
            Element matched_element = (Element) matching_elements.item(index);
            Element sibling_element = (Element) matched_element.getNextSibling();
            while (sibling_element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR)) {
                    
            index++;
            matched_element = (Element) matching_elements.item(index);
            sibling_element = (Element) matched_element.getNextSibling();
            }
            if (sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT)) {
                    
            Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
            document_element.insertBefore(newline_element, sibling_element);
            }
            return sibling_element;
        }
        // Otherwise try to find a matching 'name' and add after the last one in that group.
        else {
                
            int index = 0;
            target_element_name = target_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
            boolean found = false;
            // Skip all of the special metadata
            Element matched_element = (Element) matching_elements.item(index);
            while (matched_element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR)) {
                    
            index++;
            matched_element = (Element) matching_elements.item(index);
            }
            // Begin search
            while (!found && matched_element != null) {
                    
            if (matched_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equals(target_element_name)) {
                        
                found = true;
            }
            else {
                    
                index++;
                matched_element = (Element) matching_elements.item(index);
            }
            }
            // If we found a match, we need to continue checking until we find the last name match.
            if (found) {
                    
            index++;
            Element previous_sibling = matched_element;
            Element sibling_element = (Element) matching_elements.item(index);
            while (sibling_element != null && sibling_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equals(target_element_name)) {
    
                previous_sibling = sibling_element;
                index++;
                sibling_element = (Element) matching_elements.item(index);
            }
            // Previous sibling now holds the command immediately before where we want to add, so find its next sibling and add to that. In this one case we can ignore new lines!
            return previous_sibling.getNextSibling();
            }
            // If not found we just add after last metadata element
            else {
                    
            Element last_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
            return last_element.getNextSibling();
            }
        }

        }
        else {
          
        ///ystem.err.println("Not dealing with collection meta.");
        Element matched_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
        // One final quick test. If the matched element is immediately followed by a NewLine command, then we insert another NewLine after the matched command, then return the NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
        Node sibling_element = matched_element.getNextSibling();
        if (sibling_element != null && sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT)) {
                
            Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
            document_element.insertBefore(newline_element, sibling_element);
        }
        return sibling_element; // Note that this may be null
        }
    }
    ///ystem.err.println("No matching elements found.");
    // Locate where this command is in the ordering
    int command_index = -1;
    for (int i = 0; command_index == -1 && i < CollectionConfiguration.COMMAND_ORDER.length; i++) {
        
        if (CollectionConfiguration.COMMAND_ORDER[i].equals(target_element_name)) {
    
        command_index = i;
        }
    }
    ///ystem.err.println("Command index is: " + command_index);
    // Now move forward, checking for existing elements in each of the preceeding command orders.
    int preceeding_index = command_index - 1;
    ///ystem.err.println("Searching before the target command.");
    while (preceeding_index >= 0) {
        
        matching_elements = document_element.getElementsByTagName(CollectionConfiguration.COMMAND_ORDER[preceeding_index]);
        // If we've found a match
        if (matching_elements.getLength() > 0) {
            
        // We add after the last element
        Element matched_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
        // One final quick test. If the matched element is immediately followed by a NewLine command, then we insert another NewLine after the matched command, then return the NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
        Node sibling_element = matched_element.getNextSibling();
        if (sibling_element != null && sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT)) {
                
            Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
            document_element.insertBefore(newline_element, sibling_element);
        }
        return sibling_element; // Note that this may be null
        }
        preceeding_index--;
    }
    // If all that fails, we now move backwards through the commands
    int susceeding_index = command_index + 1;
    ///ystem.err.println("Searching after the target command.");
    while (susceeding_index < CollectionConfiguration.COMMAND_ORDER.length) {

        matching_elements = document_element.getElementsByTagName(CollectionConfiguration.COMMAND_ORDER[susceeding_index]);
        // If we've found a match
        if (matching_elements.getLength() > 0) {
            
        // We add before the first element
        Element matched_element = (Element) matching_elements.item(0);
        // One final quick test. If the matched element is immediately preceeded by a NewLine command, then we insert another NewLine before the matched command, then return this new NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
        Node sibling_element = matched_element.getPreviousSibling();
        if (sibling_element != null && sibling_element.getNodeName().equals(CollectionConfiguration.NEWLINE_ELEMENT)) {
                
            Element newline_element = document_element.getOwnerDocument().createElement(CollectionConfiguration.NEWLINE_ELEMENT);
            document_element.insertBefore(newline_element, sibling_element);
        }
        return sibling_element; // Note that this may be null
        }
        susceeding_index++;
    }
    // Well. Apparently there are no other commands in this collection configuration. So append away...
    return null;
    }


    /**
     * For now it only makes sense to allow a collection's about page description to contain HTML.
     * Collection titles may go into a pre tag or text only context or into HTML headings,
     * where preserving any HTML in coll title would become meaningless or even a hindrance.
     * TODO: if users used HTML pre tags, they'll have wanted whitespace preserved.
     * TODO There's probably a better way to do this, but I'm unable to think of it. Messy but works.
     */
    static private String preserveHTMLInDescriptionDisplayItem(Element collDescription) {
    // First get the <displayItem> as text
    String text = XMLTools.elementToString(collDescription, true);

    // We only want the contents, not the xml processing instruction or the
    // <displayItem></displayItem> or self-closing(!) <displayItem/> when no text for colldescr
    // Remove xml processing instr too by removing <displayItem until first > after that
    // This will also handle the self-closing tag case
    String lookFor = "<displayItem";
    int start = text.indexOf(lookFor);
    if(start != -1) {
        start += lookFor.length();
        start = text.indexOf(">", start);
        text = text.substring(start+1).trim(); // trim to ensure no empty lines to deceive us
        // especially after a self-closing tag
    }
    if ( text.equals("")) {
        //must have had self closing tag
        return "";
    }
    lookFor = "</displayItem>";
    int end = text.indexOf(lookFor);
    if (end == -1) {
        // there was no closing tag. something has gone wrong
        return "";
    }
    text = text.substring(0,end);

    // in case opening the file has indented the text, we strip off whitespace for each line
    String[] lines = text.split("\\r?\\n");
    text = "";              
    if(lines.length > 0) {
        for (int j = 0; j < lines.length; j++) { 
        text += lines[j].trim() + "\n"; // Easiest solution:
        // trim white space of the one extra level of indentation
        // that became apparent when enclosing <dispItem> tags removed
        }
    }

    text = text.replaceAll("&amp;", "&");
    return text;

    }

    // Append the elements, which are of Element type, in 'list' to Element 'to'
    static private void appendArrayList(Element to, ArrayList list)
    {
    if (list == null)
        return;

    for (int i = 0; i < list.size(); i++) {
        
        appendProperly(to, (Element) list.get(i));
    }
    }


}