/*
 *    GS2Retrieve.java
 *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
package org.greenstone.gsdl3.service;


// Greenstone classes
import org.greenstone.gdbm.*;
import org.greenstone.gsdl3.util.*;

// XML classes
import org.w3c.dom.Document;
import org.w3c.dom.Element; 
import org.w3c.dom.NodeList;

// General Java classes
import java.io.File;
import java.util.StringTokenizer;
import java.util.Vector;
import java.util.Set;
import java.util.Iterator;
import java.util.ArrayList;

/** Implements the generic retrieval and classifier services for GS2 
 * collections.
 *
 * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
 * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
 * @version $Revision: 6873 $
 */

public abstract class GS2Retrieve
    extends ServiceRack {

    // the services on offer
    // these strings must match what is found in the properties file
    protected static final String DOCUMENT_STRUCTURE_RETRIEVE_SERVICE = "DocumentStructureRetrieve";
    protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
    protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";


    // the browsing services -  now in here, these will only be advertised if classifiers have been specified in the config file
    private static final String CLASSIFIER_SERVICE = "ClassifierBrowse";
    private static final String CLASSIFIER_METADATA_SERVICE = "ClassifierBrowseMetadataRetrieve";
    
    protected static final String STRUCT_PARAM = "structure";
    protected static final String INFO_PARAM = "info";
    
    protected static final String STRUCT_ANCESTORS = "ancestors";
    protected static final String STRUCT_PARENT = "parent";
    protected static final String STRUCT_SIBS = "siblings";
    protected static final String STRUCT_CHILDREN = "children";
    protected static final String STRUCT_DESCENDS = "descendants";
    protected static final String STRUCT_ENTIRE = "entire";

    protected static final String INFO_NUM_SIBS = "numSiblings";
    protected static final String INFO_NUM_CHILDREN = "numChildren";
    protected static final String INFO_SIB_POS = "siblingPosition";

    protected static final int DOCUMENT=1;
    protected static final int CLASSIFIER=2;

    protected GDBMWrapper gdbm_src = null;
    protected Element config_info = null; // the xml from the config file

    protected ArrayList replacements = null;
    /** constructor */
    protected GS2Retrieve()
    {
	this.gdbm_src = new GDBMWrapper();
    }


    /** configure this service */
    public boolean configure(Element info, Element extra_info)
    {
	System.out.println("Configuring GS2Retrieve...");
	this.config_info = info;

	// set up short_service_info_ - for now just has name and type
	Element dsr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
	this.short_service_info.appendChild(dsr_service);

	Element dmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
	this.short_service_info.appendChild(dmr_service);

	Element dcr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
	this.short_service_info.appendChild(dcr_service);

	// Open GDBM database for querying
	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name);
	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
	    System.err.println("GS2Retrieve Error: Could not open GDBM database!");
	    return false;
	}

        // now do the classifier browse service
 
        // check that there are classifiers specified
        Element class_list = (Element)GSXML.getChildByTagName(info, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
        if (class_list == null) {
            // no classifiers specified
            return true;
        }
	
	// get the display and format elements from the coll config file for
	// the classifiers
	extractExtraClassifierInfo(info, extra_info);
	
	this.config_info = info;
	
        // short_service_info_ - the browse one
        Element cb_service = this.doc.createElement(GSXML.SERVICE_ELEM);
        cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
        cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
        this.short_service_info.appendChild(cb_service);
	
	// metadata retrieval for the browsing  
        Element cbmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
        cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
        cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
        this.short_service_info.appendChild(cbmr_service);

	// the format info
	Element cb_format_info = this.doc.createElement(GSXML.FORMAT_ELEM);
	boolean format_found = false;

	// try the default format first
	Element def_format = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
	if (def_format != null) {
	    cb_format_info.appendChild(GSXML.duplicateWithNewName(this.doc, def_format, GSXML.DEFAULT_ELEM, true));
	    format_found = true;
	}
	
	// add in to the description a simplified list of classifiers
        NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
	for(int i=0; i<classifiers.getLength(); i++) {
	    Element cl = (Element)classifiers.item(i);
	    Element new_cl = (Element)this.doc.importNode(cl, false); // just import this node, not the children
	    
	    // get the format info out, and put inside a classifier element
	    Element format_cl = (Element)new_cl.cloneNode(false);
	    Element format = (Element)GSXML.getChildByTagName(cl, GSXML.FORMAT_ELEM);
	    if (format != null) {
		
		//copy all the children
		NodeList elems = format.getChildNodes();
		for (int j=0; j<elems.getLength();j++) {
		    format_cl.appendChild(this.doc.importNode(elems.item(j), true));
		}
		cb_format_info.appendChild(format_cl);
		format_found = true;
	    }
	    
	    
	}
	    
	if (format_found) {
	    this.format_info_map.put(CLASSIFIER_SERVICE, cb_format_info);
	}

	// look for document display format
	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
	if (display_format != null) {
	    this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.doc.importNode(display_format, true));
	    // shoudl we make a copy?
	}

	// set up the replacement list
	Element replacement_elem = (Element)GSXML.getChildByTagName(extra_info, "replaceList");
	if (replacement_elem != null) {
	    this.replacements = new ArrayList();
	    NodeList replaces = replacement_elem.getElementsByTagName("replace");
	    for (int i=0; i<replaces.getLength(); i++) {
		Element e = (Element)replaces.item(i);
		String from = e.getAttribute("from");
		String to = e.getAttribute("to");
		System.out.println("replace "+from+" with "+to);
		this.replacements.add(from);
		this.replacements.add(to);
	    }
	}
	return true;
    }

    protected Element getServiceDescription(String service_id, String lang, String subset) {

	if (service_id.equals(CLASSIFIER_SERVICE)) {
	    
	    Element class_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
	    if (class_list == null) {
            // no classifiers specified
		return null;
	    }
	    
	    Element cb_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	    cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
	    cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
	    cb_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  getTextString(CLASSIFIER_SERVICE+".name", lang)));
	    cb_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  getTextString(CLASSIFIER_SERVICE+".description", lang)));
	    
	    Element cl_list = this.doc.createElement(GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
	    cb_service.appendChild(cl_list);
	    NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
	    for(int i=0; i<classifiers.getLength(); i++) {
		Element cl = (Element)classifiers.item(i);
		Element new_cl = (Element)this.doc.importNode(cl, false); // just import this node, not the children
		String content = cl.getAttribute(GSXML.CLASSIFIER_CONTENT_ATT);
		cl_list.appendChild(new_cl);
		String text = GSXML.getDisplayText(cl, 
						   GSXML.DISPLAY_TEXT_NAME, 
						   lang, "en");
		if (text == null || text.equals("")) {
		    // no display element was specified, use the metadata name
		    // for now this looks in the class properties file
		    // this needs to use a general metadata thing instead
		    text = getMetadataNameText(content+".buttonname", lang);
		}
		if (text == null) {
		    text = content;
		}
		
		Element cl_name = GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, text);
		new_cl.appendChild(cl_name);

		// description
		
		String meta_name = getMetadataNameText(content, lang);
		if (meta_name==null) {
		    meta_name = content;
		}
		String [] array = {meta_name};
		String description = getTextString("ClassifierBrowse.classifier_help", array, lang);
		Element cl_desc = GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, description);
		new_cl.appendChild(cl_desc);
		
	    }
	    return cb_service;
	}
	
	// these ones are probably never called, but put them here just in case
	
	if (service_id.equals(CLASSIFIER_METADATA_SERVICE)) {
	    
	    Element cbmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	    cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	    cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
	    return cbmr_service;
	}

	if (service_id.equals(DOCUMENT_STRUCTURE_RETRIEVE_SERVICE)) {
	    Element dsr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	    dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	    dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
	    return dsr_service;
	}
	if (service_id.equals(DOCUMENT_METADATA_RETRIEVE_SERVICE)) {
	    Element dmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	    dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	    dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
	    return dmr_service;
	}

	if (service_id.equals(DOCUMENT_CONTENT_RETRIEVE_SERVICE)) {
	    Element dcr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
	    dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
	    dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
	    return dcr_service;
	}

	return null;
    }

    /** this looks for any classifier specific display or format info from extra_info and adds it in to the correct place in info */
    protected boolean extractExtraClassifierInfo(Element info, Element extra_info) {
	
	if (extra_info == null) {
	    return false;
	}
	
	Document owner = info.getOwnerDocument();
	// so far we have display and format elements that we need for classifiers
	NodeList classifiers = info.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
	Element config_browse = (Element)GSXML.getChildByTagName(extra_info, GSXML.BROWSE_ELEM);

	for (int i=0; i<classifiers.getLength();i++) {
	    Element cl = (Element)classifiers.item(i);
	    String name = cl.getAttribute(GSXML.NAME_ATT);
	    Element node_extra = GSXML.getNamedElement(config_browse,
						       GSXML.CLASSIFIER_ELEM,
						       GSXML.NAME_ATT,
						       name);
	    if (node_extra == null) {
		System.err.println("GS2REtrieve: haven't found extra info for classifier named "+name);
		continue;
	    }
	    
	    // get the display elements if any - displayName
	    NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
	    if (display_names !=null) {
		Element display = owner.createElement(GSXML.DISPLAY_ELEM);
		for (int j=0; j<display_names.getLength(); j++) {
		    Element e = (Element)display_names.item(j);
		    cl.appendChild(owner.importNode(e, true));
		    
		}
	    }
	    
	    // get the format element if any 
	    Element format = (Element)GSXML.getChildByTagName(node_extra, GSXML.FORMAT_ELEM);
	    if (format==null) { // try a generic one that applies to all classifiers
		format = (Element)GSXML.getChildByTagName(extra_info,
							  GSXML.FORMAT_ELEM);
	    }
	    if (format!=null) { // append to index info
		cl.appendChild(owner.importNode(format, true));
	    }
	} // for each classifier

	// now check for default format info
	Element default_format = (Element)GSXML.getChildByTagName(config_browse, GSXML.FORMAT_ELEM);
	if (default_format!=null) { // append to  info
	    info.appendChild(owner.importNode(default_format, true));
	}

	return true;
    }
	    

    /** parent is true if this node is definitely the  parent of something, 
     * child is true is it definitely is a child of something - just for efficiency purposes */
    protected Element createDocNode(String node_id, boolean parent, boolean child) {
	
	// create this here or pass it in?
	DBInfo info = this.gdbm_src.getInfo(node_id);
	Element node;
	if (isClassifier(node_id)) {
	    node = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
	    //String childtype = info.getInfo("childtype");
	    //String orientation="";
	    //if (childtype.equals("HList")) {
	    //	orientation = "horizontal";
	    //} else { // assume vertical
	    //	orientation = "vertical";
	    //}
	    //node.setAttribute(GSXML.CLASS_NODE_ORIENTATION_ATT, orientation);
	} else {
	    
	    node = this.doc.createElement(GSXML.DOC_NODE_ELEM);

	    String top_id = OID.getTop(node_id);
	    boolean is_top = (top_id.equals(node_id) ? true : false);
	
	    String children = info.getInfo("contains");
	    boolean is_leaf = (children.equals("") ? true : false);
	    
	    // set teh node type att
	    if (is_top) {
		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_ROOT);
	    } else if (is_leaf) {
		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_LEAF);
	    } else {
		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_INTERIOR);
	    }

	    // set teh doc type att
	    if (is_top && is_leaf) { // a single section document
		node.setAttribute(GSXML.DOC_TYPE_ATT, "simple");
		
	    } else {
	    
		if (!is_top) { // we need to look at the top info
		    info = this.gdbm_src.getInfo(top_id);
		}
		
		String childtype = info.getInfo("childtype");
		if (childtype.equals("Paged")) {
		    node.setAttribute(GSXML.DOC_TYPE_ATT, "paged");
		} else {
		    node.setAttribute(GSXML.DOC_TYPE_ATT, "hierarchy");
		}
	    }
	    
	}
	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
	return node;

    }
    /** Returns the parent of a specified documentID, or null if none exists */
    protected Element getParent(String doc_id)
    {
	String parent_id = OID.getParent(doc_id);
	if (parent_id.equals(doc_id))
	    return null;
	
	return createDocNode(parent_id, true, false);
    }


    /** adds all the children of doc_id the the doc element,
     * and if recursive=true, adds all their children as well*/
    protected void addDescendants(Element doc, String doc_id, 
				  boolean recursive)
    {
	DBInfo info = this.gdbm_src.getInfo(doc_id);
	String contains = info.getInfo("contains");

	StringTokenizer st = new StringTokenizer(contains, ";");
	while (st.hasMoreTokens()) {
	    String child_id = st.nextToken().replaceAll("\"", doc_id);
	    Element child = createDocNode(child_id, false, true);
	    doc.appendChild(child);

	    // Apply recursively, if desired
	    if (recursive) {
		addDescendants(child, child_id, recursive);
	    }
	    
	}
    }

    /** adds all the siblings of current_id to the parent element. */
    protected Element addSiblings(Element parent, String parent_id, String current_id) {
	Element current_node = (Element)parent.getFirstChild();
	if (current_node ==null) {
	    // create a sensible error message
	    System.err.println("GS2Retrieve Error: there should be a first child.");
	    return null;
	}
	// remove the current child,- will add it in later in its correct place
	parent.removeChild(current_node);

	// add in all the siblings,
	addDescendants(parent, parent_id, false);

	// find the node that is now the current node
	// this assumes that the new node that was created is the same as 
	// the old one that was removed - we may want to replace the new one 
	// with the old one.
	Element new_current = GSXML.getNamedElement(parent, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
	return new_current;

    }
    /** Returns true if the OID specifies a leaf node, false otherwise
	Note: this makes a request to the GDBM database so it may not be
	      a particularly cheap operation */
    protected boolean isLeafNode(String oid)
    {
	DBInfo info = this.gdbm_src.getInfo(oid);
	String children = info.getInfo("contains");
	return (children.equals(""));
    }

    // for now just use CL for classifiers - should have a type? in teh gdbm
    // database.
    protected boolean isClassifier(String oid) {
	if (oid.startsWith("CL")) {
	    return true;
	}
	return false;
    }

    protected Element processDocumentStructureRetrieve(Element request) {
	return genericStructureRetrieve(request, DOCUMENT);
    }

    protected Element processClassifierBrowse(Element request) {
	return genericStructureRetrieve(request, CLASSIFIER);
    }

    /** Retrieve the structure of a document */
    protected Element genericStructureRetrieve(Element request, int type)
    {
	// Create a new (empty) result message
	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
	
	String node_name;
	String service_name;
	if (type==DOCUMENT) {
	    service_name = DOCUMENT_STRUCTURE_RETRIEVE_SERVICE;
	    node_name = GSXML.DOC_NODE_ELEM;
	} else {
	    service_name = CLASSIFIER_SERVICE;
	    node_name = GSXML.CLASS_NODE_ELEM;
	}
	
	result.setAttribute(GSXML.FROM_ATT, service_name);
	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);


	Element query_doc_list = (Element) GSXML.getChildByTagName(request, node_name+GSXML.LIST_MODIFIER);
	if (query_doc_list == null) {
	    System.err.println("GS2Retrieve Error: DocumentStructureRetrieve request specified no doc nodes.\n");
	    return result;
	}

	// Get the parameters of the request
	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
	if (param_list == null) {
	    System.err.println("GS2Retrieve Error: DocumentStructureRetrieve request had no paramList.");
	    return result;  // Return the empty result
	}

	// the type of info required
	boolean want_structure = false;
	boolean want_info = false;

	Vector info_types=new Vector();
	// The document structure information desired
	boolean want_ancestors = false;
	boolean want_parent = false;
	boolean want_siblings = false;
	boolean want_children = false;
	boolean want_descendants = false;

	boolean want_entire_structure = false;
	// Process the request parameters
	NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
	for (int i=0; i<params.getLength();i++) {
	    
	    Element param = (Element)params.item(i);
	    String p_name = param.getAttribute(GSXML.NAME_ATT);
	    String p_value = GSXML.getValue(param);
	    // Identify the structure information desired
	    if (p_name.equals(STRUCT_PARAM)) {
		want_structure = true;
		
		// This is NOT locale sensitive
		if (p_value.equals(STRUCT_ANCESTORS))
		    want_ancestors = true;
		else if (p_value.equals(STRUCT_PARENT))
		    want_parent = true;
		else if (p_value.equals(STRUCT_SIBS))
		    want_siblings = true;
		else if (p_value.equals(STRUCT_CHILDREN))
		    want_children = true;
		else if (p_value.equals(STRUCT_DESCENDS))
		    want_descendants = true;
		else if (p_value.equals(STRUCT_ENTIRE))
		    want_entire_structure = true;
		else
		    System.err.println("GS2Retrieve Warning: Unknown value \"" + p_value + "\".");
	    } else if (p_name.equals(INFO_PARAM)) {
		want_info = true;
		info_types.add(p_value);
	    }
	}

	// Make sure there is no repeated information
	if (want_ancestors)
	    want_parent = false;
	if (want_descendants)
	    want_children = false;
	
	
	
	Element doc_list = this.doc.createElement(node_name+GSXML.LIST_MODIFIER);
	result.appendChild(doc_list);
	
	// Get the documents
	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
							    GSXML.NODE_ID_ATT);
	for (int i = 0; i < doc_ids.length; i++) {
	    String doc_id = doc_ids[i];
	    
	    if (OID.needsTranslating(doc_id)) {
		doc_id = this.gdbm_src.translateOID(doc_id);
	    }

	    // Add the document to the list
	    Element doc = this.doc.createElement(node_name);
	    doc_list.appendChild(doc);
	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);

	    
	    if (want_info) {

		Element info_elem = this.doc.createElement("nodeStructureInfo");
		doc.appendChild(info_elem);
		
		for (int j=0; j<info_types.size(); j++) {
		    String info_type = (String)info_types.get(j);
		    Element inf = getInfo(doc_id, info_type);
		    if (inf != null) {
			info_elem.appendChild(inf);
		    }
		}
	    }
	    if (want_structure) {
		// all structure info goes into a nodeStructure elem
		Element structure_elem = this.doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
		doc.appendChild(structure_elem);
		
		if (want_entire_structure) {
		    String top_id = OID.getTop(doc_id);
		    Element top_node = createDocNode(top_id, true, false);
		    addDescendants(top_node, top_id, true);
		    structure_elem.appendChild(top_node);
		    continue; // with the next document, we dont need to do any more here
		}
		
		// Add the requested structure information
		Element current = createDocNode(doc_id, false, false);
		
		//Ancestors: continually add parent nodes until the root is reached
		Element top_node = current; // the top node so far
		if (want_ancestors) {
		    String current_id = doc_id;
		    while (true) {
			Element parent = getParent(current_id);
			if (parent == null)
			    break;
			
			parent.appendChild(top_node);
			current_id = parent.getAttribute(GSXML.NODE_ID_ATT);
			top_node = parent;
		    }
		}
		// Parent: get the parent of the selected node
		if (want_parent) {
		    Element parent = getParent(doc_id);
		    if (parent != null) {
			parent.appendChild(current);
			top_node = parent;
		    }
		}

		
		// now the top node is the root of the structure
		structure_elem.appendChild(top_node);
		
		//Siblings: get the other descendants of the selected node's parent
		if (want_siblings) {
		    Element parent = (Element)current.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
		    String parent_id = OID.getParent(doc_id);
		    
		    // add siblings, - returns a pointer to the new current node 
		    current = addSiblings(parent, parent_id, doc_id);
		}
		
	    // Children: get the descendants, but only one level deep
		if (want_children)
		    addDescendants(current, doc_id, false);
		// Descendants: recursively get every descendant of the selected node
		if (want_descendants)
		    addDescendants(current, doc_id, true);
	    } // if want structure
	} // for each doc
	return result;
    }


    protected Element processDocumentMetadataRetrieve(Element request) {
	return genericMetadataRetrieve(request, DOCUMENT);
    }

     protected Element processClassifierBrowseMetadataRetrieve(Element request) {
	return genericMetadataRetrieve(request, CLASSIFIER);
    }
   
    
    /** Retrieve metadata associated with a document or classifier node*/
    protected Element genericMetadataRetrieve(Element request, int type)
    {
	// Create a new (empty) result message
	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);

	String node_name;
	
	String service_name;
	if (type==DOCUMENT) {
	    service_name = DOCUMENT_METADATA_RETRIEVE_SERVICE; 
	    node_name = GSXML.DOC_NODE_ELEM;
	} else {
	    service_name = CLASSIFIER_METADATA_SERVICE; 
	    node_name = GSXML.CLASS_NODE_ELEM;
	}
	result.setAttribute(GSXML.FROM_ATT, service_name);
	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);

	// Get the parameters of the request
	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
	if (param_list == null) {
	    System.err.println("GS2Retrieve, DocumentMetadataRetrieve Error: missing paramList.\n");
	    return result;  // Return the empty result
	}

	// The metadata information required
	Vector metadata_list = new Vector();
	boolean all_metadata = false;
	// Process the request parameters
	Element param = (Element) param_list.getFirstChild();
	while (param != null) {
	    // Identify the metadata information desired
	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
		String metadata = GSXML.getValue(param);
		if (metadata.equals("all")) {
		    all_metadata = true;
		    break;
		}
		metadata_list.add(metadata);
	    }
	    param = (Element) param.getNextSibling();
	}
	
	Element node_list = this.doc.createElement(node_name+GSXML.LIST_MODIFIER);
	result.appendChild(node_list);
	
	// Get the documents
	Element request_node_list = (Element) GSXML.getChildByTagName(request, node_name+GSXML.LIST_MODIFIER);
	if (request_node_list == null) {
	    System.err.println("Error: DocumentMetadataRetrieve request had no "+node_name+"List.\n");
 	    return result;
 	}
	
	NodeList request_nodes = request_node_list.getChildNodes();
	for (int i = 0; i < request_nodes.getLength(); i++) {
	    Element request_node = (Element) request_nodes.item(i);
	    String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);

	    if (OID.needsTranslating(node_id)) {
		node_id = this.gdbm_src.translateOID(node_id);
	    }

	    // Add the document to the list
	    Element new_node = (Element)this.doc.importNode(request_node, false);
	    node_list.appendChild(new_node);

	    // Add the requested metadata information
	    Element node_meta_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
	    new_node.appendChild(node_meta_list);
	    DBInfo info = this.gdbm_src.getInfo(node_id);
	    if (info == null) {// I have had a case where it is null!
		continue;
	    }
	    if (all_metadata) {
		// return everything out of the database
		Set keys = info.getKeys();
		Iterator it = keys.iterator();
		while(it.hasNext()) {
		    String key = (String)it.next();
		    String value = info.getInfo(key);
		    GSXML.addMetadata(this.doc, node_meta_list, key, replaceMacros(value));
		}
	    } else { // just get the selected ones
		
		for (int m = 0; m < metadata_list.size(); m++) {
		    String metadata = (String) metadata_list.get(m);
		    String value = getMetadata(node_id, info, metadata);
		    if (!value.equals("")) {
			GSXML.addMetadata(this.doc, node_meta_list, metadata, value);
		    }
		}
	    }
	}
	
	return result;
    }

    protected String getMetadata(String node_id, DBInfo info, 
				 String metadata) {
	boolean multiple = false;
	String relation = "";
	String separator = ", ";
	int pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
	if (pos ==-1) {
	    // just a plain meta entry eg dc.Title
	    return replaceMacros((String)info.getInfo(metadata));
	}
	
	String temp = metadata.substring(0, pos);
	metadata = metadata.substring(pos+1);
	// check for all on the front
	if (temp.equals("all")) {
	    multiple=true;	    
	    pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
	    if (pos ==-1) {
		temp = "";
	    } else {
		temp = metadata.substring(0, pos);
		metadata = metadata.substring(pos+1);
	    }
	}
	
	// now check for relational info
	if (temp.equals("parent") || temp.equals("root") || temp.equals( "ancestors")) { // "current" "siblings" "children" "descendents"
	    relation = temp;
	    pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
	    if (pos == -1) {
		temp = "";
	    } else {
		temp = metadata.substring(0, pos);
		metadata = metadata.substring(pos+1);
	    }
	}
	
	// now look for separator info
	if (temp.startsWith(GSConstants.META_SEPARATOR_SEP) && temp.endsWith(GSConstants.META_SEPARATOR_SEP)) {
	    separator = temp.substring(1, temp.length()-1);
	    
	}
	
	String relation_id = node_id;
	if (relation.equals("parent") || relation.equals("ancestors")) {
	    relation_id = OID.getParent(node_id);
	    // parent or ancestor does not include self
	    if (relation_id.equals(node_id)){
		return "";
	    }
	} else if (relation.equals("root")) {
	    relation_id = OID.getTop(node_id);
	}
	
	// now we either have a single node, or we have ancestors	
	DBInfo relation_info;
	if (relation_id.equals(node_id)) {
	    relation_info = info;
	} else {
	    relation_info = this.gdbm_src.getInfo(relation_id);
	}
	if (relation_info == null) {
	    return "";
	}

	StringBuffer result = new StringBuffer();

	if (!multiple) {
	    result.append(replaceMacros(relation_info.getInfo(metadata)));
	} else {
	    // we have multiple meta
	    Vector values = relation_info.getMultiInfo(metadata);
	    if (values != null) {
		boolean first = true;
		for (int i=0; i<values.size(); i++) {
		    if (first) {
			first = false;
		    } else {
			result.append(separator);
		    }
		    result.append(replaceMacros((String)values.elementAt(i)));
		}
	    }
	}
	// if not ancestors, then this is all we do
	if (!relation.equals("ancestors")) {
	    return result.toString();
	}
	
	// now do the ancestors 
	String current_id = relation_id;
	relation_id = OID.getParent(current_id);
	while (!relation_id.equals(current_id)) {
	    relation_info = this.gdbm_src.getInfo(relation_id);
	    if (relation_info == null) return result.toString();
	    if (!multiple) {
		result.insert(0, separator);
		result.insert(0, replaceMacros(relation_info.getInfo(metadata)));
	    } else {
		Vector values = relation_info.getMultiInfo(metadata);
		if (values != null) {
		    for (int i=values.size()-1; i>=0; i--) {
			result.insert(0, separator);
			result.insert(0, replaceMacros((String)values.elementAt(i)));
		    }
		}
		
	    }
	    current_id = relation_id;
	    relation_id = OID.getParent(current_id);
	} 
	return result.toString();
    }
    

    protected String replaceMacros(String text) {
	if (this.replacements == null) {
	    return text;
	}
	for (int i=0; i<this.replacements.size()-1; i+=2) {
	    String from  = (String)this.replacements.get(i);
	    String to = (String)this.replacements.get(i+1);
	    text = text.replaceAll(from, to);
	}
	return text;
    }
    /** Retrieve the content of a document - implemented by concrete subclasses */
    protected abstract Element processDocumentContentRetrieve(Element request);

    /** needs to get info from gdbm database - if the calling code gets it already it may pay to pass it in instead */
    protected String resolveImages(String doc_content, String doc_id)
    {
	String top_doc_id = OID.getTop(doc_id);
	DBInfo info = this.gdbm_src.getInfo(top_doc_id);
	String archivedir = info.getInfo("archivedir");
	String image_dir  = this.site_http_address + "/collect/"+this.cluster_name+"/index/assoc/"+archivedir;

	// Resolve all "_httpdocimg_"s
	doc_content = doc_content.replaceAll("_httpdocimg_", image_dir);
	return doc_content;
    }

    protected Element getInfo(String doc_id, String info_type) {

	String value="";
	if (info_type.equals(INFO_NUM_SIBS)) {
	    String parent_id = OID.getParent(doc_id);
	    if (parent_id.equals(doc_id)) {
		value="0";
	    } else {
		value = String.valueOf(getNumChildren(parent_id));
	    }
	} else if (info_type.equals(INFO_NUM_CHILDREN)) {
	    value = String.valueOf(getNumChildren(doc_id));
	} else if (info_type.equals(INFO_SIB_POS)) {
	    String parent_id = OID.getParent(doc_id);
	    if (parent_id.equals(doc_id)) {
		value="-1";
	    } else {
		DBInfo info = this.gdbm_src.getInfo(parent_id);
		String contains = info.getInfo("contains");
		contains = contains.replaceAll("\"", parent_id);
		String [] children = contains.split(";");
		for (int i=0;i<children.length;i++) {
		    String child_id = children[i];
		    if (child_id.equals(doc_id)) {
			value = String.valueOf(i+1); // make it from 1 to length
			break;
		    }
		}
	    }
	} else {
	    return null;
	}
	Element info_elem = this.doc.createElement("info");
	info_elem.setAttribute(GSXML.NAME_ATT, info_type);
	info_elem.setAttribute(GSXML.VALUE_ATT, value);
	return info_elem;
    }

    protected int getNumChildren(String doc_id) {
	DBInfo info = this.gdbm_src.getInfo(doc_id);
	String contains = info.getInfo("contains");
	if (contains.equals("")) {
	    return 0;
	}
	String [] children = contains.split(";");
	return children.length;
    }
    
}