Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractBrowse.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractBrowse.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractBrowse.java	(revision 8959)
@@ -0,0 +1,733 @@
+/*
+ *    AbstractBrowse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gdbm.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.MacroResolver;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+
+/** Partially implements a generic classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractBrowse
+    extends ServiceRack {
+
+    // the browsing services
+    private static final String CLASSIFIER_SERVICE = "ClassifierBrowse";
+    private static final String CLASSIFIER_METADATA_SERVICE = "ClassifierBrowseMetadataRetrieve";
+
+    // do we want to keep info request?
+    
+    protected static final String STRUCT_PARAM = "structure";
+    protected static final String INFO_PARAM = "info";
+    
+    protected static final String STRUCT_ANCESTORS = "ancestors";
+    protected static final String STRUCT_PARENT = "parent";
+    protected static final String STRUCT_SIBS = "siblings";
+    protected static final String STRUCT_CHILDREN = "children";
+    protected static final String STRUCT_DESCENDS = "descendants";
+    protected static final String STRUCT_ENTIRE = "entire";
+
+    protected static final String INFO_NUM_SIBS = "numSiblings";
+    protected static final String INFO_NUM_CHILDREN = "numChildren";
+    protected static final String INFO_SIB_POS = "siblingPosition";
+
+    protected Element config_info = null; // the xml from the config file
+
+    protected MacroResolver macro_resolver = null;
+
+    protected String default_document_type = null;
+    /** constructor */
+    protected AbstractBrowse()
+    {
+	this.dictionary_name = CLASSIFIER_SERVICE;
+	//	this.gdbm_src = new GDBMWrapper();
+	//	this.macro_resolver = new GS2MacroResolver(gdbm_src);
+    }
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	System.out.println("Configuring AbstractBrowse...");
+	this.config_info = info;
+
+	if (macro_resolver != null) {
+	    // set up the macro resolver
+	    Element replacement_elem = (Element)GSXML.getChildByTagName(extra_info, "replaceList");
+	    if (replacement_elem != null) {
+		macro_resolver.addMacros(replacement_elem);
+	    }
+
+	}
+
+ 
+        // check that there are classifiers specified
+        Element class_list = (Element)GSXML.getChildByTagName(info, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+        if (class_list == null) {
+            // no classifiers specified
+            return false;
+        }
+	
+	// get the display and format elements from the coll config file for
+	// the classifiers
+	extractExtraClassifierInfo(info, extra_info);
+	
+        // short_service_info_ - the browse one
+        Element cb_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+        cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
+        cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
+        this.short_service_info.appendChild(cb_service);
+	
+	// metadata retrieval for the browsing  
+        Element cbmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+        cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+        cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
+        this.short_service_info.appendChild(cbmr_service);
+
+	// the format info
+	Element cb_format_info = this.doc.createElement(GSXML.FORMAT_ELEM);
+	boolean format_found = false;
+
+	// try the default format first
+	Element def_format = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+	if (def_format != null) {
+	    cb_format_info.appendChild(GSXML.duplicateWithNewName(this.doc, def_format, GSXML.DEFAULT_ELEM, true));
+	    format_found = true;
+	}
+	
+	// add in to the description a simplified list of classifiers
+        NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	for(int i=0; i<classifiers.getLength(); i++) {
+	    Element cl = (Element)classifiers.item(i);
+	    Element new_cl = (Element)this.doc.importNode(cl, false); // just import this node, not the children
+	    
+	    // get the format info out, and put inside a classifier element
+	    Element format_cl = (Element)new_cl.cloneNode(false);
+	    Element format = (Element)GSXML.getChildByTagName(cl, GSXML.FORMAT_ELEM);
+	    if (format != null) {
+		
+		//copy all the children
+		NodeList elems = format.getChildNodes();
+		for (int j=0; j<elems.getLength();j++) {
+		    format_cl.appendChild(this.doc.importNode(elems.item(j), true));
+		}
+		cb_format_info.appendChild(format_cl);
+		format_found = true;
+	    }
+	    	    
+	}
+	    
+	if (format_found) {
+	    this.format_info_map.put(CLASSIFIER_SERVICE, cb_format_info);
+	}
+
+	// look for document display format - is there a default doc type??
+	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    // check for docType option.
+	    Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+	    if (doc_type_opt != null) {
+		String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+		if (!value.equals("")) {
+		    this.default_document_type = value;
+		}
+	    }
+	}
+
+	return true;
+    }
+
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+
+	if (service_id.equals(CLASSIFIER_SERVICE)) {
+	    
+	    Element class_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+	    if (class_list == null) {
+            // no classifiers specified
+		return null;
+	    }
+	    
+	    Element cb_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
+	    cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
+	    cb_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  getTextString(CLASSIFIER_SERVICE+".name", lang)));
+	    cb_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  getTextString(CLASSIFIER_SERVICE+".description", lang)));
+	    
+	    Element cl_list = this.doc.createElement(GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+	    cb_service.appendChild(cl_list);
+	    NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	    for(int i=0; i<classifiers.getLength(); i++) {
+		Element cl = (Element)classifiers.item(i);
+		Element new_cl = (Element)this.doc.importNode(cl, false); // just import this node, not the children
+		String content = cl.getAttribute(GSXML.CLASSIFIER_CONTENT_ATT);
+		cl_list.appendChild(new_cl);
+		String text = GSXML.getDisplayText(cl, 
+						   GSXML.DISPLAY_TEXT_NAME, 
+						   lang, "en");
+		if (text == null || text.equals("")) {
+		    // no display element was specified, use the metadata name
+		    // for now this looks in the class properties file
+		    // this needs to use a general metadata thing instead
+		    text = getMetadataNameText(content+".buttonname", lang);
+		}
+		if (text == null) {
+		    text = content;
+		}
+		
+		Element cl_name = GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, text);
+		new_cl.appendChild(cl_name);
+
+		// description
+		
+		String meta_name = getMetadataNameText(content, lang);
+		if (meta_name==null) {
+		    meta_name = content;
+		}
+		String [] array = {meta_name};
+		String description = getTextString("ClassifierBrowse.classifier_help", lang, array);
+		Element cl_desc = GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, description);
+		new_cl.appendChild(cl_desc);
+		
+	    }
+	    return cb_service;
+	}
+	
+	// these ones are probably never called, but put them here just in case
+	
+	if (service_id.equals(CLASSIFIER_METADATA_SERVICE)) {
+	    
+	    Element cbmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
+	    return cbmr_service;
+	}
+
+	return null;
+    }
+
+    /** this looks for any classifier specific display or format info from extra_info and adds it in to the correct place in info */
+    protected boolean extractExtraClassifierInfo(Element info, Element extra_info) {
+	
+	if (extra_info == null) {
+	    return false;
+	}
+	
+	Document owner = info.getOwnerDocument();
+	// so far we have display and format elements that we need for classifiers
+	NodeList classifiers = info.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	Element config_browse = (Element)GSXML.getChildByTagName(extra_info, GSXML.BROWSE_ELEM);
+
+	for (int i=0; i<classifiers.getLength();i++) {
+	    Element cl = (Element)classifiers.item(i);
+	    String name = cl.getAttribute(GSXML.NAME_ATT);
+	    Element node_extra = GSXML.getNamedElement(config_browse,
+						       GSXML.CLASSIFIER_ELEM,
+						       GSXML.NAME_ATT,
+						       name);
+	    if (node_extra == null) {
+		System.err.println("GS2REtrieve: haven't found extra info for classifier named "+name);
+		continue;
+	    }
+	    
+	    // get the display elements if any - displayName
+	    NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+	    if (display_names !=null) {
+		Element display = owner.createElement(GSXML.DISPLAY_ELEM);
+		for (int j=0; j<display_names.getLength(); j++) {
+		    Element e = (Element)display_names.item(j);
+		    cl.appendChild(owner.importNode(e, true));
+		    
+		}
+	    }
+	    
+	    // get the format element if any 
+	    Element format = (Element)GSXML.getChildByTagName(node_extra, GSXML.FORMAT_ELEM);
+	    if (format==null) { // try a generic one that applies to all classifiers
+		format = (Element)GSXML.getChildByTagName(extra_info,
+							  GSXML.FORMAT_ELEM);
+	    }
+	    if (format!=null) { // append to index info
+		cl.appendChild(owner.importNode(format, true));
+	    }
+	} // for each classifier
+
+	// now check for default format info
+	Element default_format = (Element)GSXML.getChildByTagName(config_browse, GSXML.FORMAT_ELEM);
+	if (default_format!=null) { // append to  info
+	    info.appendChild(owner.importNode(default_format, true));
+	}
+
+	return true;
+    }
+	    
+    protected Element processClassifierBrowse(Element request) {
+	
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, CLASSIFIER_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	Element query_node_list = (Element) GSXML.getChildByTagName(request, GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_node_list == null) {
+	    System.err.println("AbstractBrowse Error: ClassifierBrowse request specified no doc nodes.\n");
+	    return result;
+	}
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("AbstractBrowse Error: ClassifierBrowse request had no paramList.");
+	    return result;  // Return the empty result
+	}
+	
+	// the type of info required
+	boolean want_structure = false;
+	boolean want_info = false;
+
+	ArrayList info_types=new ArrayList();
+	// The document structure information desired
+	boolean want_ancestors = false;
+	boolean want_parent = false;
+	boolean want_siblings = false;
+	boolean want_children = false;
+	boolean want_descendants = false;
+
+	boolean want_entire_structure = false;
+	// Process the request parameters
+	NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	for (int i=0; i<params.getLength();i++) {
+	    
+	    Element param = (Element)params.item(i);
+	    String p_name = param.getAttribute(GSXML.NAME_ATT);
+	    String p_value = GSXML.getValue(param);
+	    // Identify the structure information desired
+	    if (p_name.equals(STRUCT_PARAM)) {
+		want_structure = true;
+		
+		// This is NOT locale sensitive
+		if (p_value.equals(STRUCT_ANCESTORS))
+		    want_ancestors = true;
+		else if (p_value.equals(STRUCT_PARENT))
+		    want_parent = true;
+		else if (p_value.equals(STRUCT_SIBS))
+		    want_siblings = true;
+		else if (p_value.equals(STRUCT_CHILDREN))
+		    want_children = true;
+		else if (p_value.equals(STRUCT_DESCENDS))
+		    want_descendants = true;
+		else if (p_value.equals(STRUCT_ENTIRE))
+		    want_entire_structure = true;
+		else
+		    System.err.println("AbstractDocumentRetrieve Warning: Unknown value \"" + p_value + "\".");
+	    } else if (p_name.equals(INFO_PARAM)) {
+		want_info = true;
+		info_types.add(p_value);
+	    }
+	}
+
+	// Make sure there is no repeated information
+	if (want_ancestors)
+	    want_parent = false;
+	if (want_descendants)
+	    want_children = false;
+	
+	// the node list to hold the results
+	Element node_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	
+	// Get the nodes
+	String[] node_ids = GSXML.getAttributeValuesFromList(query_node_list, 
+							     GSXML.NODE_ID_ATT);
+	for (int i = 0; i < node_ids.length; i++) {
+	    // Add the document to the list
+	    Element node = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	    node_list.appendChild(node);
+
+	    String node_id = node_ids[i];
+	    node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	    
+	    if (idNeedsTranslating(node_id)) {
+		node_id = translateId(node_id);
+		if (node_id == null) {
+		    continue;
+		}
+		node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	    }
+	    
+	    if (want_info) {
+
+		Element node_info_elem = this.doc.createElement("nodeStructureInfo");
+		node.appendChild(node_info_elem);
+		
+		for (int j=0; j<info_types.size(); j++) {
+		    String info_type = (String)info_types.get(j);
+		    String info_value = getStructureInfo(node_id, info_type);
+		    if (info_value != null) {
+			Element info_elem = this.doc.createElement("info");
+			info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+			info_elem.setAttribute(GSXML.VALUE_ATT, info_value);
+			node_info_elem.appendChild(info_elem);
+		    }
+		}
+	    }
+	    
+	    if (want_structure) {
+		// all structure info goes into a nodeStructure elem
+		Element structure_elem = this.doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+		node.appendChild(structure_elem);
+		
+		if (want_entire_structure) {
+		    String root_id = getRootId(node_id);
+		    Element root_node = createClassifierNode(root_id); //, true, false);
+		    addDescendants(root_node, root_id, true);
+		    structure_elem.appendChild(root_node);
+		    continue; // with the next document, we dont need to do any more here
+		}
+		
+		// Add the requested structure information
+		Element base_node = createClassifierNode(node_id); //, false, false);
+		
+		//Ancestors: continually add parent nodes until the root is reached
+		Element top_node = base_node; // the top node so far
+		if (want_ancestors) {
+		    String current_id = node_id;
+		    while (true) {
+			String parent_id = getParentId(current_id);
+			//Element parent = getParent(current_id);
+			if (parent_id == null)
+			    break; // no parent
+			Element parent_node = createClassifierNode(parent_id);
+			parent_node.appendChild(top_node);
+			current_id = parent_id;//.getAttribute(GSXML.NODE_ID_ATT);
+			top_node = parent_node;
+		    }
+		}
+		// Parent: get the parent of the selected node
+		else if (want_parent) {
+		    String parent_id = getParentId(node_id);
+		    if (parent_id != null) {
+			Element parent_node = createClassifierNode(parent_id);
+			parent_node.appendChild(base_node);
+			top_node = parent_node;
+		    }
+		}
+		
+		// now the top node is the root of the structure
+		structure_elem.appendChild(top_node);
+		
+		//Siblings: get the other descendants of the selected node's parent
+		if (want_siblings) {
+		    
+		    Element parent_node = (Element)base_node.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
+		    String parent_id = getParentId(node_id);
+		    // add siblings, - returns a pointer to the new current node 
+		    base_node = addSiblings(parent_node, parent_id, node_id);
+		}
+		
+		// Children: get the descendants, but only one level deep
+		if (want_children) {
+		    addDescendants(base_node, node_id, false);
+		} 
+		// Descendants: recursively get every descendant 
+		else if (want_descendants) {
+		    addDescendants(base_node, node_id, true);
+		}
+	    } // if want structure
+	} // for each doc
+	return result;
+    }
+
+
+    protected Element processClassifierBrowseMetadataRetrieve(Element request) {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	result.setAttribute(GSXML.FROM_ATT, CLASSIFIER_METADATA_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("AbstractBrowse, ClassifierBrowseMetadataRetrieve Error: missing paramList.\n");
+	    return result;  // Return the empty result
+	}
+
+	// The metadata information required
+	ArrayList metadata_names_list = new ArrayList();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		metadata_names_list.add(metadata);
+	    } 
+	    param = (Element) param.getNextSibling();
+	}
+	
+	Element node_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	
+	// Get the nodes
+	Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (request_node_list == null) {
+	    System.err.println("AbstractBrowse Error: ClassifierBrowseMetadataRetrieve request had no "+GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+ 	    return result;
+ 	}
+	
+	NodeList request_nodes = request_node_list.getChildNodes();
+	for (int i = 0; i < request_nodes.getLength(); i++) {
+	    Element request_node = (Element) request_nodes.item(i);
+	    String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+	    
+	    // Add the document to the results list
+	    Element new_node = (Element)this.doc.importNode(request_node, false);
+	    node_list.appendChild(new_node);
+
+	    if (idNeedsTranslating(node_id)) {
+		node_id = translateId(node_id);
+	    }
+	    if (node_id == null) {
+		continue;
+	    }
+	    
+	    Element metadata_list = getMetadataList(node_id, all_metadata, metadata_names_list);
+	    new_node.appendChild(metadata_list);
+	}
+	
+	return result;
+    }
+
+
+    /** Creates a classifier node */
+    protected Element createClassifierNode(String node_id)
+    {
+	Element node = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	return node;
+    }
+
+    /** create an element to go into the structure. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+     */
+    protected Element createDocNode(String node_id) {
+	Element node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+
+	String doc_type = null;
+	if (default_document_type != null) {
+	    doc_type = default_document_type;
+	} else {
+	    doc_type = getDocType(node_id);
+	}
+	node.setAttribute(GSXML.DOC_TYPE_ATT, doc_type);
+	String node_type = getNodeType(node_id, doc_type);	
+	node.setAttribute(GSXML.NODE_TYPE_ATT, node_type);
+	return node;
+    }
+
+   /** returns the node type of the specified node.
+	should be one of 
+	GSXML.NODE_TYPE_LEAF, 
+	GSXML.NODE_TYPE_INTERNAL, 
+	GSXML.NODE_TYPE_ROOT
+    */
+    protected String getNodeType(String node_id, String doc_type) {
+	if (doc_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+
+	if (getParentId(node_id)==null) {
+	    return GSXML.NODE_TYPE_ROOT;
+	}
+	if (doc_type.equals(GSXML.DOC_TYPE_PAGED)) {
+	     return GSXML.NODE_TYPE_LEAF;
+	}
+	if (getChildrenIds(node_id)==null) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+	return GSXML.NODE_TYPE_INTERNAL;	
+	
+    }
+
+    /** adds all the children of doc_id the the doc element,
+     * and if recursive=true, adds all their children as well*/
+    protected void addDescendants(Element node, String node_id, 
+				  boolean recursive)
+    {
+	ArrayList child_ids = getChildrenIds(node_id);
+	if (child_ids==null) return;
+	for (int i=0; i< child_ids.size(); i++) {
+	    String child_id = (String)child_ids.get(i);
+	    Element child_elem;
+	    if (isDocumentId(child_id)) {
+		child_elem = createDocNode(child_id);
+	    } else {
+		child_elem = createClassifierNode(child_id);
+	    }
+	    node.appendChild(child_elem);
+	    if (recursive) {
+		addDescendants(child_elem, child_id, recursive);
+	    }
+	}
+    }
+
+    /** adds all the siblings of current_id to the parent element. 
+      returns the new current element*/
+    protected Element addSiblings(Element parent_node, String parent_id, 
+				  String current_id) {
+	Element current_node = (Element)parent_node.getFirstChild();
+	if (current_node == null) {
+	    // create a sensible error message
+	    System.err.println("AbstractBrowse Error: there should be a first child.");
+	    return null;
+	}
+	// remove the current child,- will add it in later in its correct place
+	parent_node.removeChild(current_node);
+
+	// add in all the siblings,
+	addDescendants(parent_node, parent_id, false);
+
+	// find the node that is now the current node
+	// this assumes that the new node that was created is the same as 
+	// the old one that was removed - we may want to replace the new one 
+	// with the old one.
+	Element new_current = GSXML.getNamedElement(parent_node, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
+	return new_current;
+    }
+
+    /** returns true if oid ends in
+	.fc (firstchild), 
+	.lc (lastchild), 
+	.pr (parent), 
+	.ns (next sibling), 
+	.ps (prev sibling), 
+	.rt (root) 
+	.ss (specified sibling), 
+	false otherwise 
+    */
+    protected boolean idNeedsTranslating(String node_id) {
+	String tail = node_id.substring(node_id.length()-3);
+	return (tail.equals(".fc") || tail.equals(".lc") || 
+		tail.equals(".pr") || tail.equals(".ns") || 
+		tail.equals(".ps") || tail.equals(".rt") || 
+		tail.equals(".ss"));
+
+    }
+
+    /** returns the list of sibling ids, including the specified node_id */
+    protected ArrayList getSiblingIds(String node_id) {
+	String parent_id = getParentId(node_id);
+	if (parent_id == null) {
+	    return null;
+	}
+	return getChildrenIds(parent_id);
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    abstract protected String translateId(String node_id);
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    abstract protected String getDocType(String node_id);
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    abstract protected String getRootId(String node_id);
+    /** returns a list of the child ids in order, null if no children */
+    abstract protected ArrayList getChildrenIds(String node_id);
+    /** returns the node id of the parent node, null if no parent */
+    abstract protected String getParentId(String node_id);
+ 
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    abstract protected boolean isDocumentId(String node_id);
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    abstract protected Element getMetadataList(String node_id,
+					       boolean all_metadata,
+					       ArrayList metadata_names);
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    abstract protected String getStructureInfo(String node_id, String info_type);    
+//      /** needs to get info from gdbm database - if the calling code gets it already it may pay to pass it in instead */
+//     protected String resolveTextMacros(String doc_content, String doc_id, String lang)
+//     {
+// 	DBInfo info = null;
+// 	if (doc_content.indexOf("_httpdocimg_")!=-1) {
+// 	    String top_doc_id = OID.getTop(doc_id);
+// 	    info = this.gdbm_src.getInfo(top_doc_id);
+// 	    if (info == null) {
+// 		// perhaps we had per.iods in the ids - just try the current id
+// 		top_doc_id = doc_id;
+// 		info = this.gdbm_src.getInfo(top_doc_id);
+// 	    }
+// 	    if (info != null) {
+// 		String archivedir = info.getInfo("archivedir");
+// 		String image_dir  = this.site_http_address + "/collect/"+this.cluster_name+"/index/assoc/"+archivedir;
+		
+// 		// Resolve all "_httpdocimg_"s
+// 		doc_content = doc_content.replaceAll("_httpdocimg_", image_dir);
+// 	    }
+// 	}
+// 	// resolve any collection specific macros
+// 	doc_content = macro_resolver.resolve(doc_content, lang, GS2MacroResolver.SCOPE_TEXT, doc_id, info);
+// 	return doc_content;
+//     }
+
+   
+}   
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java	(revision 8959)
@@ -0,0 +1,610 @@
+/*
+ *    AbstractDocumentRetrieve.java
+ *    a base class for retrieval services
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+//import org.greenstone.gdbm.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.MacroResolver;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Abstract class for Document Retrieval Services
+ *
+ * @author <a href="mailto:greenstone@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public abstract class AbstractDocumentRetrieve
+    extends ServiceRack {
+
+    // the services on offer
+    protected static final String DOCUMENT_STRUCTURE_RETRIEVE_SERVICE = "DocumentStructureRetrieve";
+    protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
+    protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";
+    
+    protected static final String STRUCT_PARAM = "structure";
+    protected static final String INFO_PARAM = "info";
+    
+    protected static final String STRUCT_ANCESTORS = "ancestors";
+    protected static final String STRUCT_PARENT = "parent";
+    protected static final String STRUCT_SIBS = "siblings";
+    protected static final String STRUCT_CHILDREN = "children";
+    protected static final String STRUCT_DESCENDS = "descendants";
+    protected static final String STRUCT_ENTIRE = "entire";
+
+    protected static final String INFO_NUM_SIBS = "numSiblings";
+    protected static final String INFO_NUM_CHILDREN = "numChildren";
+    protected static final String INFO_SIB_POS = "siblingPosition";
+
+    // means the id is not a greenstone id and needs translating
+    protected static final String EXTID_PARAM = "ext"; 
+    
+    protected Element config_info = null; // the xml from the config file
+
+    protected String default_document_type = null;
+    protected MacroResolver macro_resolver = null;
+    
+    /** constructor */
+    public AbstractDocumentRetrieve()
+    {
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	System.out.println("Configuring AbstractDocumentRetrieve...");
+	this.config_info = info;
+
+	// set up short_service_info_ - for now just has name and type
+	Element dsr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+	this.short_service_info.appendChild(dsr_service);
+
+	Element dmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	this.short_service_info.appendChild(dmr_service);
+
+	Element dcr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	this.short_service_info.appendChild(dcr_service);
+
+	// look for document display format
+	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.doc.importNode(display_format, true));
+	    // should we keep a copy?
+	    // check for docType option.
+	    Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+	    if (doc_type_opt != null) {
+		String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+		if (!value.equals("")) {
+		    this.default_document_type = value;
+		}
+	    }
+	}
+
+	if (macro_resolver != null) {
+	    // set up the macro resolver
+	    Element replacement_elem = (Element)GSXML.getChildByTagName(extra_info, "replaceList");
+	    if (replacement_elem != null) {
+		macro_resolver.addMacros(replacement_elem);
+	    }
+	}
+
+	return true;
+    }
+    
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+	
+	// these ones are probably never called, but put them here just in case
+	Element service_elem = this.doc.createElement(GSXML.SERVICE_ELEM);
+	service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	service_elem.setAttribute(GSXML.NAME_ATT, service_id);
+	return service_elem;
+    }
+
+    protected Element processDocumentMetadataRetrieve(Element request) {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("AbstractDocumentRetrieve, DocumentMetadataRetrieve Error: missing paramList.\n");
+	    return result;  // Return the empty result
+	}
+
+	boolean external_id = false;
+	// The metadata information required
+	ArrayList metadata_names_list = new ArrayList();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		metadata_names_list.add(metadata);
+	    } else if (param.getAttribute(GSXML.NAME_ATT).equals(EXTID_PARAM)&& GSXML.getValue(param).equals("1")) {
+		external_id = true;
+	    }
+	    param = (Element) param.getNextSibling();
+	}
+	
+	Element node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	
+	// Get the documents
+	Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (request_node_list == null) {
+	    System.err.println("Error: DocumentMetadataRetrieve request had no "+GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+ 	    return result;
+ 	}
+	
+	NodeList request_nodes = request_node_list.getChildNodes();
+	for (int i = 0; i < request_nodes.getLength(); i++) {
+	    Element request_node = (Element) request_nodes.item(i);
+	    String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+	    
+	    // Add the document to the results list
+	    Element new_node = (Element)this.doc.importNode(request_node, false);
+	    node_list.appendChild(new_node);
+
+	    if (external_id) {
+		// can we have .pr etc extensions with external ids?
+		node_id = translateExternalId(node_id);
+	    } else if (idNeedsTranslating(node_id)) {
+		node_id = translateId(node_id);
+	    }
+	    
+	    if (node_id == null) {
+		continue;
+	    }
+	    
+	    Element metadata_list = getMetadataList(node_id, all_metadata, metadata_names_list);
+	    new_node.appendChild(metadata_list);
+	}
+	
+	return result;
+    }
+
+    protected Element processDocumentStructureRetrieve(Element request) {
+    
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_doc_list == null) {
+	    System.err.println("AbstractDocumentRetrieve Error: DocumentStructureRetrieve request specified no doc nodes.\n");
+	    return result;
+	}
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("AbstractDocumentRetrieve Error: DocumentStructureRetrieve request had no paramList.");
+	    return result;  // Return the empty result
+	}
+	Element extid_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, EXTID_PARAM);
+	boolean external_id = false;
+	if (extid_param != null && GSXML.getValue(extid_param).equals("1")) {
+	    external_id = true;
+	}
+
+	// the type of info required
+	boolean want_structure = false;
+	boolean want_info = false;
+
+	ArrayList info_types=new ArrayList();
+	// The document structure information desired
+	boolean want_ancestors = false;
+	boolean want_parent = false;
+	boolean want_siblings = false;
+	boolean want_children = false;
+	boolean want_descendants = false;
+
+	boolean want_entire_structure = false;
+	// Process the request parameters
+	NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	for (int i=0; i<params.getLength();i++) {
+	    
+	    Element param = (Element)params.item(i);
+	    String p_name = param.getAttribute(GSXML.NAME_ATT);
+	    String p_value = GSXML.getValue(param);
+	    // Identify the structure information desired
+	    if (p_name.equals(STRUCT_PARAM)) {
+		want_structure = true;
+		
+		// This is NOT locale sensitive
+		if (p_value.equals(STRUCT_ANCESTORS))
+		    want_ancestors = true;
+		else if (p_value.equals(STRUCT_PARENT))
+		    want_parent = true;
+		else if (p_value.equals(STRUCT_SIBS))
+		    want_siblings = true;
+		else if (p_value.equals(STRUCT_CHILDREN))
+		    want_children = true;
+		else if (p_value.equals(STRUCT_DESCENDS))
+		    want_descendants = true;
+		else if (p_value.equals(STRUCT_ENTIRE))
+		    want_entire_structure = true;
+		else
+		    System.err.println("AbstractDocumentRetrieve Warning: Unknown value \"" + p_value + "\".");
+	    } else if (p_name.equals(INFO_PARAM)) {
+		want_info = true;
+		info_types.add(p_value);
+	    }
+	}
+
+	// Make sure there is no repeated information
+	if (want_ancestors)
+	    want_parent = false;
+	if (want_descendants)
+	    want_children = false;
+	
+	// the document list to hold the results
+	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_list);
+	
+	// Get the documents
+	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
+							    GSXML.NODE_ID_ATT);
+	for (int i = 0; i < doc_ids.length; i++) {
+	    // Add the document to the list
+	    Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_list.appendChild(doc);
+	    
+	    String doc_id = doc_ids[i];
+	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    if (external_id) {
+		doc_id = translateExternalId(doc_id);
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    } else if (idNeedsTranslating(doc_id)) {
+		doc_id = translateId(doc_id);	
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    }
+	    if (doc_id == null) {
+		continue;
+	    }
+
+	    if (want_info) {
+
+		Element node_info_elem = this.doc.createElement("nodeStructureInfo");
+		doc.appendChild(node_info_elem);
+		
+		for (int j=0; j<info_types.size(); j++) {
+		    String info_type = (String)info_types.get(j);
+		    String info_value = getStructureInfo(doc_id, info_type);
+		    if (info_value != null) {
+			Element info_elem = this.doc.createElement("info");
+			info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+			info_elem.setAttribute(GSXML.VALUE_ATT, info_value);
+			node_info_elem.appendChild(info_elem);
+		    }
+		}
+	    }
+	    
+	    if (want_structure) {
+		// all structure info goes into a nodeStructure elem
+		Element structure_elem = this.doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+		doc.appendChild(structure_elem);
+		
+		if (want_entire_structure) {
+		    String root_id = getRootId(doc_id);
+		    Element root_node = createDocNode(root_id); //, true, false);
+		    addDescendants(root_node, root_id, true);
+		    structure_elem.appendChild(root_node);
+		    continue; // with the next document, we dont need to do any more here
+		}
+		
+		// Add the requested structure information
+		Element base_node = createDocNode(doc_id); //, false, false);
+		
+		//Ancestors: continually add parent nodes until the root is reached
+		Element top_node = base_node; // the top node so far
+		if (want_ancestors) {
+		    System.err.println("want ancestors");
+		    String current_id = doc_id;
+		    while (true) {
+			String parent_id = getParentId(current_id);
+			//Element parent = getParent(current_id);
+			if (parent_id == null)
+			    break; // no parent
+			Element parent_node = createDocNode(parent_id);
+			parent_node.appendChild(top_node);
+			current_id = parent_id;//.getAttribute(GSXML.NODE_ID_ATT);
+			top_node = parent_node;
+		    }
+		}
+		// Parent: get the parent of the selected node
+		else if (want_parent) {
+		    String parent_id = getParentId(doc_id);
+		    if (parent_id != null) {
+			Element parent_node = createDocNode(parent_id);
+			parent_node.appendChild(base_node);
+			top_node = parent_node;
+		    }
+		}
+		
+		// now the top node is the root of the structure
+		structure_elem.appendChild(top_node);
+		
+		//Siblings: get the other descendants of the selected node's parent
+		if (want_siblings) {
+		    
+		    Element parent_node = (Element)base_node.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
+		    String parent_id = getParentId(doc_id);
+		    // add siblings, - returns a pointer to the new current node 
+		    base_node = addSiblings(parent_node, parent_id, doc_id);
+		}
+		
+		// Children: get the descendants, but only one level deep
+		if (want_children) {
+		    System.err.println("want children");
+		    addDescendants(base_node, doc_id, false);
+		} 
+		// Descendants: recursively get every descendant 
+		else if (want_descendants) {
+		    System.err.println("want descends");
+		    addDescendants(base_node, doc_id, true);
+		}
+	    } // if want structure
+	} // for each doc
+	return result;
+    }
+
+    /** Retrieve the content of a document */
+    protected Element processDocumentContentRetrieve(Element request)
+    {
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request - no parameters at this stage
+  	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element extid_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, EXTID_PARAM);
+	boolean external_id = false;
+	if (extid_param != null && GSXML.getValue(extid_param).equals("1")) {
+	    external_id = true;
+	}
+	// Get the request content 
+	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_doc_list == null) {
+	    System.err.println("Error: DocumentContentRetrieve request specified no doc nodes.\n");
+	    return result;
+	}
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_list);
+	
+	// set up the retrieval??
+
+	// Get the documents
+	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
+							    GSXML.NODE_ID_ATT);
+	for (int i = 0; i < doc_ids.length; i++) {
+	    String doc_id = doc_ids[i];
+	    // Create the document node
+	    Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    doc_list.appendChild(doc);
+
+	    if (external_id) {
+		doc_id = translateExternalId(doc_id);
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    } else if (idNeedsTranslating(doc_id)) {
+		doc_id = translateId(doc_id);
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    }
+	    if (doc_id == null) {
+		continue;
+	    }
+	    
+	    Element node_content = getNodeContent(doc_id);
+	    doc.appendChild(node_content);
+	}
+	return result;
+    }
+    
+    /** create an element to go into the structure. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+     */
+    protected Element createDocNode(String node_id) {
+	Element node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+
+	String doc_type = null;
+	if (default_document_type != null) {
+	    doc_type = default_document_type;
+	} else {
+	    doc_type = getDocType(node_id);
+	}
+	node.setAttribute(GSXML.DOC_TYPE_ATT, doc_type);
+	String node_type = getNodeType(node_id, doc_type);	
+	node.setAttribute(GSXML.NODE_TYPE_ATT, node_type);
+	return node;
+    }
+
+    /** adds all the children of doc_id the the doc element,
+     * and if recursive=true, adds all their children as well*/
+    protected void addDescendants(Element doc, String doc_id, 
+				  boolean recursive)
+    {
+	ArrayList child_ids = getChildrenIds(doc_id);
+	if (child_ids==null) return;
+	for (int i=0; i< child_ids.size(); i++) {
+	    String child_id = (String)child_ids.get(i);
+	    Element child_elem = createDocNode(child_id);
+	    doc.appendChild(child_elem);
+	    if (recursive && !child_elem.getAttribute(GSXML.NODE_TYPE_ATT).equals(GSXML.NODE_TYPE_LEAF)) {
+		addDescendants(child_elem, child_id, recursive);
+	    }
+	}
+    }
+
+    /** adds all the siblings of current_id to the parent element. 
+      returns the new current element*/
+    protected Element addSiblings(Element parent_node, String parent_id, 
+				  String current_id) {
+	Element current_node = (Element)parent_node.getFirstChild();
+	if (current_node == null) {
+	    // create a sensible error message
+	    System.err.println("AbstractDocumentRetrieve Error: there should be a first child.");
+	    return null;
+	}
+	// remove the current child,- will add it in later in its correct place
+	parent_node.removeChild(current_node);
+
+	// add in all the siblings,
+	addDescendants(parent_node, parent_id, false);
+
+	// find the node that is now the current node
+	// this assumes that the new node that was created is the same as 
+	// the old one that was removed - we may want to replace the new one 
+	// with the old one.
+	Element new_current = GSXML.getNamedElement(parent_node, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
+	return new_current;
+    }
+
+    /** returns true if oid ends in
+	.fc (firstchild), 
+	.lc (lastchild), 
+	.pr (parent), 
+	.ns (next sibling), 
+	.ps (prev sibling), 
+	.rt (root) 
+	.ss (specified sibling), 
+	false otherwise 
+    */
+    protected boolean idNeedsTranslating(String id) {
+	String tail = id.substring(id.length()-3);
+	return (tail.equals(".fc") || tail.equals(".lc") || 
+		tail.equals(".pr") || tail.equals(".ns") || 
+		tail.equals(".ps") || tail.equals(".rt") || 
+		tail.equals(".ss"));
+
+    }
+
+    /** returns the list of sibling ids, including the specified node_id */
+    protected ArrayList getSiblingIds(String node_id) {
+	String parent_id = getParentId(node_id);
+	if (parent_id == null) {
+	    return null;
+	}
+	return getChildrenIds(parent_id);
+
+    }
+
+   /** returns the node type of the specified node.
+	should be one of 
+	GSXML.NODE_TYPE_LEAF, 
+	GSXML.NODE_TYPE_INTERNAL, 
+	GSXML.NODE_TYPE_ROOT
+    */
+    protected String getNodeType(String node_id, String doc_type) {
+	if (doc_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+
+	if (getParentId(node_id)==null) {
+	    return GSXML.NODE_TYPE_ROOT;
+	}
+	if (doc_type.equals(GSXML.DOC_TYPE_PAGED)) {
+	     return GSXML.NODE_TYPE_LEAF;
+	}
+	if (getChildrenIds(node_id)==null) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+	return GSXML.NODE_TYPE_INTERNAL;	
+	
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    abstract protected String translateId(String id);
+    
+    /** if an id is not a greenstone id (an external id) then translate 
+	it to a greenstone one*/
+    abstract protected String translateExternalId(String id);
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    abstract protected String getDocType(String node_id);
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    abstract protected String getRootId(String node_id);
+    /** returns a list of the child ids in order, null if no children */
+    abstract protected ArrayList getChildrenIds(String node_id);
+    /** returns the node id of the parent node, null if no parent */
+    abstract protected String getParentId(String node_id);
+
+    /** get the metadata for the doc node doc_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    abstract protected Element getMetadataList(String doc_id, 
+					       boolean all_metadata, 
+					       ArrayList metadata_names);
+    /** returns the content of a node
+     * shoudl return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id);
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    abstract protected String getStructureInfo(String doc_id, String info_type);
+    
+}   
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java	(revision 8959)
@@ -0,0 +1,453 @@
+/*
+ *    AbstractGS2DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gdbm.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Implements the generic retrieval and classifier services for GS2 
+ * collections.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractGS2DocumentRetrieve
+    extends AbstractDocumentRetrieve {
+
+ 
+    //    protected static final String EXTLINK_PARAM = "ext"; here or in base??
+    protected GDBMWrapper gdbm_src = null;
+
+
+    /** constructor */
+    protected AbstractGS2DocumentRetrieve()
+    {
+	this.gdbm_src = new GDBMWrapper();
+	this.macro_resolver = new GS2MacroResolver(gdbm_src);
+    }
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	
+	System.out.println("Configuring AbstractGS2DocumentRetrieve...");
+	//this.config_info = info;
+
+	// Open GDBM database for querying
+	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name);
+	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
+	    System.err.println("AbstractGS2DocumentRetrieve Error: Could not open GDBM database!");
+	    return false;
+	}
+
+	return super.configure(info, extra_info);
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	return this.gdbm_src.translateOID(node_id);
+    }
+    
+    /** if an id is not a greenstone id (an external id) then translate 
+	it to a greenstone one*/
+    protected String translateExternalId(String node_id){
+	return this.gdbm_src.externalId2OID(node_id);
+    }
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return OID.getTop(node_id);
+    }
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return null;
+	}
+	ArrayList children = new ArrayList();
+	StringTokenizer st = new StringTokenizer(contains, ";");
+	while (st.hasMoreTokens()) {
+	    String child_id = st.nextToken().replaceAll("\"", node_id);
+	    children.add(child_id);
+	}
+	return children;
+
+    }
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata,
+				      ArrayList metadata_names) {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+	String lang = "en"; // why do we need this??
+	if (all_metadata) {
+	    // return everything out of the database
+	    Set keys = info.getKeys();
+	    Iterator it = keys.iterator();
+	    while(it.hasNext()) {
+		String key = (String)it.next();
+		String value = info.getInfo(key);
+		GSXML.addMetadata(this.doc, metadata_list, key, this.macro_resolver.resolve(value, lang, MacroResolver.SCOPE_META, node_id));
+	    }
+	    
+	} else {
+	    for (int i=0; i<metadata_names.size(); i++) {
+		String meta_name = (String) metadata_names.get(i);
+		String value = getMetadata(node_id, info, meta_name, lang);
+		GSXML.addMetadata(this.doc, metadata_list, meta_name, value);
+	    }
+	}
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	String value="";
+	if (info_type.equals(INFO_NUM_SIBS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="0";
+	    } else {
+		value = String.valueOf(getNumChildren(parent_id));
+	    }
+	    return value;
+	}
+	
+	if (info_type.equals(INFO_NUM_CHILDREN)) {
+	    return String.valueOf(getNumChildren(doc_id));
+	}
+
+	
+	if (info_type.equals(INFO_SIB_POS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		return "-1";
+	    }
+	    
+	    DBInfo info = this.gdbm_src.getInfo(parent_id);
+	    if (info==null) {
+		return "-1";
+	    }
+	    
+	    String contains = info.getInfo("contains");
+	    contains = contains.replaceAll("\"", parent_id);
+	    String [] children = contains.split(";");
+	    for (int i=0;i<children.length;i++) {
+		String child_id = children[i];
+		if (child_id.equals(doc_id)) {
+		    return String.valueOf(i+1); // make it from 1 to length
+			   
+		}
+	    }
+	    
+	    return "-1";
+	} else {
+	    return null;
+	}
+
+    }
+
+    protected int getNumChildren(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return 0;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return 0;
+	}
+	String [] children = contains.split(";");
+	return children.length;
+    }
+    
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.gdbm_src.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+    }
+
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id);
+
+    protected String getMetadata(String node_id, DBInfo info, 
+				 String metadata, String lang) {
+	boolean multiple = false;
+	String relation = "";
+	String separator = ", ";
+	int pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
+	if (pos ==-1) {
+	    // just a plain meta entry eg dc.Title
+	    return macro_resolver.resolve((String)info.getInfo(metadata), lang, MacroResolver.SCOPE_META, node_id);
+	}
+	
+	String temp = metadata.substring(0, pos);
+	metadata = metadata.substring(pos+1);
+	// check for all on the front
+	if (temp.equals("all")) {
+	    multiple=true;	    
+	    pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
+	    if (pos ==-1) {
+		temp = "";
+	    } else {
+		temp = metadata.substring(0, pos);
+		metadata = metadata.substring(pos+1);
+	    }
+	}
+	
+	// now check for relational info
+	if (temp.equals("parent") || temp.equals("root") || temp.equals( "ancestors")) { // "current" "siblings" "children" "descendents"
+	    relation = temp;
+	    pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
+	    if (pos == -1) {
+		temp = "";
+	    } else {
+		temp = metadata.substring(0, pos);
+		metadata = metadata.substring(pos+1);
+	    }
+	}
+	
+	// now look for separator info
+	if (temp.startsWith(GSConstants.META_SEPARATOR_SEP) && temp.endsWith(GSConstants.META_SEPARATOR_SEP)) {
+	    separator = temp.substring(1, temp.length()-1);
+	    
+	}
+	
+	String relation_id = node_id;
+	if (relation.equals("parent") || relation.equals("ancestors")) {
+	    relation_id = OID.getParent(node_id);
+	    // parent or ancestor does not include self
+	    if (relation_id.equals(node_id)){
+		return "";
+	    }
+	} else if (relation.equals("root")) {
+	    relation_id = OID.getTop(node_id);
+	}
+	
+	// now we either have a single node, or we have ancestors	
+	DBInfo relation_info;
+	if (relation_id.equals(node_id)) {
+	    relation_info = info;
+	} else {
+	    relation_info = this.gdbm_src.getInfo(relation_id);
+	}
+	if (relation_info == null) {
+	    return "";
+	}
+
+	StringBuffer result = new StringBuffer();
+
+	if (!multiple) {
+	    result.append(this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, MacroResolver.SCOPE_META, relation_id));
+	} else {
+	    // we have multiple meta
+	    Vector values = relation_info.getMultiInfo(metadata);
+	    if (values != null) {
+		boolean first = true;
+		for (int i=0; i<values.size(); i++) {
+		    if (first) {
+			first = false;
+		    } else {
+			result.append(separator);
+		    }
+		    result.append(this.macro_resolver.resolve((String)values.elementAt(i), lang, MacroResolver.SCOPE_META, relation_id));
+		}
+	    }
+	}
+	// if not ancestors, then this is all we do
+	if (!relation.equals("ancestors")) {
+	    return result.toString();
+	}
+	
+	// now do the ancestors 
+	String current_id = relation_id;
+	relation_id = OID.getParent(current_id);
+	while (!relation_id.equals(current_id)) {
+	    relation_info = this.gdbm_src.getInfo(relation_id);
+	    if (relation_info == null) return result.toString();
+	    if (!multiple) {
+		result.insert(0, separator);
+		result.insert(0, this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, MacroResolver.SCOPE_META, relation_id));
+	    } else {
+		Vector values = relation_info.getMultiInfo(metadata);
+		if (values != null) {
+		    for (int i=values.size()-1; i>=0; i--) {
+			result.insert(0, separator);
+			result.insert(0, this.macro_resolver.resolve((String)values.elementAt(i), lang, MacroResolver.SCOPE_META, relation_id));
+		    }
+		}
+		
+	    }
+	    current_id = relation_id;
+	    relation_id = OID.getParent(current_id);
+	} 
+	return result.toString();
+    }
+    
+
+    /** needs to get info from gdbm database - if the calling code gets it already it may pay to pass it in instead */
+    protected String resolveTextMacros(String doc_content, String doc_id, String lang)
+    {
+	DBInfo info = null;
+	if (doc_content.indexOf("_httpdocimg_")!=-1) {
+	    String top_doc_id = OID.getTop(doc_id);
+	    info = this.gdbm_src.getInfo(top_doc_id);
+	    if (info == null) {
+		// perhaps we had per.iods in the ids - just try the current id
+		top_doc_id = doc_id;
+		info = this.gdbm_src.getInfo(top_doc_id);
+	    }
+	    if (info != null) {
+		String archivedir = info.getInfo("archivedir");
+		String image_dir  = this.site_http_address + "/collect/"+this.cluster_name+"/index/assoc/"+archivedir;
+		
+		// Resolve all "_httpdocimg_"s
+		doc_content = doc_content.replaceAll("_httpdocimg_", image_dir);
+	    }
+	}
+	// resolve any collection specific macros
+	doc_content = macro_resolver.resolve(doc_content, lang, MacroResolver.SCOPE_TEXT, doc_id);
+	return doc_content;
+    }
+
+    protected Element getInfo(String doc_id, String info_type) {
+
+	String value="";
+	if (info_type.equals(INFO_NUM_SIBS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="0";
+	    } else {
+		value = String.valueOf(getNumChildren(parent_id));
+	    }
+	} else if (info_type.equals(INFO_NUM_CHILDREN)) {
+	    value = String.valueOf(getNumChildren(doc_id));
+	} else if (info_type.equals(INFO_SIB_POS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="-1";
+	    } else {
+		DBInfo info = this.gdbm_src.getInfo(parent_id);
+		if (info==null) {
+		    value ="-1";
+		} else {
+		    String contains = info.getInfo("contains");
+		    contains = contains.replaceAll("\"", parent_id);
+		    String [] children = contains.split(";");
+		    for (int i=0;i<children.length;i++) {
+			String child_id = children[i];
+			if (child_id.equals(doc_id)) {
+			    value = String.valueOf(i+1); // make it from 1 to length
+			    break;
+			}
+		    }
+		}
+	    }
+	} else {
+	    return null;
+	}
+	Element info_elem = this.doc.createElement("info");
+	info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+	info_elem.setAttribute(GSXML.VALUE_ATT, value);
+	return info_elem;
+    }
+
+}   
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 8959)
@@ -0,0 +1,206 @@
+/*
+ *    AbstractGS3DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Implements the generic retrieval and classifier services for GS3 
+ * collections.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractGS3DocumentRetrieve
+    extends AbstractDocumentRetrieve {
+
+    protected SQLQuery database = null;
+
+
+    /** constructor */
+    protected AbstractGS3DocumentRetrieve()
+    {
+	this.database = new SQLQuery();
+	// set up a macro resolver
+    }
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	
+	System.out.println("Configuring AbstractGS3DocumentRetrieve...");
+
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return super.configure(info, extra_info);
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+    
+    /** if an id is not a greenstone id (an external id) then translate 
+	it to a greenstone one*/
+    protected String translateExternalId(String node_id){
+	// dont have this yet
+	return node_id;
+    }
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getTop(node_id);
+    }
+    
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	System.err.println("get children ids for "+node_id);
+	if (GS3OID.isDocTop(node_id)) { // hack
+	    node_id =  GS3OID.createOID(node_id, "1");
+	}
+	System.err.println("new node id="+node_id);
+	ArrayList documents = database.getDocumentChildren(node_id);
+	if (documents == null || documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+    
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata,
+				      ArrayList metadata_names) {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getDocumentMetadata(node_id, metadata);
+		if (value != null) {
+		    System.out.println("found value "+value);
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null; // not implemented yet
+    }
+
+    
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+    }
+
+    protected String resolveRelativeLinks(String doc_content, String doc_id) {
+
+	String http_path  = this.site_http_address + "/collect/"+this.cluster_name;
+	doc_content = doc_content.replaceAll("_httpcollection_", http_path);
+	return doc_content;
+	
+    }
+
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id);
+
+}   
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Browse.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Browse.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Browse.java	(revision 8959)
@@ -0,0 +1,256 @@
+/*
+ *    GS2Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gdbm.*;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+import java.util.Set;
+import java.util.Iterator;
+
+/** Greenstone 2 collection classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public class GS2Browse 
+    extends AbstractBrowse 
+{
+    
+    protected GDBMWrapper gdbm_src = null;
+
+    public GS2Browse() 
+    {
+	this.gdbm_src = new GDBMWrapper();
+	this.macro_resolver = new GS2MacroResolver(this.gdbm_src);
+    }
+
+    public boolean configure(Element info, Element extra_info)
+    {
+	System.err.println("Configuring GS2Browse...");
+	// Open GDBM database for querying
+	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name);
+	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
+	    System.err.println("GS2Browse Error: Could not open GDBM database!");
+	    return false;
+	}
+	return super.configure(info, extra_info);
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	return this.gdbm_src.translateOID(node_id);
+    }
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.gdbm_src.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+
+    }
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return OID.getTop(node_id);
+    }
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+
+	ArrayList children = new ArrayList();
+	
+	String contains = info.getInfo("contains");
+	StringTokenizer st = new StringTokenizer(contains, ";");
+	while (st.hasMoreTokens()) {
+	    String child_id = st.nextToken().replaceAll("\"", node_id);
+	    children.add(child_id);
+	}
+	return children;
+
+    }
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata, 
+				      ArrayList metadata_names) {
+	String lang = "en";
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+	if (all_metadata) {
+	    // return everything out of the database
+	    Set keys = info.getKeys();
+	    Iterator it = keys.iterator();
+	    while(it.hasNext()) {
+		String key = (String)it.next();
+		String value = info.getInfo(key);
+		GSXML.addMetadata(this.doc, metadata_list, key, this.macro_resolver.resolve(value, lang, GS2MacroResolver.SCOPE_META, node_id));
+	    }
+	    
+	} else {
+	    for (int i=0; i<metadata_names.size(); i++) {
+		String meta_name = (String) metadata_names.get(i);
+		String value = (String)info.getInfo(meta_name);
+		GSXML.addMetadata(this.doc, metadata_list, meta_name, value);
+	    }
+	}
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	String value="";
+	if (info_type.equals(INFO_NUM_SIBS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="0";
+	    } else {
+		value = String.valueOf(getNumChildren(parent_id));
+	    }
+	    return value;
+	}
+	
+	if (info_type.equals(INFO_NUM_CHILDREN)) {
+	    return String.valueOf(getNumChildren(doc_id));
+	}
+
+	
+	if (info_type.equals(INFO_SIB_POS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		return "-1";
+	    }
+	    
+	    DBInfo info = this.gdbm_src.getInfo(parent_id);
+	    if (info==null) {
+		return "-1";
+	    }
+	    
+	    String contains = info.getInfo("contains");
+	    contains = contains.replaceAll("\"", parent_id);
+	    String [] children = contains.split(";");
+	    for (int i=0;i<children.length;i++) {
+		String child_id = children[i];
+		if (child_id.equals(doc_id)) {
+		    return String.valueOf(i+1); // make it from 1 to length
+			   
+		}
+	    }
+	    
+	    return "-1";
+	} else {
+	    return null;
+	}
+
+    }
+
+    protected int getNumChildren(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return 0;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return 0;
+	}
+	String [] children = contains.split(";");
+	return children.length;
+    }
+    
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    protected boolean isDocumentId(String node_id){
+	if (node_id.startsWith("CL")) {
+	    return false;
+	}
+	return true;
+    }
+
+}
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java	(revision 8958)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java	(revision 8959)
@@ -1,5 +1,5 @@
 /*
  *    GS2MGPPRetrieve.java
- *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
  *
  *    This program is free software; you can redistribute it and/or modify
@@ -19,27 +19,20 @@
 package org.greenstone.gsdl3.service;
 
-
 // Greenstone classes
 import org.greenstone.mgpp.*;
-import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
 
 // XML classes
-import org.w3c.dom.Element; 
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
 
 // General Java classes
 import java.io.File;
 
-
-/**
- * A ServiceRack class for retrieval in greenstone 2 MGPP collections
- *
- * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
- * @version $Revision$
- * @see ServiceRack
- */
-public class GS2MGPPRetrieve 
-    extends GS2Retrieve {
-
-    // Parameters used
+public class GS2MGPPRetrieve
+    extends AbstractGS2DocumentRetrieve
+{
+     // Parameters used
     private static final String LEVEL_PARAM = "level";
 
@@ -50,15 +43,9 @@
     
     private String default_level = null;
-
-
-    /** constructor */
-    public GS2MGPPRetrieve()
-    {
+    private String mgpp_textdir = null;
+	
+    public GS2MGPPRetrieve() {
 	this.mgpp_src = new MGPPWrapper();
-	// only the classifier service has strings at present
-	this.dictionary_name = "ClassifierBrowse";
-
     }
-
 
     /** configure this service */
@@ -78,79 +65,46 @@
 	}
 	// System.out.println("Default level: " + default_level_);
+	
+	// The location of the MGPP text files
+	mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionTextPath(this.cluster_name);
 
 	// Do generic configuration
 	return super.configure(info, extra_info);
+ 
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) {
+	String lang = "en"; //  ********** 
+	long doc_num = this.gdbm_src.OID2Docnum(doc_id);
+	if (doc_num == -1) {
+	    System.err.println("OID "+doc_id +" couldn't be converted to mgpp num");
+	    return null;
+	}
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mgpp_src.getDocument(this.mgpp_textdir, 
+						    this.default_level,
+						    doc_num);
+	    if (doc_content != null) {
+		doc_content = resolveTextMacros(doc_content, doc_id, lang);
+	    } 
+	} catch (Exception e) {
+	    System.out.println("exception happended with mgpp_src.getDocument()" + e);
+	    doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_num+"\n";
+
+	}
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
     }
 
-    
-    /** Retrieve the content of a document */
-    protected Element processDocumentContentRetrieve(Element request)
-    {
-	// Create a new (empty) result message
-	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
-	result.setAttribute(GSXML.FROM_ATT,DOCUMENT_CONTENT_RETRIEVE_SERVICE );
-	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
 
-	// Get the parameters of the request 
-	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
-	Element extlink_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, EXTLINK_PARAM);
-	boolean extlink = false;
-	if (extlink_param != null && GSXML.getValue(extlink_param).equals("1")) {
-	    extlink = true;
-	}
-
-	// Get the request doc_list
-	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
-	if (query_doc_list == null) {
-	    System.err.println("Error: DocumentContentRetrieve request specified no doc nodes.\n");
-	    return result;
-	}
-	String lang = request.getAttribute(GSXML.LANG_ATT);
-	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
-	result.appendChild(doc_list);
-
-	// The location of the MGPP text files
-	String textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
-	    File.separatorChar + GSFile.collectionTextPath(this.cluster_name);
-
-	// Get the documents
-	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
-						     GSXML.NODE_ID_ATT);
-	for (int i = 0; i < doc_ids.length; i++) {
-	    String doc_id = doc_ids[i];
-	    Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
-	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
-	    doc_list.appendChild(doc);
-	    
-	    if (extlink) {
-		doc_id = this.gdbm_src.extlink2OID(doc_id);
-		if (doc_id==null) {
-		    continue;
-		}
-		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
-	    }
-	    else if (OID.needsTranslating(doc_id)) {
-		doc_id = this.gdbm_src.translateOID(doc_id);
-		if (doc_id==null) {
-		    continue;
-		}
-		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
-	    }
-	    
-	    long doc_num = this.gdbm_src.OID2Docnum(doc_id);
-	    if (doc_num == -1) {
-		continue;
-	    }
-	    String doc_content = this.mgpp_src.getDocument(textdir, this.default_level, doc_num);
-	    doc_content = resolveTextMacros(doc_content, doc_id, lang);
-	    
-	    // For now, stick it in a text node - eventually should be parsed as xml??
-	    GSXML.addDocText(this.doc, doc, doc_content);
-	    
-
-	}
-	
-	return result;
-    }
 }
-
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGRetrieve.java	(revision 8958)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGRetrieve.java	(revision 8959)
@@ -1,5 +1,5 @@
 /*
  *    GS2MGRetrieve.java
- *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
  *
  *    This program is free software; you can redistribute it and/or modify
@@ -19,44 +19,30 @@
 package org.greenstone.gsdl3.service;
 
-
 // Greenstone classes
 import org.greenstone.mg.*;
-import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
 
 // XML classes
 import org.w3c.dom.Element;
+import org.w3c.dom.Text;
 
 // General Java classes
 import java.io.File;
 
-
-/**
- *
- * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
- * 
- */
-
 public class GS2MGRetrieve
-    extends GS2Retrieve {
-
-    // Parameters used
-    private static final String INDEX_PARAM = "index";
-
+    extends AbstractGS2DocumentRetrieve
+{
     // Elements used in the config file that are specific to this class
     private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
 
     private MGWrapper mg_src = null;
-
+    private String mg_basedir = null;
+    private String mg_textdir = null;
     private String default_index = null;
-
-
-    /** constructor */
-    public GS2MGRetrieve()
-    {
+	
+    public GS2MGRetrieve() {
 	this.mg_src = new MGWrapper();
-	// only the classifier service has strings at present
-	this.dictionary_name = "ClassifierBrowse";
     }
-
 
     /** configure this service */
@@ -79,93 +65,47 @@
 	// System.out.println("Default index: " + this.default_index);
 
+	// The location of the MG index and text files
+	mg_basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+	mg_textdir = GSFile.collectionTextPath(this.cluster_name);
+	// index is only needed to start up MG, not used so just use the default index
+	String indexpath = GSFile.collectionIndexPath(this.cluster_name, this.default_index);
+	this.mg_src.setIndex(indexpath);
+
 	// Do generic configuration
 	return super.configure(info, extra_info);
     }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) {
+	String lang = "en"; //  ********** 
+	long doc_num = this.gdbm_src.OID2Docnum(doc_id);
+	if (doc_num == -1) {
+	    System.err.println("OID "+doc_id +" couldn't be converted to mg num");
+	    return null;
+	}
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = this.mg_src.getDocument(this.mg_basedir,
+						     this.mg_textdir, doc_num);
+	
+	if (doc_content!=null) {
+	    // remove any ctrl-c or ctrl-b 
+	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
+	    // replace _httpimg_ with the correct address
+	    doc_content = resolveTextMacros(doc_content, doc_id, lang);
+	    //GSXML.addDocText(this.doc, doc, doc_content);
+	} else {
+	    System.err.println("the doc content was null, not getting that section\n");
+	    doc_content = "couldn't retrieve content for this section\n";
+	}
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
 
 
-    /** Retrieve the content of a document */
-    protected Element processDocumentContentRetrieve(Element request)
-    {
-	// Create a new (empty) result message
-	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
-	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
-	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
-
-	// Get the parameters of the request - no parameters at this stage
-  	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
-	Element extlink_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, EXTLINK_PARAM);
-	boolean extlink = false;
-	if (extlink_param != null && GSXML.getValue(extlink_param).equals("1")) {
-	    extlink = true;
-	}
-	// Get the request content 
-	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
-	if (query_doc_list == null) {
-	    System.err.println("Error: DocumentContentRetrieve request specified no doc nodes.\n");
-	    return result;
-	}
-	
-	String lang = request.getAttribute(GSXML.LANG_ATT);
-	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
-	result.appendChild(doc_list);
-	
-	// The location of the MG index and text files
-	String basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
-	String textdir = GSFile.collectionTextPath(this.cluster_name);
-	// index is only needed to start up MG, not used so just use the default index
-	String indexpath = GSFile.collectionIndexPath(this.cluster_name, this.default_index);
-	this.mg_src.setIndex(indexpath);
-	
-	// Get the documents
-	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
-							     GSXML.NODE_ID_ATT);
-	for (int i = 0; i < doc_ids.length; i++) {
-	    String doc_id = doc_ids[i];
-	    // Create the document node
-	    Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
-	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
-	    doc_list.appendChild(doc);
-	    
-	    if (extlink) {
-		if (OID.needsTranslating(doc_id)) {
-		    
-		    // just ignore the extension cos it has no meaning here
-		    // or should it be added back on for the next step???
-		    doc_id = this.gdbm_src.extlink2OID(doc_id.substring(0,doc_id.length()-3));
-		} else {
-		    doc_id = this.gdbm_src.extlink2OID(doc_id);
-		}
-		if (doc_id == null) {
-		    System.err.println("extlink "+doc_id +" couldn't be converted to proper id");
-		    continue;
-		}
-		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id); // reset the att cos it may have changed
-		    
-	    }
-	    else if (OID.needsTranslating(doc_id)) {
-		doc_id = this.gdbm_src.translateOID(doc_id);
-		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id); // reset the att cos it may have changed
-	    }
-	    
-	    long doc_num = this.gdbm_src.OID2Docnum(doc_id);
-	    if (doc_num == -1) {
-		System.err.println("OID "+doc_id +" couldn't be converted to mg num");
-		continue;
-	    }
-	    String doc_content = this.mg_src.getDocument(basedir, textdir, doc_num);
-	    
-	    if (doc_content!=null) {
-		// remove any ctrl-c or ctrl-b (I hope these are the right codes)
-		doc_content = doc_content.replaceAll("\u0002|\u0003", "");
-		// replace _httpimg_ with the correct address
-		doc_content = resolveTextMacros(doc_content, doc_id, lang);
-		GSXML.addDocText(this.doc, doc, doc_content);
-	    } else {
-		System.err.println("the doc content was null, not getting that section\n");
-		GSXML.addDocText(this.doc, doc, "couldn't retrieve content for this section\n");
-	    }
-	}
-
-	return result;
-    }
 }
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Retrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Retrieve.java	(revision 8958)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Retrieve.java	(revision 8959)
@@ -73,5 +73,5 @@
     protected static final String INFO_SIB_POS = "siblingPosition";
 
-    protected static final String EXTLINK_PARAM = "ext";
+    protected static final String EXTID_PARAM = "ext";
     
     protected static final int DOCUMENT=1;
@@ -462,5 +462,5 @@
 		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_LEAF);
 	    } else {
-		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_INTERIOR);
+		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_INTERNAL);
 	    }
 	} 
@@ -794,5 +794,5 @@
 		}
 		metadata_list.add(metadata);
-	    } else if (param.getAttribute(GSXML.NAME_ATT).equals(EXTLINK_PARAM)&& GSXML.getValue(param).equals("1")) {
+	    } else if (param.getAttribute(GSXML.NAME_ATT).equals(EXTID_PARAM)&& GSXML.getValue(param).equals("1")) {
 		extlink = true;
 	    }
@@ -824,7 +824,7 @@
 		    // just ignore the extension cos it has no meaning here
 		    // or should it be added back on for the next step???
-		    node_id = this.gdbm_src.extlink2OID(node_id.substring(0,node_id.length()-3));
+		    node_id = this.gdbm_src.externalId2OID(node_id.substring(0,node_id.length()-3));
 		} else {
-		    node_id = this.gdbm_src.extlink2OID(node_id);
+		    node_id = this.gdbm_src.externalId2OID(node_id);
 		}
 	    }
@@ -850,5 +850,5 @@
 		    String key = (String)it.next();
 		    String value = info.getInfo(key);
-		    GSXML.addMetadata(this.doc, node_meta_list, key, this.macro_resolver.resolve(value, lang, GS2MacroResolver.SCOPE_META, node_id, info));
+		    GSXML.addMetadata(this.doc, node_meta_list, key, this.macro_resolver.resolve(value, lang, GS2MacroResolver.SCOPE_META, node_id));
 		}
 	    } else { // just get the selected ones
@@ -875,5 +875,5 @@
 	if (pos ==-1) {
 	    // just a plain meta entry eg dc.Title
-	    return macro_resolver.resolve((String)info.getInfo(metadata), lang, GS2MacroResolver.SCOPE_META, node_id, info);
+	    return macro_resolver.resolve((String)info.getInfo(metadata), lang, GS2MacroResolver.SCOPE_META, node_id);
 	}
 	
@@ -935,5 +935,5 @@
 
 	if (!multiple) {
-	    result.append(this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, GS2MacroResolver.SCOPE_META, relation_id, relation_info));
+	    result.append(this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, GS2MacroResolver.SCOPE_META, relation_id));
 	} else {
 	    // we have multiple meta
@@ -947,5 +947,5 @@
 			result.append(separator);
 		    }
-		    result.append(this.macro_resolver.resolve((String)values.elementAt(i), lang, GS2MacroResolver.SCOPE_META, relation_id, relation_info));
+		    result.append(this.macro_resolver.resolve((String)values.elementAt(i), lang, GS2MacroResolver.SCOPE_META, relation_id));
 		}
 	    }
@@ -964,5 +964,5 @@
 	    if (!multiple) {
 		result.insert(0, separator);
-		result.insert(0, this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, GS2MacroResolver.SCOPE_META, relation_id, relation_info));
+		result.insert(0, this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, GS2MacroResolver.SCOPE_META, relation_id));
 	    } else {
 		Vector values = relation_info.getMultiInfo(metadata);
@@ -970,5 +970,5 @@
 		    for (int i=values.size()-1; i>=0; i--) {
 			result.insert(0, separator);
-			result.insert(0, this.macro_resolver.resolve((String)values.elementAt(i), lang, GS2MacroResolver.SCOPE_META, relation_id, relation_info));
+			result.insert(0, this.macro_resolver.resolve((String)values.elementAt(i), lang, GS2MacroResolver.SCOPE_META, relation_id));
 		    }
 		}
@@ -1005,5 +1005,5 @@
 	}
 	// resolve any collection specific macros
-	doc_content = macro_resolver.resolve(doc_content, lang, GS2MacroResolver.SCOPE_TEXT, doc_id, info);
+	doc_content = macro_resolver.resolve(doc_content, lang, GS2MacroResolver.SCOPE_TEXT, doc_id);
 	return doc_content;
     }
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Browse.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Browse.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Browse.java	(revision 8959)
@@ -0,0 +1,186 @@
+/*
+ *    GS3Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+import java.io.File;
+
+/** Greenstone 3 collection classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+public class GS3Browse
+    extends AbstractBrowse
+{
+
+    protected SQLQuery database = null;
+
+    /** constructor */
+    public GS3Browse()
+    {
+	this.database = new SQLQuery();
+
+    }
+
+    public boolean configure(Element info, Element extra_info)
+    {
+	System.err.println("Configuring GS3Browse...");
+	
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+	
+	// generic configure
+	return super.configure(info, extra_info);
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+
+        /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+   }
+    
+    /** returns the id of the root node of the document containing node 
+     * node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getDocID(node_id);
+    }
+    
+   /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	ArrayList documents = database.getClassifierDocChildren(node_id);
+	if (documents == null) {
+	    documents = new ArrayList();
+	}
+	ArrayList children = database.getClassifierChildren(node_id);
+	if (children != null) {
+	    documents.addAll(children);
+	}
+	if (documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+    
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata, ArrayList metadata_names) {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getClassifierMetadata(node_id, metadata);
+		if (value != null) {
+		    System.out.println("found value "+value);
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null;
+    }
+
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    protected boolean isDocumentId(String node_id){
+	if (node_id.startsWith("CL")) {
+	    return false;
+	}
+	return true;
+    }
+
+}
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 8959)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 8959)
@@ -0,0 +1,112 @@
+/*
+ *    GS3MGPPRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGPPRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+
+    private MGPPWrapper mgpp_src = null;
+    
+    private String default_level = null;
+    private String mgpp_textdir = null;
+	
+    public GS3MGPPRetrieve() {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do specific configuration
+	System.out.println("Configuring GS3MGPPRetrieve...");
+
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    System.err.println("Error: default level not specified!");
+	    return false;
+	}
+	// System.out.println("Default level: " + default_level_);
+	
+	// The location of the MGPP text files
+	mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionTextPath(this.cluster_name);
+
+	// Do generic configuration
+	return super.configure(info, extra_info);
+ 
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) {
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mgpp_src.getDocument(this.mgpp_textdir, 
+						    this.default_level,
+						    doc_int);
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+	} catch (Exception e) {
+	    System.out.println("exception happended with mgpp_src.getDocument()" + e);
+	    doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_int+"\n";
+	}
+
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGRetrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 8958)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 8959)
@@ -1,5 +1,5 @@
 /*
  *    GS3MGRetrieve.java
- *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
  *
  *    This program is free software; you can redistribute it and/or modify
@@ -19,52 +19,35 @@
 package org.greenstone.gsdl3.service;
 
-
 // Greenstone classes
 import org.greenstone.mg.*;
-import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
 
 // XML classes
 import org.w3c.dom.Element;
+import org.w3c.dom.Text;
 
 // General Java classes
 import java.io.File;
 
-
-/** The content retrieval class for GS3 building
- *
- * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
- * @version $Revision$
- */
-
 public class GS3MGRetrieve
-    extends ServiceRack {
-    // the services on offer
-    protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";
-    
-    // Parameters used
-    private static final String INDEX_PARAM = "index";
-
+    extends AbstractGS3DocumentRetrieve
+{
     // Elements used in the config file that are specific to this class
     private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
 
     private MGWrapper mg_src = null;
-    protected SQLQuery database = null;
-    
-    protected Element config_info = null; // the xml from the config file
-
+    private String mg_basedir = null;
+    private String mg_textdir = null;
     private String default_index = null;
-
-    /** constructor */
-    public GS3MGRetrieve()
-    {
-	this.database = new SQLQuery();
+	
+    public GS3MGRetrieve() {
 	this.mg_src = new MGWrapper();
     }
-
 
     /** configure this service */
     public boolean configure(Element info, Element extra_info)
     {
-	this.config_info = info;
 	// Do specific configuration
 	System.out.println("Configuring GS3MGRetrieve...");
@@ -81,134 +64,58 @@
 	    return false;
 	}
+	// System.out.println("Default index: " + this.default_index);
 
-	// System.out.println("Default index: " + this.default_index);
-	Element dcr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
-	dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
-	dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
-	this.short_service_info.appendChild(dcr_service);
+	// The location of the MG index and text files
+	mg_basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+	mg_textdir = GSFile.collectionTextPath("index");
+	// index is only needed to start up MG, not used so just use the default index
+	String indexpath = GSFile.collectionIndexPath("index", this.default_index);
+	this.mg_src.setIndex(indexpath);
 
-	// check that site_home is set
-	if (this.site_home == null || this.site_home.equals("")) {
-	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
-	    return false;
-	}
-	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
-	if (site_name.equals("")) {
-	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
-	    return false;
-	}
-	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
-	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
-	    return false;
+	// Do generic configuration
+	return super.configure(info, extra_info);
+    }
+    
+    /** returns the content of a node.
+     * node_id should already have been translated if necessary
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) {
+
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
 	}
 
-	// look for document display format
-	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
-	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
-	if (display_format != null) {
-	    this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.doc.importNode(display_format, true));
-	    // should we make a copy?
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mg_src.getDocument(this.mg_basedir, 
+						  this.mg_textdir, 
+						  doc_int);
+	    // remove any ctrl-c or ctrl-b 
+	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+
+	} catch (Exception e) {
+	    System.out.println("exception happended with mg_src.getDocument()");
+	    doc_content = "this is the content for section hash id "+ doc_id+", mg doc num "+doc_int+"\n";
 	}
+	
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
 
-	return true;
     }
 
-    protected Element getServiceDescription(String service_id, String lang, String subset) {
-	if (service_id.equals(DOCUMENT_CONTENT_RETRIEVE_SERVICE)) {
-	    Element dcr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
-	    dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
-	    dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
-	    return dcr_service;
-	}
-
-	return null;
-    }
-    /** Retrieve the content of a document */
-    protected Element processDocumentContentRetrieve(Element request)
-    {
-	// Create a new (empty) result message
-	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
-	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
-	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
-
-	// Get the parameters of the request - no parameters at this stage
-  	//Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
-
-	// Get the request content 
-	
-	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
-	if (query_doc_list == null) {
-	    System.err.println("Error: DocumentContentRetrieve request specified no doc nodes.\n");
-	    return result;
-	}
-	
-	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
-	result.appendChild(doc_list);
-	
-	// The location of the MG index and text files
-	String basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
-	String textdir = GSFile.collectionTextPath("index");//this.cluster_name);
-	// index is only needed to start up MG, not used so just use the default index
-	String indexpath = GSFile.collectionIndexPath("index"/*this.cluster_name*/, this.default_index);
-	this.mg_src.setIndex(indexpath);
-	
-	// Get the documents
-	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
-							     GSXML.NODE_ID_ATT);
-	for (int i = 0; i < doc_ids.length; i++) {
-	    //ystem.out.println("getting doc id "+doc_ids[i]);
-	    String doc_id = doc_ids[i];
-	    if (GS3OID.needsTranslating(doc_id)) {
-	    //doc_id = this.gdbm_src.translateOID(doc_id);
-		doc_id = GS3OID.translateOID(doc_id);
-		//ystem.out.println("translated id = "+doc_id);
-	    }
-	    if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
-		// if we have a whole doc id, and the document is hierarchical,
-		// we want to change the id to be the top id of the section 
-		// hierarchy
-	    	doc_id = GS3OID.createOID(doc_id, "1");
-	    }
-	    String doc_num = this.database.OID2MGNum(doc_id);	    
-	    // doc nums have the index prefixed 
-	    doc_num = doc_num.substring(doc_num.indexOf(".")+1);
-	    int doc_int = Integer.parseInt(doc_num);
-	    
-	    String doc_content = "";
-	    try {
-		doc_content = this.mg_src.getDocument(basedir, textdir, doc_int);
-	    } catch (Exception e) {
-		System.out.println("exception happended with mg_src.getDocument()");
-		doc_content = "this is the content for section hash id "+ doc_id+", mg doc num "+doc_int+"\n";
-	    }
-	    
-	    //ystem.out.println("Doc content: " + doc_content + "|");
-	    //ystem.out.println("Doc ID: " + doc_id);
-	    // remove any ctrl-c or ctrl-b (I hope these are the right codes)
-	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
-	    doc_content = resolveRelativeLinks(doc_content, doc_id);
-	    // replace _httpimg_ with the correct address
-	    //doc_content = resolveImages(doc_content, doc_id);
-	    // Stick it in a text node
-	    Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
-	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
-	    GSXML.addDocText(this.doc, doc, doc_content);
-	    Element nodeContent = (Element)GSXML.getChildByTagName(doc, "nodeContent");
-	    //Element file_elem = this.doc.createElement("file");
-	    //file_elem.setAttribute("mimeType", "image/jpeg");
-	    //file_elem.setAttribute("href", "import/meinahole.jpg");
-	    //nodeContent.appendChild(file_elem);
-	    doc_list.appendChild(doc);
-	}
-
-	return result;
-    }
-    protected String resolveRelativeLinks(String doc_content, String doc_id) {
-
-	String http_path  = this.site_http_address + "/collect/"+this.cluster_name;
-	doc_content = doc_content.replaceAll("_httpcollection_", http_path);
-	return doc_content;
-	
-    }
 
 }
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Retrieve.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Retrieve.java	(revision 8958)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Retrieve.java	(revision 8959)
@@ -351,5 +351,5 @@
 		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_ROOT);
 	    } else if (parent || database.documentHasChildren(node_id)){
-		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_INTERIOR);
+		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_INTERNAL);
 	    } else {
 		node.setAttribute(GSXML.NODE_TYPE_ATT, GSXML.NODE_TYPE_LEAF);
