Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractMGPPSearch.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractMGPPSearch.java	(revision 13242)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractMGPPSearch.java	(revision 13242)
@@ -0,0 +1,779 @@
+/*
+ *    AbstractMGPPSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.util.Iterator;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.Vector;
+import java.io.File;
+
+import org.apache.log4j.*;
+
+/** Partially implements a generic MGPP search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+abstract public class AbstractMGPPSearch
+    extends AbstractSearch
+{
+
+      static Category logger = Category.getInstance(org.greenstone.gsdl3.service.AbstractMGPPSearch.class.getName());
+
+    // extra services offered by mgpp collections
+    private static final String FIELD_QUERY_SERVICE = "FieldQuery";
+    private static final String ADVANCED_FIELD_QUERY_SERVICE = "AdvancedFieldQuery";
+
+    // extra parameters used
+    // private static final String INDEX_FIELD_PARAM = "index";
+    private static final String LEVEL_PARAM = "level";
+    private static final String RANK_PARAM = "sortBy";
+    private static final String RANK_PARAM_RANK = "1";
+    private static final String RANK_PARAM_NONE = "0";
+    private static final String SIMPLE_FIELD_PARAM = "simpleField";
+    private static final String ADVANCED_FIELD_PARAM = "complexField";
+
+    // more params for field query
+    private static final String FIELD_QUERY_PARAM = "fqv";
+    private static final String FIELD_STEM_PARAM = "fqs";
+    private static final String FIELD_CASE_PARAM = "fqc";
+    private static final String FIELD_ACCENT_PARAM="fqa";
+    private static final String FIELD_FIELD_PARAM = "fqf";
+    private static final String FIELD_COMBINE_PARAM = "fqk";
+    private static final String FIELD_COMBINE_PARAM_AND = "0";
+    private static final String FIELD_COMBINE_PARAM_OR = "1";
+    private static final String FIELD_COMBINE_PARAM_NOT = "2";
+   
+    private static final String ACCENT_PARAM="accent";
+
+    // some stuff for config files
+    private static final String SEARCH_TYPE_ELEM = "searchType";
+    private static final String SEARCH_TYPE_PLAIN = "plain";
+    private static final String SEARCH_TYPE_FORM = "form";
+    private static final String SEARCH_TYPE_FORM_SIMPLE = "simple";
+    private static final String SEARCH_TYPE_FORM_ADVANCED = "advanced";
+    protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    protected static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+    protected static final String LEVEL_ELEM = "level";
+
+    protected static final String STEMINDEX_OPTION = "stemIndexes";
+    protected static final String MAXNUMERIC_OPTION = "maxnumeric";
+
+    protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+    protected static final String STEM_ATT = "stem";
+    protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+    protected static final String FREQ_ATT = "freq";
+
+    private static final int TEXT_QUERY = 0;
+    private static final int SIMPLE_QUERY = 1;
+    private static final int ADVANCED_QUERY = 2;
+
+    protected static final String FIELD_ATT = "field";
+    private MGPPWrapper mgpp_src=null;
+
+   // the default level for retrieval - and we'll use it for searching too
+    private String default_level=null;
+    // the default field for searching
+    private String default_field = null;
+    // which search services will we offer??
+    private boolean plain_search = false;
+    private boolean simple_form_search = false;
+    private boolean advanced_form_search = false;
+
+    // stem indexes available
+    private boolean does_case=true;
+    private boolean does_stem=true;
+    private boolean does_accent=false;
+    
+    // maxnumeric - not used yet. needs to be passed to MGPPWrapper.
+    private int maxnumeric = 4;
+
+    /** the stem used for the index files */
+    protected String index_stem = null;
+
+    public AbstractMGPPSearch()
+    {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (super.configure(info, extra_info) == false)
+	    return false;
+	
+	// Do specific configuration
+	logger.info("Configuring AbstractMGPPSearch...");
+
+	// do we support any of the extended features?
+	does_chunking = true;
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	} // otherwise its "" and will be the first one in the list
+	
+	// the index stem is either specified in the config file or is "index"
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    logger.warn("AbstractMGPPSearch.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+
+	// get index options
+	Element index_option_list = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_OPTION_ELEM + GSXML.LIST_MODIFIER);
+	if (index_option_list != null) {
+	    NodeList options = index_option_list.getElementsByTagName(GSXML.INDEX_OPTION_ELEM);
+	    for (int i=0; i<options.getLength(); i++) {
+		Element opt = (Element)options.item(i);
+		String name = opt.getAttribute(GSXML.NAME_ATT);
+		String value = opt.getAttribute(GSXML.VALUE_ATT);
+		if (name.equals(MAXNUMERIC_OPTION)) {
+		    int maxnum = Integer.parseInt(value);
+		    if (4 <= maxnum && maxnum < 512) {
+			maxnumeric = maxnum;
+		    }
+		}
+		else if (name.equals(STEMINDEX_OPTION)) {
+		    int stemindex = Integer.parseInt(value);
+		    // stem and case are true by default, accent folding false by defualt
+		    if ((stemindex & 1) == 0) {
+			does_case = false;
+		    }
+		    if ((stemindex & 2) == 0) {
+			does_stem = false;
+		    }
+		    if ((stemindex & 4) != 0) {
+			does_accent = true;
+		    }
+		}
+	    }
+	}
+
+	// the generic config has set up the text query service, but we may not want it
+	Element search_type_list = (Element) GSXML.getChildByTagName(info, SEARCH_TYPE_ELEM + GSXML.LIST_MODIFIER);
+	if (search_type_list == null) {
+	    // assume form and plain
+	    this.plain_search = true;
+	    this.simple_form_search = true;
+	    this.advanced_form_search = true;
+	} else {
+	    NodeList types = search_type_list.getElementsByTagName(SEARCH_TYPE_ELEM);
+	    for (int i=0; i<types.getLength(); i++) {
+		Element t = (Element)types.item(i);
+		String type_name = t.getAttribute(GSXML.NAME_ATT);
+		if (type_name.equals(SEARCH_TYPE_PLAIN)) {
+		    this.plain_search = true;
+		} else if (type_name.equals(SEARCH_TYPE_FORM)) {
+		    String type_type = t.getAttribute(GSXML.TYPE_ATT);
+		    if (type_type.equals("")) {
+			this.simple_form_search = true;
+			this.advanced_form_search = true;
+		    } else if (type_type.equals(SEARCH_TYPE_FORM_SIMPLE)) {
+			this.simple_form_search = true;
+		    } else if (type_type.equals(SEARCH_TYPE_FORM_ADVANCED)) {
+			this.advanced_form_search = true;
+			
+		    }
+		}
+	    }
+	}
+
+	if (!this.plain_search) {
+	    // need to remove the TextQuery service
+	    Element tq_service = GSXML.getNamedElement(short_service_info, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	    short_service_info.removeChild(tq_service);
+	    
+	}
+
+       
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    logger.error("default level not specified!");
+	    return false;
+	}
+
+	// the default level is also the level which gdbm is expecting
+	// this must not be overwritten
+	this.mgpp_src.setReturnLevel(this.default_level);
+	// return term info
+	this.mgpp_src.setReturnTerms(true);
+	// set the default - this may be overwritten by query params
+	this.mgpp_src.setQueryLevel(this.default_level);
+
+	// set up the extra services which are available for this collection
+	// check the config info - if there is no field list, then there is no fielded searching
+	
+	Element field_list = (Element) GSXML.getChildByTagName(info, GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	if (field_list==null) {
+	    // nothing more to do
+	    return true;
+	}
+
+	// the format info is the same for all services
+	Element format_info = (Element)format_info_map.get(TEXT_QUERY_SERVICE);
+
+	// find the default field - use the first one
+	Element first_field = (Element)GSXML.getChildByTagName(field_list, GSXML.FIELD_ELEM);
+	default_field = first_field.getAttribute(GSXML.SHORTNAME_ATT);
+	// else set up the fielded query services
+	
+	if (this.simple_form_search) {
+	    // set up short_service_info_ - for now just has id and type - name will be added in on teh fly
+	    Element fq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    fq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    fq_service.setAttribute(GSXML.NAME_ATT, FIELD_QUERY_SERVICE);
+	    this.short_service_info.appendChild(fq_service);
+
+	    if (format_info != null) {
+		this.format_info_map.put(FIELD_QUERY_SERVICE, format_info);
+	    }
+	}
+	
+	if (this.advanced_form_search) {
+	    Element afq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    afq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    afq_service.setAttribute(GSXML.NAME_ATT, ADVANCED_FIELD_QUERY_SERVICE);
+	    this.short_service_info.appendChild(afq_service);
+	    
+	    if (format_info != null) {
+		this.format_info_map.put(ADVANCED_FIELD_QUERY_SERVICE, format_info);
+	    }
+	}
+	return true;
+
+    }
+
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+	// should we check that the service is actually on offer? presumably we wont get asked for services that we haven't advertised previously.
+	
+	if (!service_id.equals(FIELD_QUERY_SERVICE) && !service_id.equals(ADVANCED_FIELD_QUERY_SERVICE)) {
+	    return super.getServiceDescription(service_id, lang, subset);
+	}
+	
+	
+	Element service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	service.setAttribute(GSXML.NAME_ATT, service_id);
+	if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service_id+".name", lang)));
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service_id+".submit", lang)));
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service_id+".description", lang)));
+	
+	}
+	if (subset == null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);	
+	    service.appendChild(param_list);
+	    if (service_id.equals(FIELD_QUERY_SERVICE)) {
+		
+		addCustomQueryParams(param_list, lang);
+		createParameter(MAXDOCS_PARAM, param_list, lang);
+		// create a multi param for the fields etc
+		// text box, field
+		Element multiparam = null;
+		Element param=null;
+		multiparam = GSXML.createParameterDescription(this.doc, SIMPLE_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+		multiparam.setAttribute("occurs", "4");
+		param_list.appendChild(multiparam);
+		
+		// the components
+		createParameter(FIELD_QUERY_PARAM, multiparam,  lang);
+		createParameter(FIELD_FIELD_PARAM, multiparam,  lang);
+		
+	    } else { 
+		createParameter(LEVEL_PARAM, param_list,  lang);
+		createParameter(RANK_PARAM, param_list, lang);
+		createParameter(MAXDOCS_PARAM, param_list,  lang);
+		
+		
+		// create a multi param for the fields etc
+		// text box, stem, case, field
+		
+		Element multiparam = null;
+		Element param=null;
+		
+		multiparam = GSXML.createParameterDescription(this.doc, ADVANCED_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+		multiparam.setAttribute("occurs", "4");
+		param_list.appendChild(multiparam);
+		
+		createParameter(FIELD_COMBINE_PARAM, multiparam, lang);
+		createParameter(FIELD_QUERY_PARAM, multiparam, lang);
+		if (this.does_case) {
+		    createParameter(FIELD_CASE_PARAM, multiparam, lang);
+		}
+		if (this.does_stem) {
+		    createParameter(FIELD_STEM_PARAM, multiparam, lang);
+		}
+		if (this.does_accent) {
+		    createParameter(FIELD_ACCENT_PARAM, multiparam, lang);
+		}
+		createParameter(FIELD_FIELD_PARAM, multiparam, lang);    
+		
+	    } 
+	}
+	return service;
+	
+    }
+
+    /** add in the mgpp specific params to TextQuery */
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	createParameter(LEVEL_PARAM, param_list, lang);
+	if (this.does_case){
+	    createParameter(CASE_PARAM, param_list, lang);
+	}
+	if (this.does_stem){
+	    createParameter(STEM_PARAM, param_list, lang);
+	}
+	if (this.does_accent){
+	    createParameter(ACCENT_PARAM, param_list, lang);
+	}
+	createParameter(MATCH_PARAM, param_list, lang);
+	createParameter(RANK_PARAM, param_list, lang);
+    }
+
+    /** create a param and add to the list */
+    protected void createParameter(String name, Element param_list, String lang) 
+    {
+	Element param = null;
+	if (name.equals(LEVEL_PARAM)) {
+	    ArrayList level_ids = new ArrayList();
+	    ArrayList level_names = new ArrayList();
+	    getLevelData(level_ids, level_names, lang);
+	    if (level_ids.size()>1) {
+		// the first one is the default
+		param = GSXML.createParameterDescription2(this.doc, LEVEL_PARAM, getTextString("param."+LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)level_ids.get(0), level_ids, level_names);
+	    }
+	    
+	} else if (name.equals(RANK_PARAM)) {
+	    String [] vals1 = {RANK_PARAM_RANK, RANK_PARAM_NONE };
+	    String [] vals1_texts = { getTextString("param."+RANK_PARAM+"."+RANK_PARAM_RANK, lang), getTextString("param."+RANK_PARAM+"."+RANK_PARAM_NONE, lang)};
+	    
+	    param = GSXML.createParameterDescription(this.doc, RANK_PARAM, getTextString("param."+RANK_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, RANK_PARAM_RANK, vals1, vals1_texts );
+	    
+	} else if (name.equals(FIELD_QUERY_PARAM)) {
+	    param = GSXML.createParameterDescription(this.doc, FIELD_QUERY_PARAM, getTextString("param."+FIELD_QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+	    
+	    
+	} else if (name.equals(FIELD_CASE_PARAM) || name.equals(FIELD_STEM_PARAM) || name.equals(FIELD_ACCENT_PARAM)) {
+	    String[] bool_ops = {"0", "1"};
+	    String[] bool_texts = {getTextString("param.boolean.off", lang, "AbstractSearch"),getTextString("param.boolean.on", lang, "AbstractSearch")}; 
+	    param = GSXML.createParameterDescription(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_BOOLEAN, BOOLEAN_PARAM_ON, bool_ops, bool_texts);
+	    
+	} else if (name.equals(FIELD_FIELD_PARAM)) {
+	    ArrayList fields = new ArrayList();
+	    ArrayList field_names = new ArrayList();
+	    getIndexData(fields, field_names, lang);
+	    // the field list -  read from config file
+	    param = GSXML.createParameterDescription2(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)fields.get(0), fields, field_names );
+	    
+	} else if (name.equals(FIELD_COMBINE_PARAM)) {
+	    
+	    String []vals = {FIELD_COMBINE_PARAM_AND, FIELD_COMBINE_PARAM_OR, FIELD_COMBINE_PARAM_NOT};
+	    String []val_texts = {getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_AND, lang), getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_OR, lang), getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_NOT, lang)}; 
+		
+	    
+	    param = GSXML.createParameterDescription(this.doc, FIELD_COMBINE_PARAM, "", GSXML.PARAM_TYPE_ENUM_SINGLE, FIELD_COMBINE_PARAM_AND, vals, val_texts);
+	    param.setAttribute(GSXML.PARAM_IGNORE_POS_ATT, "0");
+	}
+
+	if (param != null) {
+	    param_list.appendChild(param);
+	} else {
+	    super.createParameter(name, param_list, lang);
+	}
+    }
+    
+    // should cache some of this
+    protected void getLevelData(ArrayList level_ids, ArrayList level_names, String lang) 
+    {
+	Element level_list = (Element)GSXML.getChildByTagName(this.config_info, LEVEL_ELEM+GSXML.LIST_MODIFIER);
+	NodeList levels = level_list.getElementsByTagName(LEVEL_ELEM);
+	for (int i=0; i<levels.getLength(); i++) {
+	    level_ids.add(((Element)levels.item(i)).getAttribute(GSXML.NAME_ATT));
+	    level_names.add(getTextString("level."+level_ids.get(i), lang));
+	}
+	
+    }
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang){
+	// we currently only have one index, and use the field data as the index
+		    
+	// the field list -  read from config file
+	Element field_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	NodeList fields = field_list.getElementsByTagName(GSXML.FIELD_ELEM);
+	for (int i=0; i< fields.getLength();i++) {
+	    String shortname = ((Element)fields.item(i)).getAttribute(GSXML.SHORTNAME_ATT);
+	    String name = ((Element)fields.item(i)).getAttribute(GSXML.NAME_ATT);
+	    if (name.equals("")) {
+		// no name, ignore
+		continue;
+	    }
+	    // TODO change field so that name is the id, and full metadata name is somthing else
+	    if (shortname.equals("")) {
+		shortname = name;
+	    }
+	    index_ids.add(shortname);
+	    // should these be changed to a text element based on lang?
+	    // or is the name of a metadata element eg dc:Title its 
+	    // name in all langs
+	    index_names.add(name);
+	}
+    }
+
+    // the following three functions are needed so the base class can 
+    // call the process+SERVICE_NAME methods
+    /** process a text query */
+    protected Element processTextQuery(Element request) {
+	return processAnyQuery(request, TEXT_QUERY);
+    }
+
+    /** process a field query */    
+    protected Element processFieldQuery(Element request) {
+	return processAnyQuery(request, SIMPLE_QUERY);
+    }
+
+    /** process an advanced field query */
+    protected Element processAdvancedFieldQuery(Element request) {
+	return processAnyQuery(request, ADVANCED_QUERY);
+    }
+    
+    /** process a  query */
+    protected Element processAnyQuery(Element request, int query_type)
+    {
+
+	String service_name=null;
+	String empty_query_test_param=null;
+	// set up the type specific bits
+	switch (query_type) {
+	case TEXT_QUERY:
+	    service_name = TEXT_QUERY_SERVICE; 
+	    empty_query_test_param = QUERY_PARAM;
+	    break;
+	case SIMPLE_QUERY:
+	    service_name = FIELD_QUERY_SERVICE;
+	    empty_query_test_param = FIELD_QUERY_PARAM;
+	    break;
+	case ADVANCED_QUERY:
+	    service_name = ADVANCED_FIELD_QUERY_SERVICE;
+	    empty_query_test_param = FIELD_QUERY_PARAM;
+	    break;
+	default:
+	    // should never get here
+	    logger.error("wrong query type!!");
+	    return null;
+	}
+	
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, service_name);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(empty_query_test_param);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If a field hasn't been specified, use the default - for textQuery
+	String field = (String) params.get(INDEX_PARAM); 
+	if (field == null) {
+	    field = default_field;
+	}
+	
+	// set up mgpp_src
+	String indexdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mgpp_src.loadIndexData(indexdir);
+	setStandardQueryParams(params);
+	
+	// if field search, create the query string
+	switch (query_type) {
+	case TEXT_QUERY:
+	    query = addFieldInfo(query, field);
+	    break;
+	case SIMPLE_QUERY:
+	    query = parseFieldQueryParams(params);
+	    break;
+	case ADVANCED_QUERY:
+	    query = parseAdvancedFieldQueryParams(params);
+	    break;
+	}
+
+	// run the query
+       	this.mgpp_src.runQuery(query);
+	MGPPQueryResult mqr= this.mgpp_src.getQueryResult();
+
+	// build up the response
+	
+	// Create a metadata list to store information about the query results
+	// should we be using metadataList? or something else?
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	long totalDocs = mqr.getTotalDocs();
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+totalDocs);
+
+	// Create a document list to store the matching documents, and add them
+	Vector docs = mqr.getDocs();
+
+	// add a metadata item to specify docs returned
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+docs.size());
+	// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+	GSXML.addMetadata(this.doc, metadata_list, "query", query);
+	if (docs.size()>0) {
+	    Element document_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    result.appendChild(document_list);
+	    for (int d = 0; d < docs.size(); d++) {
+		long docnum = ((MGPPDocInfo) docs.elementAt(d)).num_;
+		float rank = ((MGPPDocInfo) docs.elementAt(d)).rank_;
+		String doc_id = MGPPNum2OID(docnum);
+		Element doc_node = createDocNode(doc_id, Float.toString(rank));
+		//doc_node.setAttribute("rank", Float.toString(rank));
+		document_list.appendChild(doc_node);
+	    }
+	}
+	// Create a term list to store the term information, and add it
+	String query_level = (String)params.get(LEVEL_PARAM); // the current query level
+	Element term_list = this.doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);
+	Vector terms = mqr.getTerms();
+	for (int t = 0; t < terms.size(); t++) {
+	    MGPPTermInfo term_info = (MGPPTermInfo) terms.get(t);
+	    
+	    Element term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+	    term_elem.setAttribute(GSXML.NAME_ATT, term_info.term_);
+	    term_elem.setAttribute(STEM_ATT, "" + term_info.stem_method_);
+	    term_elem.setAttribute(FREQ_ATT, "" + term_info.term_freq_);
+	    term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + term_info.match_docs_);
+	    field = term_info.tag_;
+	    if (field.equals(query_level)) {
+		// ignore
+		field = "";
+	    }
+	    term_elem.setAttribute(FIELD_ATT, field);
+	    
+	    Vector equiv_terms = term_info.equiv_terms_;
+	    Element equiv_term_list = this.doc.createElement(EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+	    term_elem.appendChild(equiv_term_list);
+	    
+	    for (int et = 0; et < equiv_terms.size(); et++) {
+		String equiv_term = (String) equiv_terms.get(et);
+
+		Element equiv_term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+		equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term);
+		equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "");
+		equiv_term_elem.setAttribute(FREQ_ATT, "");
+		equiv_term_list.appendChild(equiv_term_elem);
+	    }
+
+	    term_list.appendChild(term_elem);
+	}
+
+	return result;
+    }
+
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap params) {
+
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+	   
+	    if (name.equals(CASE_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON)?true:false);
+		this.mgpp_src.setCase(val);		
+	    } else if (name.equals(STEM_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON)?true:false);
+		this.mgpp_src.setStem(val);
+	    } else if (name.equals(MAXDOCS_PARAM)&& !value.equals("")) {
+		int docs = Integer.parseInt(value);
+		this.mgpp_src.setMaxDocs(docs);
+	    } else if (name.equals(LEVEL_PARAM)) {
+		this.mgpp_src.setQueryLevel(value);
+	    } else if (name.equals(MATCH_PARAM)) {
+		int mode;
+		if (value.equals(MATCH_PARAM_ALL)) mode=1;
+		else mode=0;
+		this.mgpp_src.setMatchMode(mode);
+	    } else if (name.equals(RANK_PARAM)) {
+		if (value.equals(RANK_PARAM_RANK)) {
+		    this.mgpp_src.setSortByRank(true);
+		} else if (value.equals(RANK_PARAM_NONE)) {
+		    this.mgpp_src.setSortByRank(false);
+		}
+	    } // ignore any others
+	}
+	return true;
+    }
+
+    protected String addFieldInfo(String query, String field) {
+	if (field.equals("") || field.equals("ZZ")) {
+	    return query;
+	}
+	return "["+query+"]:"+field;
+    }
+    /** combines all the field params into a single query
+     * - for simple field query */
+    protected String parseFieldQueryParams(HashMap params) {
+
+	StringBuffer final_query = new StringBuffer(256);
+	String text_line = (String)params.get(FIELD_QUERY_PARAM);
+	String[] texts = text_line.split(",", -1);
+	String field_line = (String)params.get(FIELD_FIELD_PARAM);
+	String[] fields = field_line.split(",", -1);
+	String combine="&";
+	String match = (String)params.get(MATCH_PARAM);
+	if (match.equals(MATCH_PARAM_SOME)) {
+	    combine = "|";
+	}
+	
+	for (int i=0; i<texts.length; i++) {
+
+	    String q = texts[i].trim();
+	    if (!q.equals("")) {
+		addQueryElem(final_query, q, fields[i], combine);
+	    }
+	}
+
+	return final_query.toString();
+    }
+
+    /** combines all the field params into a single query
+     * - for advanced field query */   
+    protected String parseAdvancedFieldQueryParams(HashMap params) {
+
+	StringBuffer final_query = new StringBuffer(256);
+	String text_line = (String)params.get(FIELD_QUERY_PARAM);
+	String[] texts = text_line.split(",", -1);
+	String field_line = (String)params.get(FIELD_FIELD_PARAM);
+	String[] fields = field_line.split(",", -1);
+	String case_line = (String)params.get(FIELD_CASE_PARAM);
+	String[] cases = case_line.split(",", -1);
+	String stem_line = (String)params.get(FIELD_STEM_PARAM);
+	String[] stems = stem_line.split(",", -1);
+	String  combine_line = (String)params.get(FIELD_COMBINE_PARAM);
+	String [] combines = combine_line.split(",", -1);
+	String combine = "&";
+	for (int i=0; i<texts.length; i++) {
+	    if (i==0) {// assume first one is blank
+		combine = "";
+	    } else {
+		String x = combines[i]; 
+		if (x.equals(FIELD_COMBINE_PARAM_AND)) {
+		    combine = "&";
+		} else if (x.equals(FIELD_COMBINE_PARAM_OR)) {
+		    combine = "|";
+		} else if (x.equals(FIELD_COMBINE_PARAM_NOT)) {
+		    combine = "!";
+		} 
+		
+	    }
+
+	    String q = texts[i].trim();
+	    if (!q.equals("")) {
+		q = addStemAndCase(q, stems[i], cases[i]);
+		addQueryElem(final_query, q, fields[i], combine);
+	    }
+	}
+
+	return final_query.toString();
+    }
+
+    protected void addQueryElem(StringBuffer s, String q, String f, String c) {
+
+	String combine="";
+	if (s.length()>0) {
+	    combine = " "+c+" ";
+	}
+	if (f.equals("")||f.equals("ZZ")) {
+	    s.append(combine+q);
+	} else {
+	    s.append(combine+"["+q+"]:"+f);
+	}
+    }
+    
+    protected String addStemAndCase(String q, String s, String c) {
+	String mods = "#";
+	if (c.equals("1")) {
+	    mods += "i";
+	} else {
+	    mods += "c";
+	}
+	if (s.equals("1")) {
+	    mods += "s";
+	} else {
+	    mods+= "u";
+	}
+	StringBuffer temp = new StringBuffer();
+	String [] terms = q.split(" ");
+	for (int i=0; i<terms.length; i++) {
+	    String t = terms[i].trim();
+	    
+	    if (!t.equals("") && !t.equals("TX")) {
+		temp.append(" "+t+mods);
+	    }
+	}
+	return temp.toString();
+    }
+
+    /** convert MGPP internal id to Greenstone oid */
+    abstract protected String MGPPNum2OID(long docnum);
+
+}
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractMGSearch.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractMGSearch.java	(revision 13242)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractMGSearch.java	(revision 13242)
@@ -0,0 +1,311 @@
+/*
+ *    AbstractMGSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java
+import java.util.Vector;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.Iterator;
+import java.io.File;
+
+import org.apache.log4j.*;
+
+/** Partially implements a generic MG search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+abstract public class AbstractMGSearch
+    extends AbstractSearch
+{
+
+    static Category logger = Category.getInstance(org.greenstone.gsdl3.service.AbstractMGSearch.class.getName());
+    protected static final String CASE_PARAM = "case";
+    protected static final String STEM_PARAM = "stem";
+    protected static final String MATCH_PARAM = "matchMode";
+    protected static final String MATCH_PARAM_ALL = "all";
+    protected static final String MATCH_PARAM_SOME = "some";
+
+    protected static final String BOOLEAN_PARAM_ON = "1";
+    protected static final String BOOLEAN_PARAM_OFF = "0";
+
+    protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+    protected static final String STEM_ATT = "stem";
+    protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+    protected static final String FREQ_ATT = "freq";
+
+    // Elements used in the config file that are specific to this class
+    protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    protected static final String INDEX_STEM_ELEM = "indexStem";
+    protected static final String INDEX_ELEM = "index";
+
+    /** the stem used for the index files */
+    protected String index_stem = null;
+    protected MGWrapper mg_src = null;
+
+    
+    public AbstractMGSearch()
+    {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	// do we support any of the extended features?
+	does_chunking = true;
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	} // otherwise will be "", and the first one will be the default
+
+	// the index stem is either the collection name or is specified in the config file
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    logger.error("AbstractMGSearch.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+	
+	// get display info from extra info
+	if (extra_info !=null) {
+	    Document owner = info.getOwnerDocument();
+	    // so far we have index specific display elements, and global format elements 
+	    NodeList indexes = info.getElementsByTagName(GSXML.INDEX_ELEM);
+	    Element config_search = (Element)GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+	    
+	    for (int i=0; i<indexes.getLength();i++) {
+		Element ind = (Element)indexes.item(i);
+		String name = ind.getAttribute(GSXML.NAME_ATT);
+		Element node_extra = GSXML.getNamedElement(config_search,
+							   GSXML.INDEX_ELEM,
+							   GSXML.NAME_ATT,
+							   name);
+		if (node_extra == null) {
+		    logger.error("haven't found extra info for index named "+name);
+		    continue;
+		}
+		
+		// get the display elements if any - displayName
+		NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		if (display_names !=null) {
+		    for (int j=0; j<display_names.getLength(); j++) {
+			Element e = (Element)display_names.item(j);
+			ind.appendChild(owner.importNode(e, true));
+		    }
+		}
+	    } // for each index
+	}
+	return true;
+    }
+
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	createParameter(CASE_PARAM, param_list, lang);
+	createParameter(STEM_PARAM, param_list, lang);
+	createParameter(MATCH_PARAM, param_list, lang);	
+    }
+
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang) {
+	// the index info -
+	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, INDEX_ELEM+GSXML.LIST_MODIFIER);
+	NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+	int len = indexes.getLength();
+	// now add even if there is only one
+	for (int i=0; i<len; i++) {
+	    Element index = (Element)indexes.item(i);
+	    index_ids.add(index.getAttribute(GSXML.NAME_ATT));
+	    index_names.add(GSXML.getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+		
+	}
+
+    }
+    /** do the actual query */
+    protected Element processTextQuery(Element request) 
+    {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If an index hasn't been specified, use the default
+	String index = (String) params.get(INDEX_PARAM); 
+	if (index == null) {
+	    index = this.default_index;
+	}
+
+	// The location of the MG index and text files
+	String basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +  File.separatorChar;  // Needed for MG
+	String textdir = GSFile.collectionTextPath(this.index_stem);
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, index);
+	this.mg_src.setIndex(indexpath);
+	
+	// set the mg query parameters to the values the user has specified
+	setStandardQueryParams(params);
+	this.mg_src.runQuery(basedir, textdir, query);
+	MGQueryResult mqr = this.mg_src.getQueryResult();
+	if (mqr.isClear()) {
+	    // something has gone wrong
+	    GSXML.addError(this.doc, result, "Couldn't query the mg database", GSXML.ERROR_TYPE_SYSTEM);
+	    return result;
+	}
+	long totalDocs = mqr.getTotalDocs();
+
+	// Get the docnums out, and convert to HASH ids
+	Vector docs = mqr.getDocs();
+	if (docs.size() == 0) {
+	    logger.error("No results found...\n");
+	}
+
+	// Create a metadata list to store information about the query results
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+
+	// Add a metadata element specifying the number of matching documents
+	// because teh total number is just the number returned, use numDocsReturned, not numDocsMatched
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+totalDocs);
+	// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+	GSXML.addMetadata(this.doc, metadata_list, "query", query);
+
+	if (docs.size() > 0) {
+	    // Create a document list to store the matching documents, and add them
+	    Element document_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    result.appendChild(document_list);
+	    for (int d = 0; d < docs.size(); d++) {
+		long docnum = ((MGDocInfo) docs.elementAt(d)).num_;
+		float rank = ((MGDocInfo) docs.elementAt(d)).rank_;
+		String doc_id = MGNum2OID(docnum);
+		Element doc_node = createDocNode(doc_id, Float.toString(rank));
+		document_list.appendChild(doc_node);
+	    }
+	}
+	
+	// Create a term list to store the term information, and add it
+	Element term_list = this.doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);
+	Vector terms = mqr.getTerms();
+	for (int t = 0; t < terms.size(); t++) {
+	    MGTermInfo term_info = (MGTermInfo) terms.get(t);
+
+	    String term = term_info.term_;
+	    int stem_method = term_info.stem_method_;
+	    Vector equiv_terms = term_info.equiv_terms_;
+
+	    Element term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+	    term_elem.setAttribute(GSXML.NAME_ATT, term);
+	    term_elem.setAttribute(STEM_ATT, "" + stem_method);
+
+	    Element equiv_term_list = this.doc.createElement(EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+	    term_elem.appendChild(equiv_term_list);
+
+	    long total_term_freq = 0;
+	    for (int et = 0; et < equiv_terms.size(); et++) {
+		MGEquivTermInfo equiv_term_info = (MGEquivTermInfo) equiv_terms.get(et);
+
+		Element equiv_term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+		equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term_info.term_);
+		equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + equiv_term_info.match_docs_);
+		equiv_term_elem.setAttribute(FREQ_ATT, "" + equiv_term_info.term_freq_);
+		equiv_term_list.appendChild(equiv_term_elem);
+
+		total_term_freq += equiv_term_info.term_freq_;
+	    }
+
+	    term_elem.setAttribute(FREQ_ATT, "" + total_term_freq);
+	    term_list.appendChild(term_elem);
+	}
+	return result;
+    }
+
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap params)
+    {
+	// set the default ones
+	this.mg_src.setReturnTerms(true);
+	this.mg_src.setCase(true); // turn casefolding on by default
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+
+	    if (name.equals(CASE_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setCase(val);
+	    }
+	    else if (name.equals(STEM_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setStem(val);
+	    }
+	    else if (name.equals(MATCH_PARAM)) {
+		int mode = (value.equals(MATCH_PARAM_ALL) ? 1 : 0);
+		this.mg_src.setMatchMode(mode);
+	    }
+	    else if (name.equals(MAXDOCS_PARAM)) {
+		int docs = Integer.parseInt(value);
+		this.mg_src.setMaxDocs(docs);
+	    } // ignore any others
+	}
+	return true;
+    }
+
+    /** convert MG internal id to Greenstone oid */
+    abstract protected String MGNum2OID(long docnum);
+}
