Index: /other-projects/trunk/greenstone3-extension/gate/src/org/greenstone/gsdl3/service/GATEServices.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gate/src/org/greenstone/gsdl3/service/GATEServices.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gate/src/org/greenstone/gsdl3/service/GATEServices.java	(revision 14150)
@@ -0,0 +1,291 @@
+/*
+ *    GATEServices.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// GATE classes
+import gate.*;
+import gate.creole.*;
+import gate.gui.*;
+import gate.util.persistence.PersistenceManager;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+
+/**
+ * <p>Title: GATE Greenstone3 integration</p>
+ * <p>Description: </p>
+ * <p>Copyright: Copyright (c) 2003</p>
+ * <p>Company: University of Waikato</p>
+ * @author unascribed
+ * @version 1.0
+ */
+
+public class GATEServices
+    extends ServiceRack {
+
+    // the services on offer
+    // these strings must match what is found in the properties file
+    protected static final String GATE_TAG_SERVICE = "GateTag";
+
+    protected static final String ANNOTATION_TYPE_PARAM = "annotationType";
+
+    protected Element config_info = null;
+
+    protected CorpusController application;
+
+    protected Corpus corpus;
+
+    protected String[] annotation_types = { "Date", "Location",
+					     "Organization", "Person" };
+    // Address, 
+
+    /** constructor */
+    public GATEServices()
+    {
+    }
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	System.out.println("Configuring GATEServices...");
+	this.config_info = info;
+
+	// set up short_service_info_ - for now just has name and type
+	Element tag_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tag_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_ENRICH);
+	tag_service.setAttribute(GSXML.NAME_ATT, GATE_TAG_SERVICE);
+	this.short_service_info.appendChild(tag_service);
+
+
+	// Configure GATE for use
+	try {
+	    Gate.init();
+	    Gate.getUserConfig().put(GateConstants.DOCUMENT_ADD_SPACE_ON_UNPACK_FEATURE_NAME,
+				     new Boolean(false));
+
+	    // MainFrame mainFrame = new MainFrame();
+	    // mainFrame.setSize(new java.awt.Dimension(800, 600));
+	    // mainFrame.setVisible(true);
+
+	    // Load the (pre-created) application
+	    URL applicationFileURL = ClassLoader.getSystemResource("gate.app");
+	    File applicationFile = new File(applicationFileURL.getFile());
+	    this.application = (CorpusController) PersistenceManager.loadObjectFromFile(applicationFile);
+
+	    /* Collection processing_resources = application.getPRs();
+	    Iterator pr_iterator = processing_resources.iterator();
+	    while (pr_iterator.hasNext()) {
+		ProcessingResource pr = (ProcessingResource) pr_iterator.next();
+		String pr_name = pr.getName();
+		System.out.println("PR name: " + pr_name);
+		if (pr_name.startsWith("ANNIE POS Tagger")) {
+		    String as_name = ((POSTagger) pr).getInputASName();
+		    System.out.println("AS name: " + as_name);
+		}
+		} */
+
+	    // Create a new corpus
+	    this.corpus = Factory.newCorpus("GSDL3 Corpus");
+	    this.application.setCorpus(this.corpus);
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+
+	return true;
+    }
+
+    protected Element getServiceDescription(String service, String lang, String subset) {
+
+	if (!service.equals(GATE_TAG_SERVICE)) {
+	    return null;
+	}
+	Element tag_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tag_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_ENRICH);
+	tag_service.setAttribute(GSXML.NAME_ATT, GATE_TAG_SERVICE);
+	if (subset==null || subset.equals(GSXML.DISPLAY_TEXT_ELEM)) {
+	    tag_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service+".name", lang)));
+	    tag_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service+".description", lang)));
+	    tag_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service+".submit", lang)));
+	} 
+	if (subset==null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    createParameter(ANNOTATION_TYPE_PARAM, param_list, lang);
+	    tag_service.appendChild(param_list);
+	}
+	return tag_service;
+    }
+
+
+    /** creates a new param element and adds it to the param list */
+    protected void createParameter(String name, Element param_list, 
+				   String lang)
+    {
+	Element param = null;
+
+	if (name.equals(ANNOTATION_TYPE_PARAM)) {
+	    int len = this.annotation_types.length;
+	    String[] annotation_type_names = new String[len];
+	    for (int i = 0; i < len; i++) {
+		annotation_type_names[i] = getTextString("param." + name + "." + this.annotation_types[i], lang);
+	    }
+	    
+	    param = GSXML.createParameterDescription(this.doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_ENUM_MULTI, this.annotation_types[0], this.annotation_types, annotation_type_names);
+	    param_list.appendChild(param);
+	}
+	
+    }
+    
+    
+    protected Element processGateTag(Element request)
+    {
+	// System.out.println("(GateTag) Request:\n" + converter_.getPrettyString(request));
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, GATE_TAG_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("Error: GateTag request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	Set annTypes = new HashSet();
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the annotation types desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("annotationType")) {
+		String annotation_type = GSXML.getValue(param);
+		String [] types = annotation_type.split(",");
+		for (int i=0; i<types.length; i++) {
+		    annTypes.add(types[i]);
+		}
+	    }
+
+	    param = (Element) param.getNextSibling();
+	}
+
+	// Get the request content 
+      	Element doc_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_node_list == null) {
+	    System.err.println("Error: GateTag request specified no doc nodes.");
+	    return result;  // Return the empty result
+	}
+
+	// Process each document node in the list
+	NodeList doc_nodes = doc_node_list.getChildNodes();
+	for (int i = 0; i < doc_nodes.getLength(); i++) {
+	    Element doc_node = (Element) doc_nodes.item(i);
+	    Element content = (Element) GSXML.getChildByTagName(doc_node, "nodeContent");
+	    Node content_text = (Node) GSXML.getNodeTextNode(content);
+	    String text = content_text.getNodeValue();
+
+	    // GATE needs the text to be a valid HTML file
+	    text = "<html><head></head><body>" + text + "</body></html>";
+	    String annotated_text = processText(text, annTypes);
+
+	    // Remove the surrounding HTML tags
+	    annotated_text = annotated_text.substring(49, annotated_text.length() - 13);
+	    // System.out.println("GATE result:\n" + annotated_text);
+
+	    annotated_text = "<nodeContent>" + annotated_text + "</nodeContent>";
+	    Document annotated_content_doc = this.converter.getDOM(annotated_text);
+	    if (annotated_content_doc != null) {
+		Element annotated_content = annotated_content_doc.getDocumentElement();
+		doc_node.replaceChild(doc_node.getOwnerDocument().importNode(annotated_content, true), content);
+	    } else {
+		System.err.println("GATEServices.processGateTag Error: Couldn't parse annotated text for doc node "+i);
+	    }
+	    
+	}
+
+	result.appendChild(this.doc.importNode(doc_node_list, true));
+	// System.out.println("GateTag result:\n" + converter_.getPrettyString(result));
+	return result;
+    }
+
+
+    public String processText(String text, Set annotationTypesToExport)
+    {
+	try {
+	    // Create a new document containing the text
+	    FeatureMap parameters = Factory.newFeatureMap();
+	    parameters.put("stringContent", text);
+	    parameters.put("markupAware", new Boolean(true));
+	    parameters.put("preserveOriginalContent", new Boolean(true));
+	    parameters.put("collectRepositioningInfo", new Boolean(true));
+	    Document doc = (Document) Factory.createResource("gate.corpora.DocumentImpl",
+							     parameters);
+
+	    // Add it to the corpus
+	    this.corpus.clear();
+	    this.corpus.add(doc);
+
+	    // Process the corpus
+	    this.application.execute();
+
+	    // Extract all the annotations
+	    AnnotationSet annSet = doc.getAnnotations();
+
+	    // Return the desired annotations
+	    AnnotationSet outputAnnotations = annSet.get(annotationTypesToExport);
+	    String result = doc.toXml(outputAnnotations, false);
+	    Factory.deleteResource(doc);
+
+	    // 1. Escape the GATE result
+	    result = GSXML.xmlSafe(result);
+
+	    // 2. Unescape the annotation tags
+	    Iterator setIterator = annotationTypesToExport.iterator();
+	    while (setIterator.hasNext()) {
+		String annotationType = (String) setIterator.next();
+		result = result.replaceAll("&lt;" + annotationType + "&gt;",
+					   "<annotation type=\"" + annotationType + "\">");
+		result = result.replaceAll("&lt;/" + annotationType + "&gt;",
+					   "</annotation>");
+	    }
+
+	    return result;
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	    return null;
+	}
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/build.properties
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/build.properties	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/build.properties	(revision 14150)
@@ -0,0 +1,27 @@
+#### initial properties file with all gs3build bits from main greenstone3 system
+## java compilation properties - for javac
+#Should compilation include the debug option?
+compile.debug=true
+#Should compilation include the deprecation option?
+compile.deprecation=true
+#Should compilation include the optimize option?
+compile.optimize=true
+
+ ## Relational database 
+# set this to be non-empty if you are using an external mysql.
+mysql.installed.path=
+#change the following if the defaults are not correct
+mysql.server=localhost
+#port mysql should run on. Change if you already have mysql running on the default port
+mysql.port=3306
+#Should passwords be set for the root and greenstone mysql users?
+mysql.use.passwords=yes
+## If you are installing greenstone (including mysql) in an nfs directory, mysql may not work. The data directory needs to be on a local drive: Copy the greenstone3/packages/mysql/data directory to a local drive and set the location here
+#mysql.datadir=
+## Mysql user passwords: uncomment and set the following if you don't want to be prompted for them during installation
+#The root password for a local mysql installation
+#mysql.root.password=
+# The password for the gsdl3admin user (local and external mysql)
+#mysql.admin.password=
+# The password for the gsdl3reader user (local and external mysql)
+#mysql.reader.password=
Index: /other-projects/trunk/greenstone3-extension/gs3build/build.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/build.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/build.xml	(revision 14150)
@@ -0,0 +1,360 @@
+<?xml version="1.0"?>
+
+<!-- ====================================================================== 
+     July 2006                                                   
+
+     Greenstone3 building build and install script
+              
+     So far I have just moved the gs3build related targets out of main 
+     greenstone 3 build.xml and put them in here. It needs tidying up to 
+     make it useful.     
+     kjdon                                                                
+     ====================================================================== -->
+<project name="gs3build" default="usage" basedir=".">
+
+  <!-- ============= Base dirs for each package and component ============ -->
+  <property name="mysql.home" value="${packages.home}/mysql"/>
+  
+  <!-- can be set in build.properties if need be -->
+  <property name="mysql.datadir" value="./data"/>
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="true"/>
+  <property name="compile.optimize"    value="true"/>
+
+
+<target name="prepare" depends="prepare-mysql"/>
+
+  <!-- this sets up some initial properties -->
+  <target name="init">
+    <condition property="java.too.old">
+      <or>
+	<equals arg1="1.1" arg2="${ant.java.version}"/>
+	<equals arg1="1.2" arg2="${ant.java.version}"/>
+	<equals arg1="1.3" arg2="${ant.java.version}"/>
+      </or>
+    </condition>
+    <fail if="java.too.old" message="You need Java 1.4 or greater to run Greenstone 3"/>
+    <available file="${packages.home}/mysql/bin" property="mysql.present"/>
+    <condition property="mysql.islocal">
+      <or>
+	<not>
+	  <isset property="mysql.installed.path"/>
+	</not>
+	<equals arg1="" arg2="${mysql.installed.path}"/>
+      </or>
+    </condition>
+  </target>
+
+  <target name="prepare-collections" depends="init">
+    <property name="collect.dir" value="${web.home}/sites/localsite/collect"/>
+    <!-- gs3mgdemo -->
+    <unzip src="${collect.dir}/gs3mgdemo/import.zip" 
+      dest="${collect.dir}/gs3mgdemo"/>
+    <unzip src="${collect.dir}/gs3mgdemo/archives.zip" 
+      dest="${collect.dir}/gs3mgdemo"/>
+    <unzip src="${collect.dir}/gs3mgdemo/index/index.zip" 
+      dest="${collect.dir}/gs3mgdemo/index"/>
+    <delete file="${collect.dir}/gs3mgdemo/import.zip"/>
+    <delete file="${collect.dir}/gs3mgdemo/archives.zip"/>
+    <delete file="${collect.dir}/gs3mgdemo/index/index.zip"/>
+  </target>
+
+  <target name="configure-web" depends="init,get-mysql-reader-password,get-mysql-admin-password"
+
+        <!-- just in case these haven't been defined, we need something to write to
+    the file -->
+    <property name="mysql.admin.password" value=" "/>
+    <property name="mysql.reader.password" value=" "/>
+    <filter token="mysql.port" value="${mysql.port}"/>
+    <filter token="mysql.server" value="${mysql.server}"/>
+    <filter token="mysql.admin.pword" value="${mysql.admin.password}"/>
+    <filter token="mysql.reader.pword" value="${mysql.reader.password}"/>
+  </target>
+
+<!-- ======================= Mysql Targets =========================== -->
+
+  <!-- this is one target that we only want to run once. -->
+  <target name="prepare-mysql" depends="init,init-mysql-vars,install-mysql,start-mysql,setup-mysql,stop-mysql-no-password-force"/>
+
+  <target name="init-mysql-vars">
+    <condition property="mysql.islocal.linux">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="current.os.isunixnotmac"/>
+      </and>
+    </condition>
+    <condition property="mysql.islocal.mac">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="current.os.ismac"/>
+      </and>
+    </condition>	   
+    <condition property="mysql.islocal.windows">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="current.os.iswindows"/>
+      </and>
+    </condition>
+    <condition property="mysql.islocal.usepassword">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="mysql.use.passwords"/>
+	<equals arg1="yes" arg2="${mysql.use.passwords}"/>
+      </and>
+    </condition>
+    <condition property="mysql.islocal.usepassword.windows">
+      <and>
+	<isset property="mysql.islocal.usepassword"/>
+	<isset property="current.os.iswindows"/>
+      </and>
+    </condition>
+     <condition property="mysql.islocal.usepassword.unix">
+      <and>
+	<isset property="mysql.islocal.usepassword"/>
+	<isset property="current.os.isunix"/>
+      </and>
+    </condition>
+   <condition property="mysql.islocal.nopassword">
+      <and>
+	<isset property="mysql.islocal"/>
+	<or>
+	  <not>
+	    <isset property="mysql.use.passwords"/>
+	  </not>
+	  <equals arg1="no" arg2="${mysql.use.passwords}"/>
+	</or>
+      </and>
+    </condition>
+   </target>
+ 
+  <target name="configure-mysql" depends="init,init-mysql-vars,start-mysql,set-mysql-account-passwords,stop-mysql" description="sets up the mysql account passwords"/>
+  
+  <target name="get-mysql-root-password" depends="init,init-mysql-vars" if="mysql.islocal.usepassword" unless="mysql.root.password">
+    <echo>Enter password in the Java popup. (If you do not have a window server running, please set the mysql.root.password in the build.properties file.)</echo>
+    <getuserandpassword message="Please specify a password for the root mysql user: this is to secure your database."  username="root" pwordproperty="mysql.root.password"/>
+  </target>
+  
+  <target name="get-mysql-root-password-windows" depends="init,init-mysql-vars" if="mysql.islocal.usepassword.windows" unless="mysql.root.password">
+    <echo>(Enter password in the Java popup)</echo>
+    <getuserandpassword message="Please enter the mysql root password"  username="root" pwordproperty="mysql.root.password"/>
+  </target>
+  
+  <target name="get-mysql-reader-password" depends="init,init-mysql-vars" if="mysql.islocal.usepassword" unless="mysql.reader.password" >
+    <echo>Enter password in the Java popup. (If you do not have a window server running, please set the mysql.reader.password in the build.properties file.)</echo>
+    <getuserandpassword message="Please specify the password for the gsdl3reader mysql user: this is used by greenstone" username="gsdl3reader" pwordproperty="mysql.reader.password"/>
+  </target>
+  
+  <target name="get-mysql-admin-password" depends="init,init-mysql-vars" if="mysql.islocal.usepassword" unless="mysql.admin.password" >
+    <echo>Enter password in the Java popup. (If you do not have a window server running, please set the mysql.admin.password in the build.properties file.)</echo>
+    <getuserandpassword message="Please specify the password for the gsdl3admin mysql user: this is used by greenstone" username="gsdl3admin" pwordproperty="mysql.admin.password"/>
+  </target>
+
+ <target name="install-mysql" depends="init,init-mysql-vars,install-mysql-linux,install-mysql-windows,install-mysql-mac"/>
+  
+  <!-- install the database : linux-->
+  <target name="install-mysql-linux" depends="init,init-mysql-vars" if="mysql.islocal.linux" unless="mysql.present">
+    <get src="http://www.greenstone.org/gs3files/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"
+      dest="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"
+      usetimestamp="true"/>
+    <untar src="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"
+      dest="${packages.home}"
+      compression="gzip"/>
+    <move todir="${packages.home}/mysql">
+      <fileset dir="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686"/>
+    </move>
+    <!-- file permissions are screwed up, so make executables executable -->
+    <chmod file="${mysql.home}/scripts/mysql_install_db" perm="a+x"/>
+    <chmod perm="a+x">
+      <fileset dir="${mysql.home}/bin" />
+    </chmod>
+    <mkdir dir="${mysql.home}/var/log/"/>
+    <chmod file="${mysql.home}/var/" perm="777" type="dir"/>
+    <chmod file="${mysql.home}/var/log/" perm="777" type="dir"/>
+    
+    <exec executable="${mysql.home}/scripts/mysql_install_db" dir="${mysql.home}">
+      <arg value="--datadir=${mysql.datadir}"/>
+      <arg value="--basedir=."/>
+      <arg value="--user=root"/>
+      <arg value="--force"/>
+    </exec>
+  </target>
+
+  <!-- install the database : mac-->
+  <target name="install-mysql-mac" depends="init,init-mysql-vars" if="mysql.islocal.mac" unless="mysql.present">
+    <get src="http://www.greenstone.org/gs3files/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"
+      dest="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"
+      usetimestamp="true"/>
+    <untar src="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"
+      dest="${packages.home}"
+      compression="gzip"/>
+    <move todir="${packages.home}/mysql">
+      <fileset dir="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc"/>
+    </move>    
+    <!-- file permissions are screwed up, so make executables executable -->
+    <chmod file="${mysql.home}/scripts/mysql_install_db" perm="a+x"/>
+    <chmod perm="a+x">
+      <fileset dir="${mysql.home}/bin" />
+    </chmod>
+    <mkdir dir="${mysql.home}/var/log/"/>
+    <chmod file="${mysql.home}/var/" perm="777" type="dir"/>
+    <chmod file="${mysql.home}/var/log/" perm="777" type="dir"/>
+    <exec executable="${mysql.home}/scripts/mysql_install_db" dir="${mysql.home}">
+      <arg value="--datadir=${mysql.datadir}"/>
+      <arg value="--basedir=."/>
+      <!--<arg value="- -user=root"/>
+      <arg value="- -force"/>-->
+    </exec>
+  </target>
+
+  <!-- install the database : windows -->
+  <target name="install-mysql-windows" depends="init,init-mysql-vars" if="mysql.islocal.windows" unless="mysql.present">
+    <get src="http://www.greenstone.org/gs3files/mysql-noinstall-4.1.15-win32.zip"
+      dest="${packages.home}/mysql-noinstall-4.1.15-win32.zip"
+      usetimestamp="true"/>
+    <!-- can we run the installer?? -->
+    <unzip src="${packages.home}/mysql-noinstall-4.1.15-win32.zip"
+      dest="${packages.home}"/>
+    <move todir="${packages.home}/mysql">
+      <fileset dir="${packages.home}/mysql-4.1.15-win32"/>
+    </move>
+  </target>
+
+  <target name="setup-mysql" depends="init" if="mysql.islocal">
+    <!-- setup the gsdl3 accounts -->
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=GRANT SELECT,INSERT,DELETE,UPDATE,DROP,CREATE ON *.* TO gsdl3admin@localhost"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=GRANT SELECT ON *.* TO gsdl3reader@localhost"/>
+    </exec>
+    <!--<exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="- -user=root"/>
+      <arg value="- -execute=GRANT SELECT,INSERT,DELETE,UPDATE,DROP,CREATE ON *.* TO gsdl3admin@'%';"/>
+    </exec>-->
+
+    <!-- remove the anonymous user -->
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=delete from mysql.user where Host='localhost' and User='';"/>
+    </exec>
+    <!-- load in the demo collection database -->
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=create database localsite_gs3mgdemo;"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}"
+      input="${web.home}/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql">
+      <arg value="--user=root"/>
+      <arg value="localsite_gs3mgdemo"/>
+    </exec>
+  </target>
+
+  <target name="set-mysql-account-passwords" depends="init,init-mysql-vars,get-mysql-root-password,get-mysql-reader-password,get-mysql-admin-password" if="mysql.islocal.usepassword">
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=set password for 'root'@'localhost' = PASSWORD('${mysql.root.password}'); set password for 'gsdl3reader'@'localhost' = PASSWORD('${mysql.reader.password}'); set password for 'gsdl3admin'@'localhost' = PASSWORD('${mysql.admin.password}');  flush privileges;"/>
+    </exec>    
+  </target>
+
+  <target name="start-mysql" depends="init,init-mysql-vars" if="mysql.islocal"
+    description="Startup only mysql">
+    <echo>Starting up the mysql server</echo>
+    <exec executable="${mysql.home}/bin/mysqld_safe" dir="${mysql.home}"
+      spawn="true" os="${os.linux},${os.mac}">
+      <arg value="--datadir=${mysql.datadir}"/>
+      <arg value="--basedir=."/>
+      <arg value="--pid_file=gsdl3.pid"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="--port=${mysql.port}"/>
+      <arg value="--err-log=./var/log/mysql.log"/>
+    </exec>
+   <exec executable="${mysql.home}/bin/mysqld" dir="${mysql.home}" spawn="true" os="${os.windows}">
+      <arg value="--port=${mysql.port}"/>   
+    </exec>
+    <sleep seconds="2"/>
+  </target>
+
+  <target name="stop-mysql" depends="init,init-mysql-vars,stop-mysql-use-password-unix,stop-mysql-use-password-windows,stop-mysql-no-password" if="mysql.islocal"
+    description="Shutdown only mysql">
+    <echo>MYSQL database server shutdown successfully</echo>
+  </target>
+  
+  <target name="stop-mysql-no-password" depends="init" if="mysql.islocal.nopassword">
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.linux},${os.mac}">
+      <arg value="--user=root"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="shutdown"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.windows}">
+      <arg value="--user=root"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+  
+  <target name="stop-mysql-no-password-force" depends="init" if="mysql.islocal">
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.linux},${os.mac}">
+      <arg value="--user=root"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="shutdown"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.windows}">
+      <arg value="--user=root"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+  
+  <target name="stop-mysql-use-password-unix" depends="init" if="mysql.islocal.usepassword.unix">
+    <echo>At the password prompt, enter the mysql root password.</echo>
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.linux},${os.mac}">
+      <arg value="--user=root"/>
+      <arg value="-p"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+  
+  <target name="stop-mysql-use-password-windows" depends="init,get-mysql-root-password-windows" if="mysql.islocal.usepassword.windows">
+    <!-- on windows it doesn't work unless the password is specified here -->
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.windows}">
+      <arg value="--user=root"/>
+      <arg value="-p${mysql.root.password}"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+
+  <target name="restart-mysql" description="Shutdown and restart only mysql" depends="init,stop-mysql,start-mysql"/>
+
+  <target name="prepare-for-dist-2" depends="init">
+    <antcall target="prepare-mysql"/>
+    <delete file="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"/>
+    <delete file="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"/>
+    <delete file="${packages.home}/mysql-noinstall-4.1.15-win32.zip"/>
+    <delete dir="${packages.home}/mysql/sql-bench"/>
+    <delete dir="${packages.home}/mysql/tests"/>
+    <delete dir="${packages.home}/mysql/mysql-test"/>
+    <delete file="${packages.home}/mysql/var/log/mysql.log"/>
+  </target>
Index: /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/collectionConfig.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/collectionConfig.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/collectionConfig.xml	(revision 14150)
@@ -0,0 +1,83 @@
+<collectionConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">
+  <metadataList>
+    <metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+    <metadata name="maintainer">greenstone@cs.waikato.ac.nz</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Greenstone3 MG demo collection</displayItem>
+    <displayItem name='description' lang='en'>This is a demonstration collection for the Greenstone3 digital library software. It contains a small subset (11 books) of the Humanities and Development Libraries. It is built with mg, using the new native Greenstone3 building software.</displayItem>
+    <displayItem name='icon' lang='en'>gs3mgdemo.gif</displayItem>
+    <displayItem name='smallicon' lang='en'>gs3mgdemo_sm.gif</displayItem>
+  </displayItemList>
+  <recognise>
+      <docType name="HTML"/>
+      <docType name="Text"/>
+      <docType name="Metadata"/>
+      <docType name="JPEG"/>
+  </recognise>
+		    
+  <search type="mg" name="mgsearch">
+    <index name="sectext">
+      <field>text</field>
+      <level>section</level>
+      <displayItem name='name' lang="en">chapters</displayItem>
+      <displayItem name='name' lang="fr">chapitres</displayItem>
+      <displayItem name='name' lang="es">capÃ­tulos</displayItem>
+    </index>
+    <index name="doctext">
+      <field>text</field>
+      <level>document</level>
+      <displayItem name='name' lang="en">entire documents</displayItem>
+      <displayItem name='name' lang="fr">documents entiers</displayItem>
+      <displayItem name='name' lang="es">documentos enteros</displayItem>
+    </index>
+    <index name="sectitle">
+      <field>Title</field>
+      <level>section</level>
+      <displayItem name='name' lang="en">section titles</displayItem>
+      <displayItem name='name' lang="fr">titres des sections</displayItem>
+      <displayItem name='name' lang="es">tÃ­tulos de las secciones</displayItem>
+    </index>
+    <format>
+      <gsf:template match="documentNode"><td valign='top'><gsf:link><gsf:icon/></gsf:link></td><td><gsf:metadata name='Title'/></td></gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name="CLTit" type="AZList" horizontalAtTop='true'>
+      <field>Title</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Titles</displayItem>
+    </classifier>
+    <classifier name="CLOrg" type="Hierarchy">
+      <field>Organization</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Organizations</displayItem>
+      <!-- use an absolute URL or one relative to etc directory -->
+      <file URL="org.xml"/>
+    </classifier>
+    <classifier name="CLSubject" type="Hierarchy">
+      <field>Subject</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Subjects</displayItem>
+      <file URL="sub.xml"/>
+    </classifier>
+    <classifier name="CLKeyword" type="Hierarchy">
+      <field>Keyword</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>HowTo</displayItem>
+      <file URL="keyword.xml"/>
+      <!--<format>
+        <gsf:template match="documentNode">
+          <br /><gsf:link><gsf:metadata name='Keyword' />
+            </gsf:link></gsf:template>
+      </format>-->
+    </classifier>
+  </browse>
+  <display>
+    <format>
+      <gsf:option name="coverImages" value="false"/>
+      <!--<gsf:option name="documentTOC" value="false"/>-->
+    </format>
+  </display>
+</collectionConfig>
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/keyword.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/keyword.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/keyword.xml	(revision 14150)
@@ -0,0 +1,32 @@
+<Hierarchy>
+  <Classification>
+    <Name>introduce small animals and micro-livestock in your farm</Name>
+    <Path>1</Path>
+    <Description>introduce small animals and micro-livestock in your farm</Description>
+  </Classification>
+  <Classification>
+    <Name>introduce little-known Asian farm animals with a promising future</Name>
+    <Path>2</Path>
+    <Description>introduce little-known Asian farm animals with a promising future</Description>
+  </Classification>
+  <Classification>
+    <Name>utilize the Water Buffalo more effectively</Name>
+    <Path>3</Path>
+    <Description>utilize the Water Buffalo more effectively</Description>
+  </Classification>
+  <Classification>
+    <Name>start a butterfly farm</Name>
+    <Path>4</Path>
+    <Description>start a butterfly farm</Description>
+  </Classification>
+  <Classification>
+    <Name>farm snails</Name>
+    <Path>5</Path>
+    <Description>farm snails</Description>
+  </Classification>
+  <Classification>
+    <Name>achieve gender equality</Name>
+    <Path>6</Path>
+    <Description>achieve gender equality</Description>
+  </Classification>
+</Hierarchy>
Index: /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/org.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/org.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/org.xml	(revision 14150)
@@ -0,0 +1,84 @@
+<Hierarchy>
+<Classification>
+  <Name>ACCU</Name>
+  <Path>1</Path>
+  <Description>ACCU</Description>
+</Classification>
+<Classification>
+  <Name>Agenda 21</Name>
+  <Path>2</Path>
+  <Description>Agenda 21</Description>
+</Classification>
+<Classification>
+  <Name>Alkire</Name>
+  <Path>3</Path>
+  <Description>Alkire</Description>
+</Classification>
+<Classification>
+  <Name>BOSTID</Name>
+  <Path>4</Path>
+  <Description>BOSTID</Description>
+</Classification>
+<Classification>
+  <Name>Chandra</Name>
+  <Path>5</Path>
+  <Description>Chandra</Description>
+</Classification>
+<Classification>
+  <Name>CPS</Name>
+  <Path>6</Path>
+  <Description>CPS</Description>
+</Classification>
+<Classification>
+  <Name>EC Courier</Name>
+  <Path>7</Path>
+  <Description>EC Courier</Description>
+</Classification>
+<Classification>
+  <Name>FAO</Name>
+  <Path>8</Path>
+  <Description>FAO</Description>
+  <Children>
+    <Classification>
+      <Name>FAO Better Farming series</Name>
+      <Path>8.1</Path>
+      <Description>FAO Better Farming Series</Description>
+    </Classification>
+  </Children>
+</Classification>
+<Classification>
+  <Name>Habitat</Name>
+  <Path>9</Path>
+  <Description>Habitat</Description>
+</Classification>
+<Classification>
+  <Name>Inforse</Name>
+  <Path>10</Path>
+  <Description>Inforse</Description>
+</Classification>
+<Classification>
+  <Name>International Institute for Rural Reconstruction</Name>
+  <Path>12</Path>
+  <Description>International Institute for Rural Reconstruction</Description>
+</Classification>
+<Classification>
+  <Name>NGLS</Name>
+  <Path>13</Path>
+  <Description>NGLS</Description>
+</Classification>
+<Classification>
+  <Name>Peace Corps</Name>
+  <Path>14</Path>
+  <Description>Peace Corps</Description>
+</Classification>
+<Classification>
+  <Name>UNESCO</Name>
+  <Path>15</Path>
+  <Description>UNESCO</Description>
+</Classification>
+<Classification>
+  <Name>World Bank</Name>
+  <Path>16</Path>
+  <Description>World Bank</Description>
+</Classification>
+</Hierarchy>
Index: /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/sub.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/sub.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/etc/sub.xml	(revision 14150)
@@ -0,0 +1,94 @@
+<Hierarchy>
+  <Classification>
+    <Name>Society, Culture, Community, Woman, Youth, Population</Name>
+    <Path>10</Path>
+    <Description>Society, Culture, Community, Woman, Youth, Population</Description>
+    <Children>
+      <Classification>
+	<Name>Women, gender and development, women&apos;s organizations</Name>
+	<Path>10.5</Path>
+	<Description>Women, gender and development, women&apos;s organizations</Description>
+      </Classification>
+      <Classification>
+	<Name>Social sciences, sociology (works comprising several subgroups) incl. participatory research and training</Name>
+	<Path>10.6</Path>
+	<Description>Social sciences, sociology (works comprising several subgroups) incl. participatory research and training</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Communication, Information and Documentation</Name>
+    <Path>12</Path>
+    <Description>Communication, Information and Documentation</Description>
+    <Children>
+      <Classification>
+	<Name>Communication, telecommunication, mass communication, mass media, film-making</Name>
+	<Path>12.2</Path>
+	<Description>Communication, telecommunication, mass communication, mass media, film-making</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Agriculture and Food Processing</Name>
+    <Path>13</Path>
+    <Description>Agriculture and Food Processing</Description>
+    <Children>
+      <Classification>
+	<Name>Better Farming series of FAO and INADES - 46 booklets</Name>
+	<Path>13.8</Path>
+	<Description>Better Farming series of FAO and INADES - 46 booklets</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Animal Husbandry and Animal Product Processing</Name>
+    <Path>14</Path>
+    <Description>Animal Husbandry and Animal Product Processing</Description>
+    <Children>
+      <Classification>
+	<Name>Veterinary medicine, animal diseases</Name>
+	<Path>14.2</Path>
+	<Description>Veterinary medicine, animal diseases</Description>
+      </Classification>
+      <Classification>
+	<Name>Cattle</Name>
+	<Path>14.5</Path>
+	<Description>Cattle</Description>
+      </Classification>
+      <Classification>
+	<Name>Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)</Name>
+	<Path>14.6</Path>
+	<Description>Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Settlements, Housing, Building - Infrastructure Construction (Roads etc)</Name>
+    <Path>15</Path>
+    <Description>Settlements, Housing, Building - Infrastructure Construction (Roads etc)</Description>
+    <Children>
+      <Classification>
+	<Name>Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.</Name>
+	<Path>15.3</Path>
+	<Description>Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Development Periodicals and Magazines</Name>
+    <Path>16</Path>
+    <Description>Development Periodicals and Magazines</Description>
+    <Children>
+      <Classification>
+	<Name>Sustainable Energy Newsletters of the INFORSE network</Name>
+	<Path>16.1</Path>
+	<Description>Sustainable Energy Newsletters of the INFORSE network</Description>
+      </Classification>
+      <Classification>
+	<Name>The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union</Name>
+	<Path>16.2</Path>
+	<Description>The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union</Description>
+      </Classification>
+    </Children>
+  </Classification>
+</Hierarchy>
Index: /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/index/buildConfig.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/index/buildConfig.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/index/buildConfig.xml	(revision 14150)
@@ -0,0 +1,1 @@
+<buildConfig><metadataList /><serviceRackList><serviceRack name="GS3Browse"><classifierList><classifier name="CLTit" /><classifier name="CLOrg" /><classifier name="CLSubject" /><classifier name="CLKeyword" /></classifierList></serviceRack><serviceRack name="GS3MGSearch"><indexStem name="index"/><indexList><index name="sectext" /><index name="doctext" /><index name="sectitle" /></indexList><defaultIndex name="sectext" /><baseIndexPrefix name="mgsearch" /></serviceRack><serviceRack name="GS3MGRetrieve"><defaultIndex name="sectext" /><indexStem name="index"/><baseIndexPrefix name="mgsearch" /></serviceRack></serviceRackList></buildConfig>
Index: /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/mysqldatadump.sql
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/mysqldatadump.sql	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/collect/gs3mgdemo/mysqldatadump.sql	(revision 14150)
@@ -0,0 +1,2812 @@
+-- MySQL dump 9.11
+--
+-- Host: localhost    Database: localsite_gs3mgdemo
+-- ------------------------------------------------------
+-- Server version	4.0.20-standard
+
+--
+-- Table structure for table `build`
+--
+
+CREATE TABLE build (
+  buildRef int(11) NOT NULL auto_increment,
+  buildKey varchar(64) default NULL,
+  buildValue tinytext,
+  PRIMARY KEY  (buildRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `build`
+--
+
+INSERT INTO build VALUES (1,'NextSeqNo','7');
+INSERT INTO build VALUES (2,'lastBuildDate','20050201');
+
+--
+-- Table structure for table `classdata`
+--
+
+CREATE TABLE classdata (
+  ClassifyRef int(11) default NULL,
+  Label tinytext,
+  Value tinytext
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `classdata`
+--
+
+
+--
+-- Table structure for table `classdocuments`
+--
+
+CREATE TABLE classdocuments (
+  ClassifyRef int(11) default NULL,
+  DocID tinytext,
+  DocOrder int(11) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `classdocuments`
+--
+
+INSERT INTO classdocuments VALUES (2,'20050201:5',1);
+INSERT INTO classdocuments VALUES (3,'20050201:2',1);
+INSERT INTO classdocuments VALUES (4,'20050201:3',1);
+INSERT INTO classdocuments VALUES (4,'20050201:4',2);
+INSERT INTO classdocuments VALUES (5,'20050201:1',1);
+INSERT INTO classdocuments VALUES (10,'20050201:1',1);
+INSERT INTO classdocuments VALUES (10,'20050201:2',2);
+INSERT INTO classdocuments VALUES (15,'20050201:3',1);
+INSERT INTO classdocuments VALUES (15,'20050201:4',2);
+INSERT INTO classdocuments VALUES (22,'20050201:5',1);
+INSERT INTO classdocuments VALUES (25,'20050201:5',1);
+INSERT INTO classdocuments VALUES (30,'20050201:3',1);
+INSERT INTO classdocuments VALUES (30,'20050201:4',2);
+INSERT INTO classdocuments VALUES (34,'20050201:1',1);
+INSERT INTO classdocuments VALUES (34,'20050201:2',2);
+INSERT INTO classdocuments VALUES (36,'20050201:3',1);
+INSERT INTO classdocuments VALUES (36,'20050201:4',2);
+INSERT INTO classdocuments VALUES (41,'20050201:2',1);
+INSERT INTO classdocuments VALUES (42,'20050201:1',1);
+INSERT INTO classdocuments VALUES (45,'20050201:4',1);
+INSERT INTO classdocuments VALUES (46,'20050201:5',1);
+
+--
+-- Table structure for table `classifiers`
+--
+
+CREATE TABLE classifiers (
+  ClassifyRef int(11) NOT NULL auto_increment,
+  ClassifyID tinytext,
+  ParentID tinytext,
+  ClassifyOrder tinytext,
+  Name tinytext,
+  Description tinytext,
+  NumLeafDocs int(11) default NULL,
+  PRIMARY KEY  (ClassifyRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `classifiers`
+--
+
+INSERT INTO classifiers VALUES (1,'CLTit','','0','','Classifier',5);
+INSERT INTO classifiers VALUES (2,'CLTit.d','CLTit','1','d','d',1);
+INSERT INTO classifiers VALUES (3,'CLTit.m','CLTit','2','m','m',1);
+INSERT INTO classifiers VALUES (4,'CLTit.f','CLTit','3','f','f',2);
+INSERT INTO classifiers VALUES (5,'CLTit.l','CLTit','4','l','l',1);
+INSERT INTO classifiers VALUES (6,'CLOrg','','0','','Classifier',5);
+INSERT INTO classifiers VALUES (7,'CLOrg.1','CLOrg','1','ACCU','ACCU',0);
+INSERT INTO classifiers VALUES (8,'CLOrg.2','CLOrg','2','Agenda 21','Agenda 21',0);
+INSERT INTO classifiers VALUES (9,'CLOrg.3','CLOrg','3','Alkire','Alkire',0);
+INSERT INTO classifiers VALUES (10,'CLOrg.4','CLOrg','4','BOSTID','BOSTID',2);
+INSERT INTO classifiers VALUES (11,'CLOrg.5','CLOrg','5','Chandra','Chandra',0);
+INSERT INTO classifiers VALUES (12,'CLOrg.6','CLOrg','6','CPS','CPS',0);
+INSERT INTO classifiers VALUES (13,'CLOrg.7','CLOrg','7','EC Courier','EC Courier',0);
+INSERT INTO classifiers VALUES (14,'CLOrg.8','CLOrg','8','FAO','FAO',2);
+INSERT INTO classifiers VALUES (15,'CLOrg.8.1','CLOrg.8','1','FAO Better Farming series','FAO Better Farming Series',2);
+INSERT INTO classifiers VALUES (16,'CLOrg.9','CLOrg','9','Habitat','Habitat',0);
+INSERT INTO classifiers VALUES (17,'CLOrg.10','CLOrg','10','Inforse','Inforse',0);
+INSERT INTO classifiers VALUES (18,'CLOrg.12','CLOrg','12','International Institute for Rural Reconstruction','International Institute for Rural Reconstruction',0);
+INSERT INTO classifiers VALUES (19,'CLOrg.13','CLOrg','13','NGLS','NGLS',0);
+INSERT INTO classifiers VALUES (20,'CLOrg.14','CLOrg','14','Peace Corps','Peace Corps',0);
+INSERT INTO classifiers VALUES (21,'CLOrg.15','CLOrg','15','UNESCO','UNESCO',0);
+INSERT INTO classifiers VALUES (22,'CLOrg.16','CLOrg','16','World Bank','World Bank',1);
+INSERT INTO classifiers VALUES (23,'CLSubject','','0','','Classifier',7);
+INSERT INTO classifiers VALUES (24,'CLSubject.10','CLSubject','10','Society, Culture, Community, Woman, Youth, Population','Society, Culture, Community, Woman, Youth, Population',1);
+INSERT INTO classifiers VALUES (25,'CLSubject.10.5','CLSubject.10','5','Women, gender and development, women\'s organizations','Women, gender and development, women\'s organizations',1);
+INSERT INTO classifiers VALUES (26,'CLSubject.10.6','CLSubject.10','6','Social sciences, sociology (works comprising several subgroups) incl. participatory research and training','Social sciences, sociology (works comprising several subgroups) incl. participatory research and training',0);
+INSERT INTO classifiers VALUES (27,'CLSubject.12','CLSubject','12','Communication, Information and Documentation','Communication, Information and Documentation',0);
+INSERT INTO classifiers VALUES (28,'CLSubject.12.2','CLSubject.12','2','Communication, telecommunication, mass communication, mass media, film-making','Communication, telecommunication, mass communication, mass media, film-making',0);
+INSERT INTO classifiers VALUES (29,'CLSubject.13','CLSubject','13','Agriculture and Food Processing','Agriculture and Food Processing',2);
+INSERT INTO classifiers VALUES (30,'CLSubject.13.8','CLSubject.13','8','Better Farming series of FAO and INADES - 46 booklets','Better Farming series of FAO and INADES - 46 booklets',2);
+INSERT INTO classifiers VALUES (31,'CLSubject.14','CLSubject','14','Animal Husbandry and Animal Product Processing','Animal Husbandry and Animal Product Processing',2);
+INSERT INTO classifiers VALUES (32,'CLSubject.14.2','CLSubject.14','2','Veterinary medicine, animal diseases','Veterinary medicine, animal diseases',0);
+INSERT INTO classifiers VALUES (33,'CLSubject.14.5','CLSubject.14','5','Cattle','Cattle',0);
+INSERT INTO classifiers VALUES (34,'CLSubject.14.6','CLSubject.14','6','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)',2);
+INSERT INTO classifiers VALUES (35,'CLSubject.15','CLSubject','15','Settlements, Housing, Building - Infrastructure Construction (Roads etc)','Settlements, Housing, Building - Infrastructure Construction (Roads etc)',2);
+INSERT INTO classifiers VALUES (36,'CLSubject.15.3','CLSubject.15','3','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.',2);
+INSERT INTO classifiers VALUES (37,'CLSubject.16','CLSubject','16','Development Periodicals and Magazines','Development Periodicals and Magazines',0);
+INSERT INTO classifiers VALUES (38,'CLSubject.16.1','CLSubject.16','1','Sustainable Energy Newsletters of the INFORSE network','Sustainable Energy Newsletters of the INFORSE network',0);
+INSERT INTO classifiers VALUES (39,'CLSubject.16.2','CLSubject.16','2','The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union','The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union',0);
+INSERT INTO classifiers VALUES (40,'CLKeyword','','0','','Classifier',4);
+INSERT INTO classifiers VALUES (41,'CLKeyword.1','CLKeyword','1','introduce small animals and micro-livestock in your farm','introduce small animals and micro-livestock in your farm',1);
+INSERT INTO classifiers VALUES (42,'CLKeyword.2','CLKeyword','2','introduce little-known Asian farm animals with a promising future','introduce little-known Asian farm animals with a promising future',1);
+INSERT INTO classifiers VALUES (43,'CLKeyword.3','CLKeyword','3','utilize the Water Buffalo more effectively','utilize the Water Buffalo more effectively',0);
+INSERT INTO classifiers VALUES (44,'CLKeyword.4','CLKeyword','4','start a butterfly farm','start a butterfly farm',0);
+INSERT INTO classifiers VALUES (45,'CLKeyword.5','CLKeyword','5','farm snails','farm snails',1);
+INSERT INTO classifiers VALUES (46,'CLKeyword.6','CLKeyword','6','achieve gender equality','achieve gender equality',1);
+
+--
+-- Table structure for table `divisionfilerefs`
+--
+
+CREATE TABLE divisionfilerefs (
+  DocID varchar(64) default NULL,
+  DivisionRef int(11) default NULL,
+  DivisionType varchar(16) default NULL,
+  FileID varchar(32) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `divisionfilerefs`
+--
+
+INSERT INTO divisionfilerefs VALUES ('20050201:1',1,'Group','FILEGROUP_PRELUDE28');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',2,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',3,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',4,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',5,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',6,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',7,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',8,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',9,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',10,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',11,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',12,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',13,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',14,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',15,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',16,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',17,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',18,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',19,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',20,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',21,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',22,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',23,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',24,'Group','FILEGROUP_PRELUDE25');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',25,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',26,'Group','FILEGROUP_PRELUDE24');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',27,'Group','FILEGROUP_PRELUDE26');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',28,'Group','FILEGROUP_PRELUDE27');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',30,'Group','FILEGROUP_PRELUDE51');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',31,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',32,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',33,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',34,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',35,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',36,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',37,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',38,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',39,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',40,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',41,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',42,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',43,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',44,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',45,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',46,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',47,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',48,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',49,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',50,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',51,'Group','FILEGROUP_PRELUDE32');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',52,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',53,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',54,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',55,'Group','FILEGROUP_PRELUDE24');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',56,'Group','FILEGROUP_PRELUDE25');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',57,'Group','FILEGROUP_PRELUDE26');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',58,'Group','FILEGROUP_PRELUDE27');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',59,'Group','FILEGROUP_PRELUDE28');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',60,'Group','FILEGROUP_PRELUDE29');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',61,'Group','FILEGROUP_PRELUDE30');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',62,'Group','FILEGROUP_PRELUDE31');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',63,'Group','FILEGROUP_PRELUDE40');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',64,'Group','FILEGROUP_PRELUDE33');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',65,'Group','FILEGROUP_PRELUDE34');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',66,'Group','FILEGROUP_PRELUDE35');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',67,'Group','FILEGROUP_PRELUDE36');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',68,'Group','FILEGROUP_PRELUDE37');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',69,'Group','FILEGROUP_PRELUDE38');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',70,'Group','FILEGROUP_PRELUDE39');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',71,'Group','FILEGROUP_PRELUDE43');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',72,'Group','FILEGROUP_PRELUDE41');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',73,'Group','FILEGROUP_PRELUDE42');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',74,'Group','FILEGROUP_PRELUDE45');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',75,'Group','FILEGROUP_PRELUDE44');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',76,'Group','FILEGROUP_PRELUDE49');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',77,'Group','FILEGROUP_PRELUDE46');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',78,'Group','FILEGROUP_PRELUDE47');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',79,'Group','FILEGROUP_PRELUDE48');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',80,'Group','FILEGROUP_PRELUDE50');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.65');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.68');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.67');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.66');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.27');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.33');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.34');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.35');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.36');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.37');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.38');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.39');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.40');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.41');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.42');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.43');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.44');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.45');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.46');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.47');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.48');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.49');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.50');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.51');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.52');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.53');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.54');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.55');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.56');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.57');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.58');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.59');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.60');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.61');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.69');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.62');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.63');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.64');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',82,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',83,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',84,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',85,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',86,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',87,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',88,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',89,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',90,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',91,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',92,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',93,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',94,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',95,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',96,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',97,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',98,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',99,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',100,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',101,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',102,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',103,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.79');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.78');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.77');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.76');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.75');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.74');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.73');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.72');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.71');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.70');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.69');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.68');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.67');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.66');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.65');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.64');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.63');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.62');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.61');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.60');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.59');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.58');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.57');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.56');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.55');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.54');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.53');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.52');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.51');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.50');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.49');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.48');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.47');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.46');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.45');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.44');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.43');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.42');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.41');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.40');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.39');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.38');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.37');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.36');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.35');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.34');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.33');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.27');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',105,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',106,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',107,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',108,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',109,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',110,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',111,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',112,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',113,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',114,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',115,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',116,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',117,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',118,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',119,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',120,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',121,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',122,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',123,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',124,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',125,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',126,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',127,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.35');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.38');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.37');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.36');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.27');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.39');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.33');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.34');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',129,'Group','FILEGROUP_PRELUDE46');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',130,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',131,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',132,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',133,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',134,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',135,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',136,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',137,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',138,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',139,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',140,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',141,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',142,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',143,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',144,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',145,'Group','FILEGROUP_PRELUDE26');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',146,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',147,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',148,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',149,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',150,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',151,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',152,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',153,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',154,'Group','FILEGROUP_PRELUDE24');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',155,'Group','FILEGROUP_PRELUDE25');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',156,'Group','FILEGROUP_PRELUDE43');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',157,'Group','FILEGROUP_PRELUDE27');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',158,'Group','FILEGROUP_PRELUDE28');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',159,'Group','FILEGROUP_PRELUDE29');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',160,'Group','FILEGROUP_PRELUDE30');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',161,'Group','FILEGROUP_PRELUDE31');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',162,'Group','FILEGROUP_PRELUDE32');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',163,'Group','FILEGROUP_PRELUDE33');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',164,'Group','FILEGROUP_PRELUDE34');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',165,'Group','FILEGROUP_PRELUDE35');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',166,'Group','FILEGROUP_PRELUDE36');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',167,'Group','FILEGROUP_PRELUDE37');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',168,'Group','FILEGROUP_PRELUDE38');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',169,'Group','FILEGROUP_PRELUDE39');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',170,'Group','FILEGROUP_PRELUDE40');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',171,'Group','FILEGROUP_PRELUDE41');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',172,'Group','FILEGROUP_PRELUDE42');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',173,'Group','FILEGROUP_PRELUDE44');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',174,'Group','FILEGROUP_PRELUDE45');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:6',176,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.80');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.27');
+
+--
+-- Table structure for table `divisionmetarefs`
+--
+
+CREATE TABLE divisionmetarefs (
+  DocID varchar(64) default NULL,
+  DivisionRef int(11) default NULL,
+  DivisionType varchar(16) default NULL,
+  MetaID varchar(64) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `divisionmetarefs`
+--
+
+INSERT INTO divisionmetarefs VALUES ('20050201:1',1,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',2,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',3,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',4,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',5,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',6,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',7,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',8,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',9,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',10,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',11,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',12,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',13,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',14,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',15,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',16,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',17,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',18,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',19,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',20,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',21,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',22,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',23,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',24,'Group','DM25');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',25,'Group','DM26');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',26,'Group','DM27');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',27,'Group','DM28');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',28,'Group','DM29');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',29,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',30,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',31,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',32,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',33,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',34,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',35,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',36,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',37,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',38,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',39,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',40,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',41,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',42,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',43,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',44,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',45,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',46,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',47,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',48,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',49,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',50,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',51,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',52,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',53,'Group','DM25');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',54,'Group','DM26');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',55,'Group','DM27');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',56,'Group','DM28');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',57,'Group','DM29');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',58,'Group','DM30');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',59,'Group','DM31');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',60,'Group','DM32');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',61,'Group','DM33');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',62,'Group','DM34');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',63,'Group','DM35');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',64,'Group','DM36');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',65,'Group','DM37');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',66,'Group','DM38');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',67,'Group','DM39');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',68,'Group','DM40');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',69,'Group','DM41');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',70,'Group','DM42');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',71,'Group','DM43');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',72,'Group','DM44');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',73,'Group','DM45');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',74,'Group','DM46');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',75,'Group','DM47');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',76,'Group','DM48');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',77,'Group','DM49');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',78,'Group','DM50');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',79,'Group','DM51');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',80,'Group','DM52');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',81,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',82,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',83,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',84,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',85,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',86,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',87,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',88,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',89,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',90,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',91,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',92,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',93,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',94,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',95,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',96,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',97,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',98,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',99,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',100,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',101,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',102,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',103,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',104,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',105,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',106,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',107,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',108,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',109,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',110,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',111,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',112,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',113,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',114,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',115,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',116,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',117,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',118,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',119,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',120,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',121,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',122,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',123,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',124,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',125,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',126,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',127,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',128,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',129,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',130,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',131,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',132,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',133,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',134,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',135,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',136,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',137,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',138,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',139,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',140,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',141,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',142,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',143,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',144,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',145,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',146,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',147,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',148,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',149,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',150,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',151,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',152,'Group','DM25');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',153,'Group','DM26');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',154,'Group','DM27');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',155,'Group','DM28');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',156,'Group','DM29');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',157,'Group','DM30');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',158,'Group','DM31');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',159,'Group','DM32');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',160,'Group','DM33');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',161,'Group','DM34');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',162,'Group','DM35');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',163,'Group','DM36');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',164,'Group','DM37');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',165,'Group','DM38');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',166,'Group','DM39');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',167,'Group','DM40');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',168,'Group','DM41');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',169,'Group','DM42');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',170,'Group','DM43');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',171,'Group','DM44');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',172,'Group','DM45');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',173,'Group','DM46');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',174,'Group','DM47');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',175,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:6',176,'Group','DM1');
+
+--
+-- Table structure for table `divisions`
+--
+
+CREATE TABLE divisions (
+  DivisionRef int(11) NOT NULL auto_increment,
+  StructureRef int(11) default NULL,
+  DocID varchar(64) default NULL,
+  ParentType varchar(64) default NULL,
+  ParentRef varchar(64) default NULL,
+  SectionID varchar(64) default NULL,
+  DivisionType varchar(64) default NULL,
+  LabelOrder varchar(64) default NULL,
+  ShortLabel varchar(128) default NULL,
+  UserLabel varchar(255) default NULL,
+  PRIMARY KEY  (DivisionRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `divisions`
+--
+
+INSERT INTO divisions VALUES (1,NULL,'20050201:1','Structure','1','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (2,NULL,'20050201:1','Division','1','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (3,NULL,'20050201:1','Division','1','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (4,NULL,'20050201:1','Division','1','DS3','Section','3','1.3','1.3');
+INSERT INTO divisions VALUES (5,NULL,'20050201:1','Division','1','DS4','Section','4','1.4','1.4');
+INSERT INTO divisions VALUES (6,NULL,'20050201:1','Division','5','DS5','Section','5','1.4.1','1.4.1');
+INSERT INTO divisions VALUES (7,NULL,'20050201:1','Division','5','DS6','Section','6','1.4.2','1.4.2');
+INSERT INTO divisions VALUES (8,NULL,'20050201:1','Division','5','DS7','Section','7','1.4.3','1.4.3');
+INSERT INTO divisions VALUES (9,NULL,'20050201:1','Division','5','DS8','Section','8','1.4.4','1.4.4');
+INSERT INTO divisions VALUES (10,NULL,'20050201:1','Division','5','DS9','Section','9','1.4.5','1.4.5');
+INSERT INTO divisions VALUES (11,NULL,'20050201:1','Division','1','DS10','Section','10','1.5','1.5');
+INSERT INTO divisions VALUES (12,NULL,'20050201:1','Division','11','DS11','Section','11','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (13,NULL,'20050201:1','Division','11','DS12','Section','12','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (14,NULL,'20050201:1','Division','11','DS13','Section','13','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (15,NULL,'20050201:1','Division','11','DS14','Section','14','1.5.4','1.5.4');
+INSERT INTO divisions VALUES (16,NULL,'20050201:1','Division','11','DS15','Section','15','1.5.5','1.5.5');
+INSERT INTO divisions VALUES (17,NULL,'20050201:1','Division','1','DS16','Section','16','1.6','1.6');
+INSERT INTO divisions VALUES (18,NULL,'20050201:1','Division','17','DS17','Section','17','1.6.1','1.6.1');
+INSERT INTO divisions VALUES (19,NULL,'20050201:1','Division','17','DS18','Section','18','1.6.2','1.6.2');
+INSERT INTO divisions VALUES (20,NULL,'20050201:1','Division','17','DS19','Section','19','1.6.3','1.6.3');
+INSERT INTO divisions VALUES (21,NULL,'20050201:1','Division','17','DS20','Section','20','1.6.4','1.6.4');
+INSERT INTO divisions VALUES (22,NULL,'20050201:1','Division','17','DS21','Section','21','1.6.5','1.6.5');
+INSERT INTO divisions VALUES (23,NULL,'20050201:1','Division','1','DS22','Section','22','1.7','1.7');
+INSERT INTO divisions VALUES (24,NULL,'20050201:1','Division','1','DS23','Section','23','1.8','1.8');
+INSERT INTO divisions VALUES (25,NULL,'20050201:1','Division','24','DS24','Section','24','1.8.1','1.8.1');
+INSERT INTO divisions VALUES (26,NULL,'20050201:1','Division','24','DS25','Section','25','1.8.2','1.8.2');
+INSERT INTO divisions VALUES (27,NULL,'20050201:1','Division','1','DS26','Section','26','1.9','1.9');
+INSERT INTO divisions VALUES (28,NULL,'20050201:1','Division','1','DS27','Section','27','1.10','1.10');
+INSERT INTO divisions VALUES (29,NULL,'20050201:1','Structure','2','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (30,NULL,'20050201:2','Structure','3','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (31,NULL,'20050201:2','Division','30','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (32,NULL,'20050201:2','Division','30','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (33,NULL,'20050201:2','Division','30','DS3','Section','3','1.3','1.3');
+INSERT INTO divisions VALUES (34,NULL,'20050201:2','Division','30','DS4','Section','4','1.4','1.4');
+INSERT INTO divisions VALUES (35,NULL,'20050201:2','Division','34','DS5','Section','5','1.4.1','1.4.1');
+INSERT INTO divisions VALUES (36,NULL,'20050201:2','Division','34','DS6','Section','6','1.4.2','1.4.2');
+INSERT INTO divisions VALUES (37,NULL,'20050201:2','Division','34','DS7','Section','7','1.4.3','1.4.3');
+INSERT INTO divisions VALUES (38,NULL,'20050201:2','Division','34','DS8','Section','8','1.4.4','1.4.4');
+INSERT INTO divisions VALUES (39,NULL,'20050201:2','Division','30','DS9','Section','9','1.5','1.5');
+INSERT INTO divisions VALUES (40,NULL,'20050201:2','Division','39','DS10','Section','10','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (41,NULL,'20050201:2','Division','39','DS11','Section','11','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (42,NULL,'20050201:2','Division','39','DS12','Section','12','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (43,NULL,'20050201:2','Division','39','DS13','Section','13','1.5.4','1.5.4');
+INSERT INTO divisions VALUES (44,NULL,'20050201:2','Division','39','DS14','Section','14','1.5.5','1.5.5');
+INSERT INTO divisions VALUES (45,NULL,'20050201:2','Division','39','DS15','Section','15','1.5.6','1.5.6');
+INSERT INTO divisions VALUES (46,NULL,'20050201:2','Division','39','DS16','Section','16','1.5.7','1.5.7');
+INSERT INTO divisions VALUES (47,NULL,'20050201:2','Division','39','DS17','Section','17','1.5.8','1.5.8');
+INSERT INTO divisions VALUES (48,NULL,'20050201:2','Division','39','DS18','Section','18','1.5.9','1.5.9');
+INSERT INTO divisions VALUES (49,NULL,'20050201:2','Division','30','DS19','Section','19','1.6','1.6');
+INSERT INTO divisions VALUES (50,NULL,'20050201:2','Division','49','DS20','Section','20','1.6.1','1.6.1');
+INSERT INTO divisions VALUES (51,NULL,'20050201:2','Division','30','DS21','Section','21','1.7','1.7');
+INSERT INTO divisions VALUES (52,NULL,'20050201:2','Division','51','DS22','Section','22','1.7.1','1.7.1');
+INSERT INTO divisions VALUES (53,NULL,'20050201:2','Division','51','DS23','Section','23','1.7.2','1.7.2');
+INSERT INTO divisions VALUES (54,NULL,'20050201:2','Division','51','DS24','Section','24','1.7.3','1.7.3');
+INSERT INTO divisions VALUES (55,NULL,'20050201:2','Division','51','DS25','Section','25','1.7.4','1.7.4');
+INSERT INTO divisions VALUES (56,NULL,'20050201:2','Division','51','DS26','Section','26','1.7.5','1.7.5');
+INSERT INTO divisions VALUES (57,NULL,'20050201:2','Division','51','DS27','Section','27','1.7.6','1.7.6');
+INSERT INTO divisions VALUES (58,NULL,'20050201:2','Division','51','DS28','Section','28','1.7.7','1.7.7');
+INSERT INTO divisions VALUES (59,NULL,'20050201:2','Division','51','DS29','Section','29','1.7.8','1.7.8');
+INSERT INTO divisions VALUES (60,NULL,'20050201:2','Division','51','DS30','Section','30','1.7.9','1.7.9');
+INSERT INTO divisions VALUES (61,NULL,'20050201:2','Division','51','DS31','Section','31','1.7.10','1.7.10');
+INSERT INTO divisions VALUES (62,NULL,'20050201:2','Division','51','DS32','Section','32','1.7.11','1.7.11');
+INSERT INTO divisions VALUES (63,NULL,'20050201:2','Division','30','DS33','Section','33','1.8','1.8');
+INSERT INTO divisions VALUES (64,NULL,'20050201:2','Division','63','DS34','Section','34','1.8.1','1.8.1');
+INSERT INTO divisions VALUES (65,NULL,'20050201:2','Division','63','DS35','Section','35','1.8.2','1.8.2');
+INSERT INTO divisions VALUES (66,NULL,'20050201:2','Division','63','DS36','Section','36','1.8.3','1.8.3');
+INSERT INTO divisions VALUES (67,NULL,'20050201:2','Division','63','DS37','Section','37','1.8.4','1.8.4');
+INSERT INTO divisions VALUES (68,NULL,'20050201:2','Division','63','DS38','Section','38','1.8.5','1.8.5');
+INSERT INTO divisions VALUES (69,NULL,'20050201:2','Division','63','DS39','Section','39','1.8.6','1.8.6');
+INSERT INTO divisions VALUES (70,NULL,'20050201:2','Division','63','DS40','Section','40','1.8.7','1.8.7');
+INSERT INTO divisions VALUES (71,NULL,'20050201:2','Division','30','DS41','Section','41','1.9','1.9');
+INSERT INTO divisions VALUES (72,NULL,'20050201:2','Division','71','DS42','Section','42','1.9.1','1.9.1');
+INSERT INTO divisions VALUES (73,NULL,'20050201:2','Division','71','DS43','Section','43','1.9.2','1.9.2');
+INSERT INTO divisions VALUES (74,NULL,'20050201:2','Division','30','DS44','Section','44','1.10','1.10');
+INSERT INTO divisions VALUES (75,NULL,'20050201:2','Division','74','DS45','Section','45','1.10.1','1.10.1');
+INSERT INTO divisions VALUES (76,NULL,'20050201:2','Division','30','DS46','Section','46','1.11','1.11');
+INSERT INTO divisions VALUES (77,NULL,'20050201:2','Division','76','DS47','Section','47','1.11.1','1.11.1');
+INSERT INTO divisions VALUES (78,NULL,'20050201:2','Division','76','DS48','Section','48','1.11.2','1.11.2');
+INSERT INTO divisions VALUES (79,NULL,'20050201:2','Division','76','DS49','Section','49','1.11.3','1.11.3');
+INSERT INTO divisions VALUES (80,NULL,'20050201:2','Division','30','DS50','Section','50','1.12','1.12');
+INSERT INTO divisions VALUES (81,NULL,'20050201:2','Structure','4','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (82,NULL,'20050201:3','Structure','5','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (83,NULL,'20050201:3','Division','82','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (84,NULL,'20050201:3','Division','83','DS2','Section','2','1.1.1','1.1.1');
+INSERT INTO divisions VALUES (85,NULL,'20050201:3','Division','82','DS3','Section','3','1.2','1.2');
+INSERT INTO divisions VALUES (86,NULL,'20050201:3','Division','85','DS4','Section','4','1.2.1','1.2.1');
+INSERT INTO divisions VALUES (87,NULL,'20050201:3','Division','85','DS5','Section','5','1.2.2','1.2.2');
+INSERT INTO divisions VALUES (88,NULL,'20050201:3','Division','85','DS6','Section','6','1.2.3','1.2.3');
+INSERT INTO divisions VALUES (89,NULL,'20050201:3','Division','85','DS7','Section','7','1.2.4','1.2.4');
+INSERT INTO divisions VALUES (90,NULL,'20050201:3','Division','85','DS8','Section','8','1.2.5','1.2.5');
+INSERT INTO divisions VALUES (91,NULL,'20050201:3','Division','85','DS9','Section','9','1.2.6','1.2.6');
+INSERT INTO divisions VALUES (92,NULL,'20050201:3','Division','85','DS10','Section','10','1.2.7','1.2.7');
+INSERT INTO divisions VALUES (93,NULL,'20050201:3','Division','85','DS11','Section','11','1.2.8','1.2.8');
+INSERT INTO divisions VALUES (94,NULL,'20050201:3','Division','85','DS12','Section','12','1.2.9','1.2.9');
+INSERT INTO divisions VALUES (95,NULL,'20050201:3','Division','85','DS13','Section','13','1.2.10','1.2.10');
+INSERT INTO divisions VALUES (96,NULL,'20050201:3','Division','85','DS14','Section','14','1.2.11','1.2.11');
+INSERT INTO divisions VALUES (97,NULL,'20050201:3','Division','85','DS15','Section','15','1.2.12','1.2.12');
+INSERT INTO divisions VALUES (98,NULL,'20050201:3','Division','82','DS16','Section','16','1.3','1.3');
+INSERT INTO divisions VALUES (99,NULL,'20050201:3','Division','98','DS17','Section','17','1.3.1','1.3.1');
+INSERT INTO divisions VALUES (100,NULL,'20050201:3','Division','98','DS18','Section','18','1.3.2','1.3.2');
+INSERT INTO divisions VALUES (101,NULL,'20050201:3','Division','82','DS19','Section','19','1.4','1.4');
+INSERT INTO divisions VALUES (102,NULL,'20050201:3','Division','82','DS20','Section','20','1.5','1.5');
+INSERT INTO divisions VALUES (103,NULL,'20050201:3','Division','102','DS21','Section','21','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (104,NULL,'20050201:3','Structure','6','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (105,NULL,'20050201:4','Structure','7','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (106,NULL,'20050201:4','Division','105','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (107,NULL,'20050201:4','Division','105','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (108,NULL,'20050201:4','Division','107','DS3','Section','3','1.2.1','1.2.1');
+INSERT INTO divisions VALUES (109,NULL,'20050201:4','Division','107','DS4','Section','4','1.2.2','1.2.2');
+INSERT INTO divisions VALUES (110,NULL,'20050201:4','Division','107','DS5','Section','5','1.2.3','1.2.3');
+INSERT INTO divisions VALUES (111,NULL,'20050201:4','Division','105','DS6','Section','6','1.3','1.3');
+INSERT INTO divisions VALUES (112,NULL,'20050201:4','Division','111','DS7','Section','7','1.3.1','1.3.1');
+INSERT INTO divisions VALUES (113,NULL,'20050201:4','Division','111','DS8','Section','8','1.3.2','1.3.2');
+INSERT INTO divisions VALUES (114,NULL,'20050201:4','Division','111','DS9','Section','9','1.3.3','1.3.3');
+INSERT INTO divisions VALUES (115,NULL,'20050201:4','Division','111','DS10','Section','10','1.3.4','1.3.4');
+INSERT INTO divisions VALUES (116,NULL,'20050201:4','Division','105','DS11','Section','11','1.4','1.4');
+INSERT INTO divisions VALUES (117,NULL,'20050201:4','Division','116','DS12','Section','12','1.4.1','1.4.1');
+INSERT INTO divisions VALUES (118,NULL,'20050201:4','Division','116','DS13','Section','13','1.4.2','1.4.2');
+INSERT INTO divisions VALUES (119,NULL,'20050201:4','Division','105','DS14','Section','14','1.5','1.5');
+INSERT INTO divisions VALUES (120,NULL,'20050201:4','Division','119','DS15','Section','15','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (121,NULL,'20050201:4','Division','119','DS16','Section','16','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (122,NULL,'20050201:4','Division','119','DS17','Section','17','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (123,NULL,'20050201:4','Division','105','DS18','Section','18','1.6','1.6');
+INSERT INTO divisions VALUES (124,NULL,'20050201:4','Division','105','DS19','Section','19','1.7','1.7');
+INSERT INTO divisions VALUES (125,NULL,'20050201:4','Division','124','DS20','Section','20','1.7.1','1.7.1');
+INSERT INTO divisions VALUES (126,NULL,'20050201:4','Division','124','DS21','Section','21','1.7.2','1.7.2');
+INSERT INTO divisions VALUES (127,NULL,'20050201:4','Division','124','DS22','Section','22','1.7.3','1.7.3');
+INSERT INTO divisions VALUES (128,NULL,'20050201:4','Structure','8','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (129,NULL,'20050201:5','Structure','9','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (130,NULL,'20050201:5','Division','129','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (131,NULL,'20050201:5','Division','129','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (132,NULL,'20050201:5','Division','129','DS3','Section','3','1.3','1.3');
+INSERT INTO divisions VALUES (133,NULL,'20050201:5','Division','132','DS4','Section','4','1.3.1','1.3.1');
+INSERT INTO divisions VALUES (134,NULL,'20050201:5','Division','132','DS5','Section','5','1.3.2','1.3.2');
+INSERT INTO divisions VALUES (135,NULL,'20050201:5','Division','132','DS6','Section','6','1.3.3','1.3.3');
+INSERT INTO divisions VALUES (136,NULL,'20050201:5','Division','132','DS7','Section','7','1.3.4','1.3.4');
+INSERT INTO divisions VALUES (137,NULL,'20050201:5','Division','132','DS8','Section','8','1.3.5','1.3.5');
+INSERT INTO divisions VALUES (138,NULL,'20050201:5','Division','132','DS9','Section','9','1.3.6','1.3.6');
+INSERT INTO divisions VALUES (139,NULL,'20050201:5','Division','129','DS10','Section','10','1.4','1.4');
+INSERT INTO divisions VALUES (140,NULL,'20050201:5','Division','129','DS11','Section','11','1.5','1.5');
+INSERT INTO divisions VALUES (141,NULL,'20050201:5','Division','140','DS12','Section','12','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (142,NULL,'20050201:5','Division','140','DS13','Section','13','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (143,NULL,'20050201:5','Division','140','DS14','Section','14','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (144,NULL,'20050201:5','Division','140','DS15','Section','15','1.5.4','1.5.4');
+INSERT INTO divisions VALUES (145,NULL,'20050201:5','Division','129','DS16','Section','16','1.6','1.6');
+INSERT INTO divisions VALUES (146,NULL,'20050201:5','Division','145','DS17','Section','17','1.6.1','1.6.1');
+INSERT INTO divisions VALUES (147,NULL,'20050201:5','Division','145','DS18','Section','18','1.6.2','1.6.2');
+INSERT INTO divisions VALUES (148,NULL,'20050201:5','Division','145','DS19','Section','19','1.6.3','1.6.3');
+INSERT INTO divisions VALUES (149,NULL,'20050201:5','Division','145','DS20','Section','20','1.6.4','1.6.4');
+INSERT INTO divisions VALUES (150,NULL,'20050201:5','Division','145','DS21','Section','21','1.6.5','1.6.5');
+INSERT INTO divisions VALUES (151,NULL,'20050201:5','Division','145','DS22','Section','22','1.6.6','1.6.6');
+INSERT INTO divisions VALUES (152,NULL,'20050201:5','Division','145','DS23','Section','23','1.6.7','1.6.7');
+INSERT INTO divisions VALUES (153,NULL,'20050201:5','Division','145','DS24','Section','24','1.6.8','1.6.8');
+INSERT INTO divisions VALUES (154,NULL,'20050201:5','Division','145','DS25','Section','25','1.6.9','1.6.9');
+INSERT INTO divisions VALUES (155,NULL,'20050201:5','Division','145','DS26','Section','26','1.6.10','1.6.10');
+INSERT INTO divisions VALUES (156,NULL,'20050201:5','Division','129','DS27','Section','27','1.7','1.7');
+INSERT INTO divisions VALUES (157,NULL,'20050201:5','Division','156','DS28','Section','28','1.7.1','1.7.1');
+INSERT INTO divisions VALUES (158,NULL,'20050201:5','Division','156','DS29','Section','29','1.7.2','1.7.2');
+INSERT INTO divisions VALUES (159,NULL,'20050201:5','Division','156','DS30','Section','30','1.7.3','1.7.3');
+INSERT INTO divisions VALUES (160,NULL,'20050201:5','Division','156','DS31','Section','31','1.7.4','1.7.4');
+INSERT INTO divisions VALUES (161,NULL,'20050201:5','Division','156','DS32','Section','32','1.7.5','1.7.5');
+INSERT INTO divisions VALUES (162,NULL,'20050201:5','Division','156','DS33','Section','33','1.7.6','1.7.6');
+INSERT INTO divisions VALUES (163,NULL,'20050201:5','Division','156','DS34','Section','34','1.7.7','1.7.7');
+INSERT INTO divisions VALUES (164,NULL,'20050201:5','Division','156','DS35','Section','35','1.7.8','1.7.8');
+INSERT INTO divisions VALUES (165,NULL,'20050201:5','Division','156','DS36','Section','36','1.7.9','1.7.9');
+INSERT INTO divisions VALUES (166,NULL,'20050201:5','Division','156','DS37','Section','37','1.7.10','1.7.10');
+INSERT INTO divisions VALUES (167,NULL,'20050201:5','Division','156','DS38','Section','38','1.7.11','1.7.11');
+INSERT INTO divisions VALUES (168,NULL,'20050201:5','Division','156','DS39','Section','39','1.7.12','1.7.12');
+INSERT INTO divisions VALUES (169,NULL,'20050201:5','Division','156','DS40','Section','40','1.7.13','1.7.13');
+INSERT INTO divisions VALUES (170,NULL,'20050201:5','Division','156','DS41','Section','41','1.7.14','1.7.14');
+INSERT INTO divisions VALUES (171,NULL,'20050201:5','Division','156','DS42','Section','42','1.7.15','1.7.15');
+INSERT INTO divisions VALUES (172,NULL,'20050201:5','Division','156','DS43','Section','43','1.7.16','1.7.16');
+INSERT INTO divisions VALUES (173,NULL,'20050201:5','Division','129','DS44','Section','44','1.8','1.8');
+INSERT INTO divisions VALUES (174,NULL,'20050201:5','Division','129','DS45','Section','45','1.9','1.9');
+INSERT INTO divisions VALUES (175,NULL,'20050201:5','Structure','10','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (176,NULL,'20050201:6','Structure','11','All','Document','All','All','Whole Document');
+
+--
+-- Table structure for table `document`
+--
+
+CREATE TABLE document (
+  DocID varchar(64) NOT NULL default '',
+  DocType varchar(64) default NULL,
+  AccessionDate datetime default NULL,
+  IndexedDate datetime default NULL,
+  ModifiedDate datetime default NULL,
+  PRIMARY KEY  (DocID)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `document`
+--
+
+INSERT INTO document VALUES ('20050201:1','HTML','2005-02-01 22:03:55','2005-02-01 22:03:55','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:2','HTML','2005-02-01 22:03:56','2005-02-01 22:03:56','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:3','HTML','2005-02-01 22:03:59','2005-02-01 22:03:59','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:4','HTML','2005-02-01 22:03:59','2005-02-01 22:03:59','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:5','HTML','2005-02-01 22:04:00','2005-02-01 22:04:00','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:6','GSMETADATA','2005-02-01 22:04:01','2005-02-01 22:04:01','2004-07-20 04:36:03');
+
+--
+-- Table structure for table `filegroups`
+--
+
+CREATE TABLE filegroups (
+  FileGroupRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  FileGroupID varchar(64) default NULL,
+  ParentRef int(11) default NULL,
+  ParentType varchar(16) default NULL,
+  PRIMARY KEY  (FileGroupRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `filegroups`
+--
+
+INSERT INTO filegroups VALUES (1,'20050201:1','FILEGROUP_PRELUDE4',1,'Section');
+INSERT INTO filegroups VALUES (2,'20050201:1','FILEGROUP_PRELUDE11',1,'Section');
+INSERT INTO filegroups VALUES (3,'20050201:1','FILEGROUP_PRELUDE1',1,'Section');
+INSERT INTO filegroups VALUES (4,'20050201:1','FILEGROUP_PRELUDE7',1,'Section');
+INSERT INTO filegroups VALUES (5,'20050201:1','FILEGROUP_PRELUDE25',1,'Section');
+INSERT INTO filegroups VALUES (6,'20050201:1','FILEGROUP_PRELUDE28',1,'Section');
+INSERT INTO filegroups VALUES (7,'20050201:1','FILEGROUP_PRELUDE14',1,'Section');
+INSERT INTO filegroups VALUES (8,'20050201:1','FILEGROUP_PRELUDE24',1,'Section');
+INSERT INTO filegroups VALUES (9,'20050201:1','FILEGROUP_PRELUDE6',1,'Section');
+INSERT INTO filegroups VALUES (10,'20050201:1','FILEGROUP_PRELUDE13',1,'Section');
+INSERT INTO filegroups VALUES (11,'20050201:1','FILEGROUP_PRELUDE26',1,'Section');
+INSERT INTO filegroups VALUES (12,'20050201:1','FILEGROUP_PRELUDE10',1,'Section');
+INSERT INTO filegroups VALUES (13,'20050201:1','FILEGROUP_PRELUDE27',1,'Section');
+INSERT INTO filegroups VALUES (14,'20050201:1','FILEGROUP_PRELUDE12',1,'Section');
+INSERT INTO filegroups VALUES (15,'20050201:1','FILEGROUP_PRELUDE5',1,'Section');
+INSERT INTO filegroups VALUES (16,'20050201:1','FILEGROUP_PRELUDE15',1,'Section');
+INSERT INTO filegroups VALUES (17,'20050201:1','FILEGROUP_PRELUDE8',1,'Section');
+INSERT INTO filegroups VALUES (18,'20050201:1','FILEGROUP_PRELUDE22',1,'Section');
+INSERT INTO filegroups VALUES (19,'20050201:1','FILEGROUP_PRELUDE18',1,'Section');
+INSERT INTO filegroups VALUES (20,'20050201:1','FILEGROUP_PRELUDE3',1,'Section');
+INSERT INTO filegroups VALUES (21,'20050201:1','default',1,'Section');
+INSERT INTO filegroups VALUES (22,'20050201:1','FILEGROUP_PRELUDE17',1,'Section');
+INSERT INTO filegroups VALUES (23,'20050201:1','FILEGROUP_PRELUDE2',1,'Section');
+INSERT INTO filegroups VALUES (24,'20050201:1','FILEGROUP_PRELUDE20',1,'Section');
+INSERT INTO filegroups VALUES (25,'20050201:1','FILEGROUP_PRELUDE23',1,'Section');
+INSERT INTO filegroups VALUES (26,'20050201:1','FILEGROUP_PRELUDE9',1,'Section');
+INSERT INTO filegroups VALUES (27,'20050201:1','FILEGROUP_PRELUDE16',1,'Section');
+INSERT INTO filegroups VALUES (28,'20050201:1','FILEGROUP_PRELUDE19',1,'Section');
+INSERT INTO filegroups VALUES (29,'20050201:1','FILEGROUP_PRELUDE21',1,'Section');
+INSERT INTO filegroups VALUES (30,'20050201:2','FILEGROUP_PRELUDE51',2,'Section');
+INSERT INTO filegroups VALUES (31,'20050201:2','FILEGROUP_PRELUDE33',2,'Section');
+INSERT INTO filegroups VALUES (32,'20050201:2','FILEGROUP_PRELUDE48',2,'Section');
+INSERT INTO filegroups VALUES (33,'20050201:2','FILEGROUP_PRELUDE36',2,'Section');
+INSERT INTO filegroups VALUES (34,'20050201:2','FILEGROUP_PRELUDE14',2,'Section');
+INSERT INTO filegroups VALUES (35,'20050201:2','FILEGROUP_PRELUDE35',2,'Section');
+INSERT INTO filegroups VALUES (36,'20050201:2','FILEGROUP_PRELUDE13',2,'Section');
+INSERT INTO filegroups VALUES (37,'20050201:2','FILEGROUP_PRELUDE34',2,'Section');
+INSERT INTO filegroups VALUES (38,'20050201:2','FILEGROUP_PRELUDE49',2,'Section');
+INSERT INTO filegroups VALUES (39,'20050201:2','FILEGROUP_PRELUDE22',2,'Section');
+INSERT INTO filegroups VALUES (40,'20050201:2','FILEGROUP_PRELUDE29',2,'Section');
+INSERT INTO filegroups VALUES (41,'20050201:2','FILEGROUP_PRELUDE40',2,'Section');
+INSERT INTO filegroups VALUES (42,'20050201:2','FILEGROUP_PRELUDE44',2,'Section');
+INSERT INTO filegroups VALUES (43,'20050201:2','default',2,'Section');
+INSERT INTO filegroups VALUES (44,'20050201:2','FILEGROUP_PRELUDE3',2,'Section');
+INSERT INTO filegroups VALUES (45,'20050201:2','FILEGROUP_PRELUDE42',2,'Section');
+INSERT INTO filegroups VALUES (46,'20050201:2','FILEGROUP_PRELUDE39',2,'Section');
+INSERT INTO filegroups VALUES (47,'20050201:2','FILEGROUP_PRELUDE23',2,'Section');
+INSERT INTO filegroups VALUES (48,'20050201:2','FILEGROUP_PRELUDE9',2,'Section');
+INSERT INTO filegroups VALUES (49,'20050201:2','FILEGROUP_PRELUDE43',2,'Section');
+INSERT INTO filegroups VALUES (50,'20050201:2','FILEGROUP_PRELUDE50',2,'Section');
+INSERT INTO filegroups VALUES (51,'20050201:2','FILEGROUP_PRELUDE45',2,'Section');
+INSERT INTO filegroups VALUES (52,'20050201:2','FILEGROUP_PRELUDE4',2,'Section');
+INSERT INTO filegroups VALUES (53,'20050201:2','FILEGROUP_PRELUDE1',2,'Section');
+INSERT INTO filegroups VALUES (54,'20050201:2','FILEGROUP_PRELUDE11',2,'Section');
+INSERT INTO filegroups VALUES (55,'20050201:2','FILEGROUP_PRELUDE7',2,'Section');
+INSERT INTO filegroups VALUES (56,'20050201:2','FILEGROUP_PRELUDE30',2,'Section');
+INSERT INTO filegroups VALUES (57,'20050201:2','FILEGROUP_PRELUDE25',2,'Section');
+INSERT INTO filegroups VALUES (58,'20050201:2','FILEGROUP_PRELUDE41',2,'Section');
+INSERT INTO filegroups VALUES (59,'20050201:2','FILEGROUP_PRELUDE28',2,'Section');
+INSERT INTO filegroups VALUES (60,'20050201:2','FILEGROUP_PRELUDE47',2,'Section');
+INSERT INTO filegroups VALUES (61,'20050201:2','FILEGROUP_PRELUDE46',2,'Section');
+INSERT INTO filegroups VALUES (62,'20050201:2','FILEGROUP_PRELUDE24',2,'Section');
+INSERT INTO filegroups VALUES (63,'20050201:2','FILEGROUP_PRELUDE6',2,'Section');
+INSERT INTO filegroups VALUES (64,'20050201:2','FILEGROUP_PRELUDE10',2,'Section');
+INSERT INTO filegroups VALUES (65,'20050201:2','FILEGROUP_PRELUDE26',2,'Section');
+INSERT INTO filegroups VALUES (66,'20050201:2','FILEGROUP_PRELUDE32',2,'Section');
+INSERT INTO filegroups VALUES (67,'20050201:2','FILEGROUP_PRELUDE27',2,'Section');
+INSERT INTO filegroups VALUES (68,'20050201:2','FILEGROUP_PRELUDE5',2,'Section');
+INSERT INTO filegroups VALUES (69,'20050201:2','FILEGROUP_PRELUDE12',2,'Section');
+INSERT INTO filegroups VALUES (70,'20050201:2','FILEGROUP_PRELUDE15',2,'Section');
+INSERT INTO filegroups VALUES (71,'20050201:2','FILEGROUP_PRELUDE8',2,'Section');
+INSERT INTO filegroups VALUES (72,'20050201:2','FILEGROUP_PRELUDE18',2,'Section');
+INSERT INTO filegroups VALUES (73,'20050201:2','FILEGROUP_PRELUDE37',2,'Section');
+INSERT INTO filegroups VALUES (74,'20050201:2','FILEGROUP_PRELUDE17',2,'Section');
+INSERT INTO filegroups VALUES (75,'20050201:2','FILEGROUP_PRELUDE2',2,'Section');
+INSERT INTO filegroups VALUES (76,'20050201:2','FILEGROUP_PRELUDE31',2,'Section');
+INSERT INTO filegroups VALUES (77,'20050201:2','FILEGROUP_PRELUDE20',2,'Section');
+INSERT INTO filegroups VALUES (78,'20050201:2','FILEGROUP_PRELUDE19',2,'Section');
+INSERT INTO filegroups VALUES (79,'20050201:2','FILEGROUP_PRELUDE16',2,'Section');
+INSERT INTO filegroups VALUES (80,'20050201:2','FILEGROUP_PRELUDE38',2,'Section');
+INSERT INTO filegroups VALUES (81,'20050201:2','FILEGROUP_PRELUDE21',2,'Section');
+INSERT INTO filegroups VALUES (82,'20050201:3','FILEGROUP_PRELUDE15',3,'Section');
+INSERT INTO filegroups VALUES (83,'20050201:3','FILEGROUP_PRELUDE4',3,'Section');
+INSERT INTO filegroups VALUES (84,'20050201:3','FILEGROUP_PRELUDE22',3,'Section');
+INSERT INTO filegroups VALUES (85,'20050201:3','FILEGROUP_PRELUDE8',3,'Section');
+INSERT INTO filegroups VALUES (86,'20050201:3','FILEGROUP_PRELUDE18',3,'Section');
+INSERT INTO filegroups VALUES (87,'20050201:3','FILEGROUP_PRELUDE1',3,'Section');
+INSERT INTO filegroups VALUES (88,'20050201:3','FILEGROUP_PRELUDE11',3,'Section');
+INSERT INTO filegroups VALUES (89,'20050201:3','FILEGROUP_PRELUDE7',3,'Section');
+INSERT INTO filegroups VALUES (90,'20050201:3','default',3,'Section');
+INSERT INTO filegroups VALUES (91,'20050201:3','FILEGROUP_PRELUDE3',3,'Section');
+INSERT INTO filegroups VALUES (92,'20050201:3','FILEGROUP_PRELUDE17',3,'Section');
+INSERT INTO filegroups VALUES (93,'20050201:3','FILEGROUP_PRELUDE14',3,'Section');
+INSERT INTO filegroups VALUES (94,'20050201:3','FILEGROUP_PRELUDE2',3,'Section');
+INSERT INTO filegroups VALUES (95,'20050201:3','FILEGROUP_PRELUDE20',3,'Section');
+INSERT INTO filegroups VALUES (96,'20050201:3','FILEGROUP_PRELUDE13',3,'Section');
+INSERT INTO filegroups VALUES (97,'20050201:3','FILEGROUP_PRELUDE6',3,'Section');
+INSERT INTO filegroups VALUES (98,'20050201:3','FILEGROUP_PRELUDE10',3,'Section');
+INSERT INTO filegroups VALUES (99,'20050201:3','FILEGROUP_PRELUDE19',3,'Section');
+INSERT INTO filegroups VALUES (100,'20050201:3','FILEGROUP_PRELUDE16',3,'Section');
+INSERT INTO filegroups VALUES (101,'20050201:3','FILEGROUP_PRELUDE9',3,'Section');
+INSERT INTO filegroups VALUES (102,'20050201:3','FILEGROUP_PRELUDE21',3,'Section');
+INSERT INTO filegroups VALUES (103,'20050201:3','FILEGROUP_PRELUDE5',3,'Section');
+INSERT INTO filegroups VALUES (104,'20050201:3','FILEGROUP_PRELUDE12',3,'Section');
+INSERT INTO filegroups VALUES (105,'20050201:4','FILEGROUP_PRELUDE15',4,'Section');
+INSERT INTO filegroups VALUES (106,'20050201:4','FILEGROUP_PRELUDE4',4,'Section');
+INSERT INTO filegroups VALUES (107,'20050201:4','FILEGROUP_PRELUDE22',4,'Section');
+INSERT INTO filegroups VALUES (108,'20050201:4','FILEGROUP_PRELUDE8',4,'Section');
+INSERT INTO filegroups VALUES (109,'20050201:4','FILEGROUP_PRELUDE18',4,'Section');
+INSERT INTO filegroups VALUES (110,'20050201:4','FILEGROUP_PRELUDE1',4,'Section');
+INSERT INTO filegroups VALUES (111,'20050201:4','FILEGROUP_PRELUDE11',4,'Section');
+INSERT INTO filegroups VALUES (112,'20050201:4','FILEGROUP_PRELUDE7',4,'Section');
+INSERT INTO filegroups VALUES (113,'20050201:4','default',4,'Section');
+INSERT INTO filegroups VALUES (114,'20050201:4','FILEGROUP_PRELUDE3',4,'Section');
+INSERT INTO filegroups VALUES (115,'20050201:4','FILEGROUP_PRELUDE17',4,'Section');
+INSERT INTO filegroups VALUES (116,'20050201:4','FILEGROUP_PRELUDE14',4,'Section');
+INSERT INTO filegroups VALUES (117,'20050201:4','FILEGROUP_PRELUDE2',4,'Section');
+INSERT INTO filegroups VALUES (118,'20050201:4','FILEGROUP_PRELUDE20',4,'Section');
+INSERT INTO filegroups VALUES (119,'20050201:4','FILEGROUP_PRELUDE13',4,'Section');
+INSERT INTO filegroups VALUES (120,'20050201:4','FILEGROUP_PRELUDE6',4,'Section');
+INSERT INTO filegroups VALUES (121,'20050201:4','FILEGROUP_PRELUDE23',4,'Section');
+INSERT INTO filegroups VALUES (122,'20050201:4','FILEGROUP_PRELUDE10',4,'Section');
+INSERT INTO filegroups VALUES (123,'20050201:4','FILEGROUP_PRELUDE19',4,'Section');
+INSERT INTO filegroups VALUES (124,'20050201:4','FILEGROUP_PRELUDE16',4,'Section');
+INSERT INTO filegroups VALUES (125,'20050201:4','FILEGROUP_PRELUDE9',4,'Section');
+INSERT INTO filegroups VALUES (126,'20050201:4','FILEGROUP_PRELUDE21',4,'Section');
+INSERT INTO filegroups VALUES (127,'20050201:4','FILEGROUP_PRELUDE5',4,'Section');
+INSERT INTO filegroups VALUES (128,'20050201:4','FILEGROUP_PRELUDE12',4,'Section');
+INSERT INTO filegroups VALUES (129,'20050201:5','FILEGROUP_PRELUDE4',5,'Section');
+INSERT INTO filegroups VALUES (130,'20050201:5','FILEGROUP_PRELUDE11',5,'Section');
+INSERT INTO filegroups VALUES (131,'20050201:5','FILEGROUP_PRELUDE1',5,'Section');
+INSERT INTO filegroups VALUES (132,'20050201:5','FILEGROUP_PRELUDE33',5,'Section');
+INSERT INTO filegroups VALUES (133,'20050201:5','FILEGROUP_PRELUDE30',5,'Section');
+INSERT INTO filegroups VALUES (134,'20050201:5','FILEGROUP_PRELUDE7',5,'Section');
+INSERT INTO filegroups VALUES (135,'20050201:5','FILEGROUP_PRELUDE25',5,'Section');
+INSERT INTO filegroups VALUES (136,'20050201:5','FILEGROUP_PRELUDE41',5,'Section');
+INSERT INTO filegroups VALUES (137,'20050201:5','FILEGROUP_PRELUDE28',5,'Section');
+INSERT INTO filegroups VALUES (138,'20050201:5','FILEGROUP_PRELUDE36',5,'Section');
+INSERT INTO filegroups VALUES (139,'20050201:5','FILEGROUP_PRELUDE14',5,'Section');
+INSERT INTO filegroups VALUES (140,'20050201:5','FILEGROUP_PRELUDE46',5,'Section');
+INSERT INTO filegroups VALUES (141,'20050201:5','FILEGROUP_PRELUDE24',5,'Section');
+INSERT INTO filegroups VALUES (142,'20050201:5','FILEGROUP_PRELUDE35',5,'Section');
+INSERT INTO filegroups VALUES (143,'20050201:5','FILEGROUP_PRELUDE6',5,'Section');
+INSERT INTO filegroups VALUES (144,'20050201:5','FILEGROUP_PRELUDE13',5,'Section');
+INSERT INTO filegroups VALUES (145,'20050201:5','FILEGROUP_PRELUDE26',5,'Section');
+INSERT INTO filegroups VALUES (146,'20050201:5','FILEGROUP_PRELUDE10',5,'Section');
+INSERT INTO filegroups VALUES (147,'20050201:5','FILEGROUP_PRELUDE32',5,'Section');
+INSERT INTO filegroups VALUES (148,'20050201:5','FILEGROUP_PRELUDE34',5,'Section');
+INSERT INTO filegroups VALUES (149,'20050201:5','FILEGROUP_PRELUDE27',5,'Section');
+INSERT INTO filegroups VALUES (150,'20050201:5','FILEGROUP_PRELUDE12',5,'Section');
+INSERT INTO filegroups VALUES (151,'20050201:5','FILEGROUP_PRELUDE5',5,'Section');
+INSERT INTO filegroups VALUES (152,'20050201:5','FILEGROUP_PRELUDE15',5,'Section');
+INSERT INTO filegroups VALUES (153,'20050201:5','FILEGROUP_PRELUDE8',5,'Section');
+INSERT INTO filegroups VALUES (154,'20050201:5','FILEGROUP_PRELUDE22',5,'Section');
+INSERT INTO filegroups VALUES (155,'20050201:5','FILEGROUP_PRELUDE40',5,'Section');
+INSERT INTO filegroups VALUES (156,'20050201:5','FILEGROUP_PRELUDE29',5,'Section');
+INSERT INTO filegroups VALUES (157,'20050201:5','FILEGROUP_PRELUDE18',5,'Section');
+INSERT INTO filegroups VALUES (158,'20050201:5','FILEGROUP_PRELUDE44',5,'Section');
+INSERT INTO filegroups VALUES (159,'20050201:5','FILEGROUP_PRELUDE37',5,'Section');
+INSERT INTO filegroups VALUES (160,'20050201:5','FILEGROUP_PRELUDE3',5,'Section');
+INSERT INTO filegroups VALUES (161,'20050201:5','default',5,'Section');
+INSERT INTO filegroups VALUES (162,'20050201:5','FILEGROUP_PRELUDE17',5,'Section');
+INSERT INTO filegroups VALUES (163,'20050201:5','FILEGROUP_PRELUDE31',5,'Section');
+INSERT INTO filegroups VALUES (164,'20050201:5','FILEGROUP_PRELUDE2',5,'Section');
+INSERT INTO filegroups VALUES (165,'20050201:5','FILEGROUP_PRELUDE42',5,'Section');
+INSERT INTO filegroups VALUES (166,'20050201:5','FILEGROUP_PRELUDE39',5,'Section');
+INSERT INTO filegroups VALUES (167,'20050201:5','FILEGROUP_PRELUDE20',5,'Section');
+INSERT INTO filegroups VALUES (168,'20050201:5','FILEGROUP_PRELUDE23',5,'Section');
+INSERT INTO filegroups VALUES (169,'20050201:5','FILEGROUP_PRELUDE9',5,'Section');
+INSERT INTO filegroups VALUES (170,'20050201:5','FILEGROUP_PRELUDE16',5,'Section');
+INSERT INTO filegroups VALUES (171,'20050201:5','FILEGROUP_PRELUDE19',5,'Section');
+INSERT INTO filegroups VALUES (172,'20050201:5','FILEGROUP_PRELUDE43',5,'Section');
+INSERT INTO filegroups VALUES (173,'20050201:5','FILEGROUP_PRELUDE38',5,'Section');
+INSERT INTO filegroups VALUES (174,'20050201:5','FILEGROUP_PRELUDE45',5,'Section');
+INSERT INTO filegroups VALUES (175,'20050201:5','FILEGROUP_PRELUDE21',5,'Section');
+INSERT INTO filegroups VALUES (176,'20050201:6','default',6,'Section');
+
+--
+-- Table structure for table `files`
+--
+
+CREATE TABLE files (
+  FileGroupRef int(11) default NULL,
+  FileLocType varchar(64) default NULL,
+  FileLocation tinytext,
+  MIMEType varchar(64) default NULL,
+  FileID varchar(32) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `files`
+--
+
+INSERT INTO files VALUES (1,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/166/comment()[1]/range-to(/1/2/213/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (2,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/437/comment()[1]/range-to(/1/2/489/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (3,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/3/comment()[1]/range-to(/1/2/119/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (4,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/301/comment()[1]/range-to(/1/2/343/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (5,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/861/comment()[1]/range-to(/1/2/1369/comment()[1]))','text/html','FILE25');
+INSERT INTO files VALUES (6,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE28');
+INSERT INTO files VALUES (7,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/564/comment()[1]/range-to(/1/2/600/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (8,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1042/comment()[1]/range-to(/1/2/1369/comment()[1]))','text/html','FILE24');
+INSERT INTO files VALUES (9,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/263/comment()[1]/range-to(/1/2/301/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (10,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/532/comment()[1]/range-to(/1/2/564/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (11,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1369/comment()[1]/range-to(/1/2/1380/comment()[1]))','text/html','FILE26');
+INSERT INTO files VALUES (12,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/389/comment()[1]/range-to(/1/2/437/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (13,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1380/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE27');
+INSERT INTO files VALUES (14,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/489/comment()[1]/range-to(/1/2/532/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (15,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/213/comment()[1]/range-to(/1/2/263/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (16,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/374/comment()[1]/range-to(/1/2/600/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (17,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/343/comment()[1]/range-to(/1/2/374/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (18,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/793/comment()[1]/range-to(/1/2/861/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (19,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/685/comment()[1]/range-to(/1/2/718/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (20,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/131/comment()[1]/range-to(/1/2/151/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm','text/html','default.1');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p00.png','image/png','default.2');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p06.png','image/png','default.3');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p09.png','image/png','default.4');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p13.png','image/png','default.5');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p23.png','image/png','default.6');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p26.png','image/png','default.7');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p29.png','image/png','default.8');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p33.png','image/png','default.9');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p40.png','image/png','default.10');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p43.png','image/png','default.11');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p45.png','image/png','default.12');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p48.png','image/png','default.13');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p52.png','image/png','default.14');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p55.png','image/png','default.15');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p58.png','image/png','default.16');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p61.png','image/png','default.17');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p63.png','image/png','default.18');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p66.png','image/png','default.19');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p68.png','image/png','default.20');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p70.png','image/png','default.21');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p73.png','image/png','default.22');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p77.png','image/png','default.23');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p79.png','image/png','default.24');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p82.png','image/png','default.25');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p84.png','image/png','default.26');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p87.png','image/png','default.27');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p88.png','image/png','default.28');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p91.png','image/png','default.29');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p93.png','image/png','default.30');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p94.png','image/png','default.31');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p101.png','image/png','default.32');
+INSERT INTO files VALUES (22,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/648/comment()[1]/range-to(/1/2/685/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (23,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/119/comment()[1]/range-to(/1/2/131/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (24,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/751/comment()[1]/range-to(/1/2/793/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (25,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/862/comment()[1]/range-to(/1/2/1042/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (26,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/151/comment()[1]/range-to(/1/2/374/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (27,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/615/comment()[1]/range-to(/1/2/648/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (28,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/718/comment()[1]/range-to(/1/2/751/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (29,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/600/comment()[1]/range-to(/1/2/793/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (30,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE51');
+INSERT INTO files VALUES (31,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2189/comment()[1]/range-to(/1/2/2234/comment()[1]))','text/html','FILE33');
+INSERT INTO files VALUES (32,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/4085/comment()[1]/range-to(/1/2/4112/comment()[1]))','text/html','FILE48');
+INSERT INTO files VALUES (33,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2332/comment()[1]/range-to(/1/2/2363/comment()[1]))','text/html','FILE36');
+INSERT INTO files VALUES (34,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1151/comment()[1]/range-to(/1/2/1233/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (35,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2283/comment()[1]/range-to(/1/2/2332/comment()[1]))','text/html','FILE35');
+INSERT INTO files VALUES (36,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1072/comment()[1]/range-to(/1/2/1151/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (37,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2234/comment()[1]/range-to(/1/2/2283/comment()[1]))','text/html','FILE34');
+INSERT INTO files VALUES (38,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2732/comment()[1]/range-to(/1/2/4112/comment()[1]))','text/html','FILE49');
+INSERT INTO files VALUES (39,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1619/comment()[1]/range-to(/1/2/1683/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (40,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2004/comment()[1]/range-to(/1/2/2059/comment()[1]))','text/html','FILE29');
+INSERT INTO files VALUES (41,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2166/comment()[1]/range-to(/1/2/2538/comment()[1]))','text/html','FILE40');
+INSERT INTO files VALUES (42,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2662/comment()[1]/range-to(/1/2/2732/comment()[1]))','text/html','FILE44');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm','text/html','default.1');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g001a.png','image/png','default.2');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p9a.png','image/png','default.3');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g002a.png','image/png','default.4');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g003a.png','image/png','default.5');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g004a.png','image/png','default.6');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p24a.png','image/png','default.7');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p39a.png','image/png','default.8');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g005a.png','image/png','default.9');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g006a.png','image/png','default.10');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g007a.png','image/png','default.11');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g008a.png','image/png','default.12');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p81a.png','image/png','default.13');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g009a.png','image/png','default.14');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p96a.png','image/png','default.15');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g010a.png','image/png','default.16');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p112a.png','image/png','default.17');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g011a.png','image/png','default.18');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p117a.png','image/png','default.19');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g012a.png','image/png','default.20');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p129a.png','image/png','default.21');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g013a.png','image/png','default.22');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p138a.png','image/png','default.23');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g014a.png','image/png','default.24');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p149a.png','image/png','default.25');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g015a.png','image/png','default.26');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p159a.png','image/png','default.27');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g016a.png','image/png','default.28');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g017a.png','image/png','default.29');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g018a.png','image/png','default.30');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g019a.png','image/png','default.31');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g020a.png','image/png','default.32');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p203a.png','image/png','default.33');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g021a.png','image/png','default.34');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p210a.png','image/png','default.35');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g022a.png','image/png','default.36');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p219a.png','image/png','default.37');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p227a.png','image/png','default.38');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g023a.png','image/png','default.39');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p236a.png','image/png','default.40');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g024a.png','image/png','default.41');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p243a.png','image/png','default.42');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g025a.png','image/png','default.43');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p252a.png','image/png','default.44');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g026a.png','image/png','default.45');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p260a.png','image/png','default.46');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p265a.png','image/png','default.47');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g027a.png','image/png','default.48');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p274a.png','image/png','default.49');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g028a.png','image/png','default.50');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g029a.png','image/png','default.51');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p292a.png','image/png','default.52');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p293a.png','image/png','default.53');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g030a.png','image/png','default.54');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p301a.png','image/png','default.55');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p309a.png','image/png','default.56');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g031a.png','image/png','default.57');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g032a.png','image/png','default.58');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g033a.png','image/png','default.59');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g034a.png','image/png','default.60');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p323a.png','image/png','default.61');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g035a.png','image/png','default.62');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g036a.png','image/png','default.63');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g037a.png','image/png','default.64');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g038a.png','image/png','default.65');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p349a.png','image/png','default.66');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g039a.png','image/png','default.67');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p357a.png','image/png','default.68');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g040a.png','image/png','default.69');
+INSERT INTO files VALUES (44,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/216/comment()[1]/range-to(/1/2/320/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (45,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2607/comment()[1]/range-to(/1/2/2658/comment()[1]))','text/html','FILE42');
+INSERT INTO files VALUES (46,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2474/comment()[1]/range-to(/1/2/2538/comment()[1]))','text/html','FILE39');
+INSERT INTO files VALUES (47,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1683/comment()[1]/range-to(/1/2/1731/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (48,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/760/comment()[1]/range-to(/1/2/842/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (49,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2538/comment()[1]/range-to(/1/2/2658/comment()[1]))','text/html','FILE43');
+INSERT INTO files VALUES (50,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/4112/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE50');
+INSERT INTO files VALUES (51,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2658/comment()[1]/range-to(/1/2/2732/comment()[1]))','text/html','FILE45');
+INSERT INTO files VALUES (52,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/327/comment()[1]/range-to(/1/2/424/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (53,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2/comment()[1]/range-to(/1/2/191/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (54,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/913/comment()[1]/range-to(/1/2/1000/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (55,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/652/comment()[1]/range-to(/1/2/733/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (56,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2059/comment()[1]/range-to(/1/2/2108/comment()[1]))','text/html','FILE30');
+INSERT INTO files VALUES (57,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1788/comment()[1]/range-to(/1/2/1841/comment()[1]))','text/html','FILE25');
+INSERT INTO files VALUES (58,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2557/comment()[1]/range-to(/1/2/2607/comment()[1]))','text/html','FILE41');
+INSERT INTO files VALUES (59,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1952/comment()[1]/range-to(/1/2/2004/comment()[1]))','text/html','FILE28');
+INSERT INTO files VALUES (60,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/3152/comment()[1]/range-to(/1/2/4085/comment()[1]))','text/html','FILE47');
+INSERT INTO files VALUES (61,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2733/comment()[1]/range-to(/1/2/3152/comment()[1]))','text/html','FILE46');
+INSERT INTO files VALUES (62,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1731/comment()[1]/range-to(/1/2/1788/comment()[1]))','text/html','FILE24');
+INSERT INTO files VALUES (63,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/531/comment()[1]/range-to(/1/2/652/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (64,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/842/comment()[1]/range-to(/1/2/913/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (65,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1841/comment()[1]/range-to(/1/2/1902/comment()[1]))','text/html','FILE26');
+INSERT INTO files VALUES (66,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1537/comment()[1]/range-to(/1/2/2166/comment()[1]))','text/html','FILE32');
+INSERT INTO files VALUES (67,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1902/comment()[1]/range-to(/1/2/1952/comment()[1]))','text/html','FILE27');
+INSERT INTO files VALUES (68,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/424/comment()[1]/range-to(/1/2/531/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (69,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1000/comment()[1]/range-to(/1/2/1072/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (70,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1233/comment()[1]/range-to(/1/2/1307/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (71,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/320/comment()[1]/range-to(/1/2/733/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (72,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/733/comment()[1]/range-to(/1/2/1447/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (73,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2363/comment()[1]/range-to(/1/2/2410/comment()[1]))','text/html','FILE37');
+INSERT INTO files VALUES (74,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1363/comment()[1]/range-to(/1/2/1447/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (75,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/191/comment()[1]/range-to(/1/2/216/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (76,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2108/comment()[1]/range-to(/1/2/2166/comment()[1]))','text/html','FILE31');
+INSERT INTO files VALUES (77,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1447/comment()[1]/range-to(/1/2/1537/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (78,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1464/comment()[1]/range-to(/1/2/1537/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (79,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1307/comment()[1]/range-to(/1/2/1363/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (80,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2410/comment()[1]/range-to(/1/2/2474/comment()[1]))','text/html','FILE38');
+INSERT INTO files VALUES (81,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1560/comment()[1]/range-to(/1/2/1619/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (82,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/36/comment()[1]/range-to(/1/2/230/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (83,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/101/comment()[1]/range-to(/1/2/113/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (84,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (85,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/164/comment()[1]/range-to(/1/2/174/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (86,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/230/comment()[1]/range-to(/1/2/274/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (87,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/23/comment()[1]/range-to(/1/2/36/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (88,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/185/comment()[1]/range-to(/1/2/205/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (89,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/143/comment()[1]/range-to(/1/2/164/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm','text/html','default.1');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p60a.png','image/png','default.2');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61a.png','image/png','default.3');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61b.png','image/png','default.4');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61c.png','image/png','default.5');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61d.png','image/png','default.6');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p62a.png','image/png','default.7');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p62b.png','image/png','default.8');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p63a.png','image/png','default.9');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p63b.png','image/png','default.10');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64a.png','image/png','default.11');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64b.png','image/png','default.12');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64c.png','image/png','default.13');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64d.png','image/png','default.14');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65a.png','image/png','default.15');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65b.png','image/png','default.16');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65c.png','image/png','default.17');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65d.png','image/png','default.18');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p66a.png','image/png','default.19');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p66b.png','image/png','default.20');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p66c.png','image/png','default.21');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p67a.png','image/png','default.22');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p67b.png','image/png','default.23');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68a.png','image/png','default.24');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68b.png','image/png','default.25');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68c.png','image/png','default.26');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68d.png','image/png','default.27');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p69a.png','image/png','default.28');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p69b.png','image/png','default.29');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p69c.png','image/png','default.30');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p70a.png','image/png','default.31');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p70b.png','image/png','default.32');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p70c.png','image/png','default.33');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p71a.png','image/png','default.34');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p71b.png','image/png','default.35');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p71c.png','image/png','default.36');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p72a.png','image/png','default.37');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p72b.png','image/png','default.38');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p73a.png','image/png','default.39');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p73b.png','image/png','default.40');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p74a.png','image/png','default.41');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p74b.png','image/png','default.42');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p75a.png','image/png','default.43');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p75b.png','image/png','default.44');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p75c.png','image/png','default.45');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p76a.png','image/png','default.46');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p76b.png','image/png','default.47');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p76c.png','image/png','default.48');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p77a.png','image/png','default.49');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p77b.png','image/png','default.50');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p77c.png','image/png','default.51');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p78a.png','image/png','default.52');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p78b.png','image/png','default.53');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79a.png','image/png','default.54');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79b.png','image/png','default.55');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79c.png','image/png','default.56');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79d.png','image/png','default.57');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p80a.png','image/png','default.58');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p80b.png','image/png','default.59');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p81a.png','image/png','default.60');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p81b.png','image/png','default.61');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82a.png','image/png','default.62');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82b.png','image/png','default.63');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82c.png','image/png','default.64');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82d.png','image/png','default.65');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p83a.png','image/png','default.66');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84a.png','image/png','default.67');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84b.png','image/png','default.68');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84c.png','image/png','default.69');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84d.png','image/png','default.70');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p85a.png','image/png','default.71');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p85b.png','image/png','default.72');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p85c.png','image/png','default.73');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p86a.png','image/png','default.74');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p86b.png','image/png','default.75');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p86c.png','image/png','default.76');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p87a.png','image/png','default.77');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p87b.png','image/png','default.78');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p87c.png','image/png','default.79');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p88a.png','image/png','default.80');
+INSERT INTO files VALUES (91,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/48/comment()[1]/range-to(/1/2/101/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (92,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/246/comment()[1]/range-to(/1/2/274/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (93,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/218/comment()[1]/range-to(/1/2/230/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (94,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/7/comment()[1]/range-to(/1/2/36/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (95,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/399/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (96,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/213/comment()[1]/range-to(/1/2/218/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (97,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/117/comment()[1]/range-to(/1/2/143/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (98,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/178/comment()[1]/range-to(/1/2/185/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (99,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/274/comment()[1]/range-to(/1/2/340/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (100,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/231/comment()[1]/range-to(/1/2/246/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (101,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/174/comment()[1]/range-to(/1/2/178/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (102,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/340/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (103,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/113/comment()[1]/range-to(/1/2/117/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (104,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/205/comment()[1]/range-to(/1/2/213/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (105,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/172/comment()[1]/range-to(/1/2/183/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (106,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/66/comment()[1]/range-to(/1/2/73/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (107,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/225/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (108,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/106/comment()[1]/range-to(/1/2/115/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (109,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/200/comment()[1]/range-to(/1/2/225/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (110,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/7/comment()[1]/range-to(/1/2/12/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (111,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/139/comment()[1]/range-to(/1/2/147/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (112,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/100/comment()[1]/range-to(/1/2/106/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm','text/html','default.1');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p093a.png','image/png','default.2');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p094a.png','image/png','default.3');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p094b.png','image/png','default.4');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p095a.png','image/png','default.5');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p095b.png','image/png','default.6');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p095c.png','image/png','default.7');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p096a.png','image/png','default.8');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p096b.png','image/png','default.9');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p096c.png','image/png','default.10');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p097a.png','image/png','default.11');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p097b.png','image/png','default.12');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p098a.png','image/png','default.13');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p099a.png','image/png','default.14');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p099b.png','image/png','default.15');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p101a.png','image/png','default.16');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p101b.png','image/png','default.17');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p102a.png','image/png','default.18');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p103a.png','image/png','default.19');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p103b.png','image/png','default.20');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p104a.png','image/png','default.21');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p104b.png','image/png','default.22');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p105a.png','image/png','default.23');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p105b.png','image/png','default.24');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p105c.png','image/png','default.25');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106a.png','image/png','default.26');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106b.png','image/png','default.27');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106c.png','image/png','default.28');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106d.png','image/png','default.29');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p107a.png','image/png','default.30');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p107b.png','image/png','default.31');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p107c.png','image/png','default.32');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p108a.png','image/png','default.33');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p108b.png','image/png','default.34');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p109a.png','image/png','default.35');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p109b.png','image/png','default.36');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p110a.png','image/png','default.37');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p110b.png','image/png','default.38');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p111a.png','image/png','default.39');
+INSERT INTO files VALUES (114,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/38/comment()[1]/range-to(/1/2/66/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (115,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/163/comment()[1]/range-to(/1/2/200/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (116,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/164/comment()[1]/range-to(/1/2/172/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (117,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/13/comment()[1]/range-to(/1/2/38/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (118,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/244/comment()[1]/range-to(/1/2/258/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (119,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/131/comment()[1]/range-to(/1/2/163/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (120,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/80/comment()[1]/range-to(/1/2/100/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (121,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (122,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/73/comment()[1]/range-to(/1/2/131/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (123,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/229/comment()[1]/range-to(/1/2/244/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (124,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/183/comment()[1]/range-to(/1/2/200/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (125,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/115/comment()[1]/range-to(/1/2/131/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (126,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/258/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (127,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/12/comment()[1]/range-to(/1/2/73/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (128,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/147/comment()[1]/range-to(/1/2/163/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (129,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/41/comment()[1]/range-to(/1/2/51/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (130,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/91/comment()[1]/range-to(/1/2/93/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (131,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/21/comment()[1]/range-to(/1/2/29/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (132,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/275/comment()[1]/range-to(/1/2/281/comment()[1]))','text/html','FILE33');
+INSERT INTO files VALUES (133,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/254/comment()[1]/range-to(/1/2/267/comment()[1]))','text/html','FILE30');
+INSERT INTO files VALUES (134,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/65/comment()[1]/range-to(/1/2/78/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (135,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/241/comment()[1]/range-to(/1/2/244/comment()[1]))','text/html','FILE25');
+INSERT INTO files VALUES (136,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/347/comment()[1]/range-to(/1/2/353/comment()[1]))','text/html','FILE41');
+INSERT INTO files VALUES (137,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/247/comment()[1]/range-to(/1/2/251/comment()[1]))','text/html','FILE28');
+INSERT INTO files VALUES (138,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/297/comment()[1]/range-to(/1/2/311/comment()[1]))','text/html','FILE36');
+INSERT INTO files VALUES (139,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/115/comment()[1]/range-to(/1/2/138/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (140,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE46');
+INSERT INTO files VALUES (141,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/229/comment()[1]/range-to(/1/2/241/comment()[1]))','text/html','FILE24');
+INSERT INTO files VALUES (142,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/284/comment()[1]/range-to(/1/2/297/comment()[1]))','text/html','FILE35');
+INSERT INTO files VALUES (143,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/56/comment()[1]/range-to(/1/2/65/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (144,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/102/comment()[1]/range-to(/1/2/115/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (145,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/138/comment()[1]/range-to(/1/2/244/comment()[1]))','text/html','FILE26');
+INSERT INTO files VALUES (146,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/88/comment()[1]/range-to(/1/2/90/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (147,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/269/comment()[1]/range-to(/1/2/275/comment()[1]))','text/html','FILE32');
+INSERT INTO files VALUES (148,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/281/comment()[1]/range-to(/1/2/284/comment()[1]))','text/html','FILE34');
+INSERT INTO files VALUES (149,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/245/comment()[1]/range-to(/1/2/247/comment()[1]))','text/html','FILE27');
+INSERT INTO files VALUES (150,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/93/comment()[1]/range-to(/1/2/102/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (151,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/51/comment()[1]/range-to(/1/2/56/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (152,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/90/comment()[1]/range-to(/1/2/138/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (153,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/78/comment()[1]/range-to(/1/2/88/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (154,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/208/comment()[1]/range-to(/1/2/218/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (155,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/339/comment()[1]/range-to(/1/2/347/comment()[1]))','text/html','FILE40');
+INSERT INTO files VALUES (156,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/251/comment()[1]/range-to(/1/2/254/comment()[1]))','text/html','FILE29');
+INSERT INTO files VALUES (157,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/150/comment()[1]/range-to(/1/2/176/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (158,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/357/comment()[1]/range-to(/1/2/366/comment()[1]))','text/html','FILE44');
+INSERT INTO files VALUES (159,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/311/comment()[1]/range-to(/1/2/328/comment()[1]))','text/html','FILE37');
+INSERT INTO files VALUES (160,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/36/comment()[1]/range-to(/1/2/41/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm','text/html','default.1');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p16.png','image/png','default.2');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p17.png','image/png','default.3');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p18.png','image/png','default.4');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p19.png','image/png','default.5');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p20.png','image/png','default.6');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p23.png','image/png','default.7');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p24.png','image/png','default.8');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p28.png','image/png','default.9');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p29a.png','image/png','default.10');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p29b.png','image/png','default.11');
+INSERT INTO files VALUES (162,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/142/comment()[1]/range-to(/1/2/150/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (163,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/267/comment()[1]/range-to(/1/2/269/comment()[1]))','text/html','FILE31');
+INSERT INTO files VALUES (164,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/29/comment()[1]/range-to(/1/2/35/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (165,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/353/comment()[1]/range-to(/1/2/357/comment()[1]))','text/html','FILE42');
+INSERT INTO files VALUES (166,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/332/comment()[1]/range-to(/1/2/339/comment()[1]))','text/html','FILE39');
+INSERT INTO files VALUES (167,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/182/comment()[1]/range-to(/1/2/197/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (168,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/218/comment()[1]/range-to(/1/2/229/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (169,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/35/comment()[1]/range-to(/1/2/88/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (170,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/139/comment()[1]/range-to(/1/2/142/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (171,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/176/comment()[1]/range-to(/1/2/182/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (172,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/244/comment()[1]/range-to(/1/2/357/comment()[1]))','text/html','FILE43');
+INSERT INTO files VALUES (173,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/328/comment()[1]/range-to(/1/2/332/comment()[1]))','text/html','FILE38');
+INSERT INTO files VALUES (174,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/366/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE45');
+INSERT INTO files VALUES (175,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/197/comment()[1]/range-to(/1/2/208/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (176,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/metadata.xml','application/xml','default.1');
+
+--
+-- Table structure for table `filesection`
+--
+
+CREATE TABLE filesection (
+  FileSectionRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  FileSecID varchar(64) default NULL,
+  PRIMARY KEY  (FileSectionRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `filesection`
+--
+
+INSERT INTO filesection VALUES (1,'20050201:1','test');
+INSERT INTO filesection VALUES (2,'20050201:2','test');
+INSERT INTO filesection VALUES (3,'20050201:3','test');
+INSERT INTO filesection VALUES (4,'20050201:4','test');
+INSERT INTO filesection VALUES (5,'20050201:5','test');
+INSERT INTO filesection VALUES (6,'20050201:6','test');
+
+--
+-- Table structure for table `mdgroups`
+--
+
+CREATE TABLE mdgroups (
+  NamespaceRef int(11) default NULL,
+  MdGroupRef int(11) NOT NULL auto_increment,
+  MdGroupID varchar(32) default NULL,
+  PRIMARY KEY  (MdGroupRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `mdgroups`
+--
+
+
+--
+-- Table structure for table `mdqualifiers`
+--
+
+CREATE TABLE mdqualifiers (
+  MdGroupRef int(11) default NULL,
+  QualifierName varchar(64) default NULL,
+  QualifierValue varchar(64) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `mdqualifiers`
+--
+
+
+--
+-- Table structure for table `mdvalues`
+--
+
+CREATE TABLE mdvalues (
+  NamespaceRef int(11) default NULL,
+  GroupRef int(11) default NULL,
+  Label tinytext,
+  Value tinytext
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `mdvalues`
+--
+
+INSERT INTO mdvalues VALUES (1,NULL,'mgseqno','mgsearch.20');
+INSERT INTO mdvalues VALUES (2,NULL,'mgseqno','mgsearch.26');
+INSERT INTO mdvalues VALUES (3,NULL,'mgseqno','mgsearch.13');
+INSERT INTO mdvalues VALUES (4,NULL,'mgseqno','mgsearch.5');
+INSERT INTO mdvalues VALUES (6,NULL,'mgseqno','mgsearch.7');
+INSERT INTO mdvalues VALUES (7,NULL,'mgseqno','mgsearch.21');
+INSERT INTO mdvalues VALUES (11,NULL,'mgseqno','mgsearch.3');
+INSERT INTO mdvalues VALUES (14,NULL,'mgseqno','mgsearch.23');
+INSERT INTO mdvalues VALUES (21,NULL,'mgseqno','mgsearch.11');
+INSERT INTO mdvalues VALUES (20,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (23,NULL,'mgseqno','mgsearch.19');
+INSERT INTO mdvalues VALUES (24,NULL,'mgseqno','mgsearch.12');
+INSERT INTO mdvalues VALUES (25,NULL,'mgseqno','mgsearch.18');
+INSERT INTO mdvalues VALUES (27,NULL,'mgseqno','mgsearch.8');
+INSERT INTO mdvalues VALUES (29,NULL,'mgseqno','mgsearch.27');
+INSERT INTO mdvalues VALUES (30,NULL,'mgseqno','mgsearch.58');
+INSERT INTO mdvalues VALUES (31,NULL,'mgseqno','mgsearch.66');
+INSERT INTO mdvalues VALUES (35,NULL,'mgseqno','mgsearch.75');
+INSERT INTO mdvalues VALUES (36,NULL,'mgseqno','mgsearch.37');
+INSERT INTO mdvalues VALUES (37,NULL,'mgseqno','mgsearch.59');
+INSERT INTO mdvalues VALUES (42,NULL,'mgseqno','mgsearch.74');
+INSERT INTO mdvalues VALUES (44,NULL,'mgseqno','mgsearch.33');
+INSERT INTO mdvalues VALUES (45,NULL,'mgseqno','mgsearch.76');
+INSERT INTO mdvalues VALUES (47,NULL,'mgseqno','mgsearch.45');
+INSERT INTO mdvalues VALUES (51,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (51,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (51,NULL,'classified','BOSTID');
+INSERT INTO mdvalues VALUES (52,NULL,'mgseqno','mgsearch.64');
+INSERT INTO mdvalues VALUES (53,NULL,'mgseqno','mgsearch.36');
+INSERT INTO mdvalues VALUES (54,NULL,'mgseqno','mgsearch.41');
+INSERT INTO mdvalues VALUES (55,NULL,'mgseqno','mgsearch.72');
+INSERT INTO mdvalues VALUES (57,NULL,'mgseqno','mgsearch.69');
+INSERT INTO mdvalues VALUES (58,NULL,'mgseqno','mgsearch.70');
+INSERT INTO mdvalues VALUES (59,NULL,'mgseqno','mgsearch.65');
+INSERT INTO mdvalues VALUES (63,NULL,'mgseqno','mgsearch.73');
+INSERT INTO mdvalues VALUES (68,NULL,'mgseqno','mgsearch.53');
+INSERT INTO mdvalues VALUES (70,NULL,'mgseqno','mgsearch.52');
+INSERT INTO mdvalues VALUES (71,NULL,'mgseqno','mgsearch.51');
+INSERT INTO mdvalues VALUES (76,NULL,'mgseqno','mgsearch.80');
+INSERT INTO mdvalues VALUES (77,NULL,'mgseqno','mgsearch.79');
+INSERT INTO mdvalues VALUES (81,NULL,'mgseqno','mgsearch.50');
+INSERT INTO mdvalues VALUES (82,NULL,'mgseqno','mgsearch.100');
+INSERT INTO mdvalues VALUES (84,NULL,'mgseqno','mgsearch.95');
+INSERT INTO mdvalues VALUES (86,NULL,'mgseqno','mgsearch.87');
+INSERT INTO mdvalues VALUES (87,NULL,'mgseqno','mgsearch.91');
+INSERT INTO mdvalues VALUES (91,NULL,'mgseqno','mgsearch.90');
+INSERT INTO mdvalues VALUES (90,NULL,'mgseqno','mgsearch.94');
+INSERT INTO mdvalues VALUES (93,NULL,'mgseqno','mgsearch.104');
+INSERT INTO mdvalues VALUES (94,NULL,'mgseqno','mgsearch.105');
+INSERT INTO mdvalues VALUES (99,NULL,'mgseqno','mgsearch.89');
+INSERT INTO mdvalues VALUES (101,NULL,'mgseqno','mgsearch.92');
+INSERT INTO mdvalues VALUES (102,NULL,'mgseqno','mgsearch.88');
+INSERT INTO mdvalues VALUES (105,NULL,'mgseqno','mgsearch.131');
+INSERT INTO mdvalues VALUES (106,NULL,'mgseqno','mgsearch.124');
+INSERT INTO mdvalues VALUES (107,NULL,'mgseqno','mgsearch.115');
+INSERT INTO mdvalues VALUES (109,NULL,'mgseqno','mgsearch.128');
+INSERT INTO mdvalues VALUES (112,NULL,'mgseqno','mgsearch.125');
+INSERT INTO mdvalues VALUES (113,NULL,'mgseqno','mgsearch.113');
+INSERT INTO mdvalues VALUES (116,NULL,'mgseqno','mgsearch.120');
+INSERT INTO mdvalues VALUES (117,NULL,'mgseqno','mgsearch.127');
+INSERT INTO mdvalues VALUES (118,NULL,'mgseqno','mgsearch.123');
+INSERT INTO mdvalues VALUES (119,NULL,'mgseqno','mgsearch.130');
+INSERT INTO mdvalues VALUES (120,NULL,'mgseqno','mgsearch.122');
+INSERT INTO mdvalues VALUES (125,NULL,'mgseqno','mgsearch.129');
+INSERT INTO mdvalues VALUES (126,NULL,'mgseqno','mgsearch.112');
+INSERT INTO mdvalues VALUES (127,NULL,'mgseqno','mgsearch.117');
+INSERT INTO mdvalues VALUES (131,NULL,'mgseqno','mgsearch.164');
+INSERT INTO mdvalues VALUES (132,NULL,'mgseqno','mgsearch.146');
+INSERT INTO mdvalues VALUES (135,NULL,'mgseqno','mgsearch.137');
+INSERT INTO mdvalues VALUES (140,NULL,'mgseqno','mgsearch.175');
+INSERT INTO mdvalues VALUES (143,NULL,'mgseqno','mgsearch.142');
+INSERT INTO mdvalues VALUES (144,NULL,'mgseqno','mgsearch.149');
+INSERT INTO mdvalues VALUES (146,NULL,'mgseqno','mgsearch.155');
+INSERT INTO mdvalues VALUES (148,NULL,'mgseqno','mgsearch.168');
+INSERT INTO mdvalues VALUES (149,NULL,'mgseqno','mgsearch.136');
+INSERT INTO mdvalues VALUES (150,NULL,'mgseqno','mgsearch.156');
+INSERT INTO mdvalues VALUES (154,NULL,'mgseqno','mgsearch.159');
+INSERT INTO mdvalues VALUES (155,NULL,'mgseqno','mgsearch.161');
+INSERT INTO mdvalues VALUES (156,NULL,'classified','achieve gender equality');
+INSERT INTO mdvalues VALUES (156,NULL,'classified','Women, gender and development, women\'s organizations');
+INSERT INTO mdvalues VALUES (159,NULL,'mgseqno','mgsearch.158');
+INSERT INTO mdvalues VALUES (162,NULL,'mgseqno','mgsearch.151');
+INSERT INTO mdvalues VALUES (165,NULL,'mgseqno','mgsearch.154');
+INSERT INTO mdvalues VALUES (169,NULL,'mgseqno','mgsearch.141');
+INSERT INTO mdvalues VALUES (170,NULL,'mgseqno','mgsearch.171');
+INSERT INTO mdvalues VALUES (171,NULL,'mgseqno','mgsearch.157');
+INSERT INTO mdvalues VALUES (172,NULL,'mgseqno','mgsearch.162');
+INSERT INTO mdvalues VALUES (174,NULL,'mgseqno','mgsearch.169');
+INSERT INTO mdvalues VALUES (175,NULL,'mgseqno','mgsearch.143');
+INSERT INTO mdvalues VALUES (51,NULL,'classified','introduce small animals and micro-livestock in your farm');
+INSERT INTO mdvalues VALUES (51,NULL,'Subject','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (21,NULL,'mgseqno','mgsearch.11');
+INSERT INTO mdvalues VALUES (20,NULL,'classified','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (20,NULL,'classified','introduce little-known Asian farm animals with a promising future');
+INSERT INTO mdvalues VALUES (20,NULL,'classified','BOSTID');
+INSERT INTO mdvalues VALUES (20,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (111,NULL,'Subject','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (111,NULL,'Subject','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (89,NULL,'classified','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (90,NULL,'mgseqno','mgsearch.94');
+INSERT INTO mdvalues VALUES (89,NULL,'classified','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (89,NULL,'mgseqno','mgsearch.84');
+INSERT INTO mdvalues VALUES (89,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (156,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (156,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (156,NULL,'mgseqno','mgsearch.133');
+INSERT INTO mdvalues VALUES (156,NULL,'Title','Development in practice: Toward Gender Equality (wb34te)');
+INSERT INTO mdvalues VALUES (20,NULL,'mgseqno','mgsearch.1');
+INSERT INTO mdvalues VALUES (20,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (51,NULL,'classified','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (89,NULL,'Title','Farming snails 1: Learning about snails; Building a pen; Food and shelter plants (fb33fe)');
+INSERT INTO mdvalues VALUES (111,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','farm snails');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (156,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (20,NULL,'Title','Little Known Asian Animals With a Promising Economic Future (b18ase)');
+INSERT INTO mdvalues VALUES (51,NULL,'Title','Microlivestock - Little-Known Small Animals with a Promising Economic Future (b17mie)');
+INSERT INTO mdvalues VALUES (51,NULL,'mgseqno','mgsearch.31');
+INSERT INTO mdvalues VALUES (89,NULL,'Subject','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (111,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (111,NULL,'mgseqno','mgsearch.108');
+INSERT INTO mdvalues VALUES (20,NULL,'Organization','BOSTID');
+INSERT INTO mdvalues VALUES (20,NULL,'Subject','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (51,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (51,NULL,'Organization','BOSTID');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (111,NULL,'Organization','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (156,NULL,'Organization','World Bank');
+INSERT INTO mdvalues VALUES (156,NULL,'Subject','Women, gender and development, women\'s organizations');
+INSERT INTO mdvalues VALUES (20,NULL,'Keyword','introduce little-known Asian farm animals with a promising future');
+INSERT INTO mdvalues VALUES (51,NULL,'Keyword','introduce small animals and micro-livestock in your farm');
+INSERT INTO mdvalues VALUES (89,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (89,NULL,'Subject','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (111,NULL,'Title','Farming snails 2: Choosing snails; Care and harvesting; Further improvement (fb34fe)');
+INSERT INTO mdvalues VALUES (1,NULL,'mgseqno','mgsearch.20');
+INSERT INTO mdvalues VALUES (2,NULL,'mgseqno','mgsearch.26');
+INSERT INTO mdvalues VALUES (3,NULL,'mgseqno','mgsearch.13');
+INSERT INTO mdvalues VALUES (5,NULL,'mgseqno','mgsearch.25');
+INSERT INTO mdvalues VALUES (4,NULL,'mgseqno','mgsearch.5');
+INSERT INTO mdvalues VALUES (6,NULL,'mgseqno','mgsearch.7');
+INSERT INTO mdvalues VALUES (7,NULL,'mgseqno','mgsearch.21');
+INSERT INTO mdvalues VALUES (8,NULL,'mgseqno','mgsearch.14');
+INSERT INTO mdvalues VALUES (9,NULL,'mgseqno','mgsearch.2');
+INSERT INTO mdvalues VALUES (11,NULL,'mgseqno','mgsearch.3');
+INSERT INTO mdvalues VALUES (10,NULL,'mgseqno','mgsearch.29');
+INSERT INTO mdvalues VALUES (12,NULL,'mgseqno','mgsearch.9');
+INSERT INTO mdvalues VALUES (13,NULL,'mgseqno','mgsearch.16');
+INSERT INTO mdvalues VALUES (14,NULL,'mgseqno','mgsearch.23');
+INSERT INTO mdvalues VALUES (15,NULL,'mgseqno','mgsearch.4');
+INSERT INTO mdvalues VALUES (16,NULL,'mgseqno','mgsearch.28');
+INSERT INTO mdvalues VALUES (17,NULL,'mgseqno','mgsearch.17');
+INSERT INTO mdvalues VALUES (18,NULL,'mgseqno','mgsearch.24');
+INSERT INTO mdvalues VALUES (19,NULL,'mgseqno','mgsearch.10');
+INSERT INTO mdvalues VALUES (22,NULL,'mgseqno','mgsearch.6');
+INSERT INTO mdvalues VALUES (23,NULL,'mgseqno','mgsearch.19');
+INSERT INTO mdvalues VALUES (26,NULL,'mgseqno','mgsearch.22');
+INSERT INTO mdvalues VALUES (24,NULL,'mgseqno','mgsearch.12');
+INSERT INTO mdvalues VALUES (25,NULL,'mgseqno','mgsearch.18');
+INSERT INTO mdvalues VALUES (27,NULL,'mgseqno','mgsearch.8');
+INSERT INTO mdvalues VALUES (28,NULL,'mgseqno','mgsearch.15');
+INSERT INTO mdvalues VALUES (29,NULL,'mgseqno','mgsearch.27');
+INSERT INTO mdvalues VALUES (30,NULL,'mgseqno','mgsearch.58');
+INSERT INTO mdvalues VALUES (31,NULL,'mgseqno','mgsearch.66');
+INSERT INTO mdvalues VALUES (32,NULL,'mgseqno','mgsearch.42');
+INSERT INTO mdvalues VALUES (33,NULL,'mgseqno','mgsearch.56');
+INSERT INTO mdvalues VALUES (34,NULL,'mgseqno','mgsearch.35');
+INSERT INTO mdvalues VALUES (35,NULL,'mgseqno','mgsearch.75');
+INSERT INTO mdvalues VALUES (36,NULL,'mgseqno','mgsearch.37');
+INSERT INTO mdvalues VALUES (37,NULL,'mgseqno','mgsearch.59');
+INSERT INTO mdvalues VALUES (38,NULL,'mgseqno','mgsearch.71');
+INSERT INTO mdvalues VALUES (39,NULL,'mgseqno','mgsearch.43');
+INSERT INTO mdvalues VALUES (40,NULL,'mgseqno','mgsearch.77');
+INSERT INTO mdvalues VALUES (41,NULL,'mgseqno','mgsearch.82');
+INSERT INTO mdvalues VALUES (42,NULL,'mgseqno','mgsearch.74');
+INSERT INTO mdvalues VALUES (43,NULL,'mgseqno','mgsearch.81');
+INSERT INTO mdvalues VALUES (45,NULL,'mgseqno','mgsearch.76');
+INSERT INTO mdvalues VALUES (44,NULL,'mgseqno','mgsearch.33');
+INSERT INTO mdvalues VALUES (46,NULL,'mgseqno','mgsearch.49');
+INSERT INTO mdvalues VALUES (47,NULL,'mgseqno','mgsearch.45');
+INSERT INTO mdvalues VALUES (48,NULL,'mgseqno','mgsearch.57');
+INSERT INTO mdvalues VALUES (49,NULL,'mgseqno','mgsearch.34');
+INSERT INTO mdvalues VALUES (50,NULL,'mgseqno','mgsearch.40');
+INSERT INTO mdvalues VALUES (52,NULL,'mgseqno','mgsearch.64');
+INSERT INTO mdvalues VALUES (53,NULL,'mgseqno','mgsearch.36');
+INSERT INTO mdvalues VALUES (54,NULL,'mgseqno','mgsearch.41');
+INSERT INTO mdvalues VALUES (55,NULL,'mgseqno','mgsearch.72');
+INSERT INTO mdvalues VALUES (56,NULL,'mgseqno','mgsearch.63');
+INSERT INTO mdvalues VALUES (57,NULL,'mgseqno','mgsearch.69');
+INSERT INTO mdvalues VALUES (58,NULL,'mgseqno','mgsearch.70');
+INSERT INTO mdvalues VALUES (59,NULL,'mgseqno','mgsearch.65');
+INSERT INTO mdvalues VALUES (60,NULL,'mgseqno','mgsearch.32');
+INSERT INTO mdvalues VALUES (61,NULL,'mgseqno','mgsearch.39');
+INSERT INTO mdvalues VALUES (62,NULL,'mgseqno','mgsearch.60');
+INSERT INTO mdvalues VALUES (63,NULL,'mgseqno','mgsearch.73');
+INSERT INTO mdvalues VALUES (64,NULL,'mgseqno','mgsearch.48');
+INSERT INTO mdvalues VALUES (65,NULL,'mgseqno','mgsearch.46');
+INSERT INTO mdvalues VALUES (66,NULL,'mgseqno','mgsearch.62');
+INSERT INTO mdvalues VALUES (67,NULL,'mgseqno','mgsearch.78');
+INSERT INTO mdvalues VALUES (68,NULL,'mgseqno','mgsearch.53');
+INSERT INTO mdvalues VALUES (69,NULL,'mgseqno','mgsearch.47');
+INSERT INTO mdvalues VALUES (70,NULL,'mgseqno','mgsearch.52');
+INSERT INTO mdvalues VALUES (71,NULL,'mgseqno','mgsearch.51');
+INSERT INTO mdvalues VALUES (72,NULL,'mgseqno','mgsearch.55');
+INSERT INTO mdvalues VALUES (73,NULL,'mgseqno','mgsearch.38');
+INSERT INTO mdvalues VALUES (74,NULL,'mgseqno','mgsearch.44');
+INSERT INTO mdvalues VALUES (75,NULL,'mgseqno','mgsearch.68');
+INSERT INTO mdvalues VALUES (76,NULL,'mgseqno','mgsearch.80');
+INSERT INTO mdvalues VALUES (77,NULL,'mgseqno','mgsearch.79');
+INSERT INTO mdvalues VALUES (78,NULL,'mgseqno','mgsearch.54');
+INSERT INTO mdvalues VALUES (80,NULL,'mgseqno','mgsearch.67');
+INSERT INTO mdvalues VALUES (79,NULL,'mgseqno','mgsearch.61');
+INSERT INTO mdvalues VALUES (81,NULL,'mgseqno','mgsearch.50');
+INSERT INTO mdvalues VALUES (82,NULL,'mgseqno','mgsearch.100');
+INSERT INTO mdvalues VALUES (83,NULL,'mgseqno','mgsearch.106');
+INSERT INTO mdvalues VALUES (84,NULL,'mgseqno','mgsearch.95');
+INSERT INTO mdvalues VALUES (85,NULL,'mgseqno','mgsearch.101');
+INSERT INTO mdvalues VALUES (86,NULL,'mgseqno','mgsearch.87');
+INSERT INTO mdvalues VALUES (87,NULL,'mgseqno','mgsearch.91');
+INSERT INTO mdvalues VALUES (88,NULL,'mgseqno','mgsearch.99');
+INSERT INTO mdvalues VALUES (89,NULL,'Organization','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (90,NULL,'Title','Dew and rain');
+INSERT INTO mdvalues VALUES (91,NULL,'mgseqno','mgsearch.90');
+INSERT INTO mdvalues VALUES (92,NULL,'mgseqno','mgsearch.85');
+INSERT INTO mdvalues VALUES (93,NULL,'mgseqno','mgsearch.104');
+INSERT INTO mdvalues VALUES (94,NULL,'mgseqno','mgsearch.105');
+INSERT INTO mdvalues VALUES (95,NULL,'mgseqno','mgsearch.102');
+INSERT INTO mdvalues VALUES (96,NULL,'mgseqno','mgsearch.86');
+INSERT INTO mdvalues VALUES (97,NULL,'mgseqno','mgsearch.93');
+INSERT INTO mdvalues VALUES (98,NULL,'mgseqno','mgsearch.97');
+INSERT INTO mdvalues VALUES (99,NULL,'mgseqno','mgsearch.89');
+INSERT INTO mdvalues VALUES (100,NULL,'mgseqno','mgsearch.103');
+INSERT INTO mdvalues VALUES (101,NULL,'mgseqno','mgsearch.92');
+INSERT INTO mdvalues VALUES (103,NULL,'mgseqno','mgsearch.96');
+INSERT INTO mdvalues VALUES (102,NULL,'mgseqno','mgsearch.88');
+INSERT INTO mdvalues VALUES (104,NULL,'mgseqno','mgsearch.98');
+INSERT INTO mdvalues VALUES (105,NULL,'mgseqno','mgsearch.131');
+INSERT INTO mdvalues VALUES (106,NULL,'mgseqno','mgsearch.124');
+INSERT INTO mdvalues VALUES (107,NULL,'mgseqno','mgsearch.115');
+INSERT INTO mdvalues VALUES (108,NULL,'mgseqno','mgsearch.114');
+INSERT INTO mdvalues VALUES (109,NULL,'mgseqno','mgsearch.128');
+INSERT INTO mdvalues VALUES (110,NULL,'mgseqno','mgsearch.121');
+INSERT INTO mdvalues VALUES (111,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (111,NULL,'Keyword','farm snails');
+INSERT INTO mdvalues VALUES (112,NULL,'mgseqno','mgsearch.125');
+INSERT INTO mdvalues VALUES (113,NULL,'mgseqno','mgsearch.113');
+INSERT INTO mdvalues VALUES (114,NULL,'mgseqno','mgsearch.118');
+INSERT INTO mdvalues VALUES (115,NULL,'mgseqno','mgsearch.109');
+INSERT INTO mdvalues VALUES (116,NULL,'mgseqno','mgsearch.120');
+INSERT INTO mdvalues VALUES (118,NULL,'mgseqno','mgsearch.123');
+INSERT INTO mdvalues VALUES (117,NULL,'mgseqno','mgsearch.127');
+INSERT INTO mdvalues VALUES (121,NULL,'mgseqno','mgsearch.116');
+INSERT INTO mdvalues VALUES (120,NULL,'mgseqno','mgsearch.122');
+INSERT INTO mdvalues VALUES (119,NULL,'mgseqno','mgsearch.130');
+INSERT INTO mdvalues VALUES (123,NULL,'mgseqno','mgsearch.110');
+INSERT INTO mdvalues VALUES (122,NULL,'mgseqno','mgsearch.126');
+INSERT INTO mdvalues VALUES (124,NULL,'mgseqno','mgsearch.119');
+INSERT INTO mdvalues VALUES (125,NULL,'mgseqno','mgsearch.129');
+INSERT INTO mdvalues VALUES (126,NULL,'mgseqno','mgsearch.112');
+INSERT INTO mdvalues VALUES (127,NULL,'mgseqno','mgsearch.117');
+INSERT INTO mdvalues VALUES (128,NULL,'mgseqno','mgsearch.111');
+INSERT INTO mdvalues VALUES (130,NULL,'mgseqno','mgsearch.152');
+INSERT INTO mdvalues VALUES (129,NULL,'mgseqno','mgsearch.166');
+INSERT INTO mdvalues VALUES (132,NULL,'mgseqno','mgsearch.146');
+INSERT INTO mdvalues VALUES (131,NULL,'mgseqno','mgsearch.164');
+INSERT INTO mdvalues VALUES (135,NULL,'mgseqno','mgsearch.137');
+INSERT INTO mdvalues VALUES (134,NULL,'mgseqno','mgsearch.176');
+INSERT INTO mdvalues VALUES (133,NULL,'mgseqno','mgsearch.178');
+INSERT INTO mdvalues VALUES (137,NULL,'mgseqno','mgsearch.153');
+INSERT INTO mdvalues VALUES (136,NULL,'mgseqno','mgsearch.167');
+INSERT INTO mdvalues VALUES (138,NULL,'mgseqno','mgsearch.147');
+INSERT INTO mdvalues VALUES (139,NULL,'mgseqno','mgsearch.134');
+INSERT INTO mdvalues VALUES (140,NULL,'mgseqno','mgsearch.175');
+INSERT INTO mdvalues VALUES (141,NULL,'mgseqno','mgsearch.177');
+INSERT INTO mdvalues VALUES (142,NULL,'mgseqno','mgsearch.135');
+INSERT INTO mdvalues VALUES (143,NULL,'mgseqno','mgsearch.142');
+INSERT INTO mdvalues VALUES (144,NULL,'mgseqno','mgsearch.149');
+INSERT INTO mdvalues VALUES (145,NULL,'mgseqno','mgsearch.139');
+INSERT INTO mdvalues VALUES (146,NULL,'mgseqno','mgsearch.155');
+INSERT INTO mdvalues VALUES (147,NULL,'mgseqno','mgsearch.165');
+INSERT INTO mdvalues VALUES (148,NULL,'mgseqno','mgsearch.168');
+INSERT INTO mdvalues VALUES (149,NULL,'mgseqno','mgsearch.136');
+INSERT INTO mdvalues VALUES (150,NULL,'mgseqno','mgsearch.156');
+INSERT INTO mdvalues VALUES (151,NULL,'mgseqno','mgsearch.179');
+INSERT INTO mdvalues VALUES (152,NULL,'mgseqno','mgsearch.173');
+INSERT INTO mdvalues VALUES (154,NULL,'mgseqno','mgsearch.159');
+INSERT INTO mdvalues VALUES (153,NULL,'mgseqno','mgsearch.170');
+INSERT INTO mdvalues VALUES (155,NULL,'mgseqno','mgsearch.161');
+INSERT INTO mdvalues VALUES (156,NULL,'classified','World Bank');
+INSERT INTO mdvalues VALUES (157,NULL,'mgseqno','mgsearch.172');
+INSERT INTO mdvalues VALUES (160,NULL,'mgseqno','mgsearch.140');
+INSERT INTO mdvalues VALUES (159,NULL,'mgseqno','mgsearch.158');
+INSERT INTO mdvalues VALUES (158,NULL,'mgseqno','mgsearch.160');
+INSERT INTO mdvalues VALUES (161,NULL,'mgseqno','mgsearch.144');
+INSERT INTO mdvalues VALUES (162,NULL,'mgseqno','mgsearch.151');
+INSERT INTO mdvalues VALUES (165,NULL,'mgseqno','mgsearch.154');
+INSERT INTO mdvalues VALUES (164,NULL,'mgseqno','mgsearch.150');
+INSERT INTO mdvalues VALUES (163,NULL,'mgseqno','mgsearch.145');
+INSERT INTO mdvalues VALUES (167,NULL,'mgseqno','mgsearch.138');
+INSERT INTO mdvalues VALUES (166,NULL,'mgseqno','mgsearch.163');
+INSERT INTO mdvalues VALUES (168,NULL,'mgseqno','mgsearch.148');
+INSERT INTO mdvalues VALUES (169,NULL,'mgseqno','mgsearch.141');
+INSERT INTO mdvalues VALUES (170,NULL,'mgseqno','mgsearch.171');
+INSERT INTO mdvalues VALUES (171,NULL,'mgseqno','mgsearch.157');
+INSERT INTO mdvalues VALUES (172,NULL,'mgseqno','mgsearch.162');
+INSERT INTO mdvalues VALUES (174,NULL,'mgseqno','mgsearch.169');
+INSERT INTO mdvalues VALUES (173,NULL,'mgseqno','mgsearch.174');
+INSERT INTO mdvalues VALUES (175,NULL,'mgseqno','mgsearch.143');
+INSERT INTO mdvalues VALUES (1,NULL,'Title','12 The Sulawesi Warty Pig');
+INSERT INTO mdvalues VALUES (2,NULL,'Title','Selected Readings');
+INSERT INTO mdvalues VALUES (3,NULL,'Title','6  Wild Banteng');
+INSERT INTO mdvalues VALUES (5,NULL,'mgseqno','mgsearch.25');
+INSERT INTO mdvalues VALUES (5,NULL,'Title','Appendixes');
+INSERT INTO mdvalues VALUES (4,NULL,'Title','Introduction');
+INSERT INTO mdvalues VALUES (6,NULL,'Title','1 Domesticated Banteng');
+INSERT INTO mdvalues VALUES (7,NULL,'Title','13 Javan Warty Pig');
+INSERT INTO mdvalues VALUES (8,NULL,'mgseqno','mgsearch.14');
+INSERT INTO mdvalues VALUES (8,NULL,'Title','7 Gaur');
+INSERT INTO mdvalues VALUES (9,NULL,'Title','Little Known Asian Animals With a Promising Economic Future');
+INSERT INTO mdvalues VALUES (11,NULL,'Title','Acknowledgments');
+INSERT INTO mdvalues VALUES (10,NULL,'Title','Board on Science and Technology for International Development');
+INSERT INTO mdvalues VALUES (12,NULL,'mgseqno','mgsearch.9');
+INSERT INTO mdvalues VALUES (12,NULL,'Title','3 Mithan');
+INSERT INTO mdvalues VALUES (13,NULL,'mgseqno','mgsearch.16');
+INSERT INTO mdvalues VALUES (13,NULL,'Title','9 Tamaraw');
+INSERT INTO mdvalues VALUES (14,NULL,'Title','15 The Babirusa');
+INSERT INTO mdvalues VALUES (15,NULL,'mgseqno','mgsearch.4');
+INSERT INTO mdvalues VALUES (15,NULL,'Title','Preface');
+INSERT INTO mdvalues VALUES (16,NULL,'Title','Advisory Committee on Technology Innovation');
+INSERT INTO mdvalues VALUES (17,NULL,'mgseqno','mgsearch.17');
+INSERT INTO mdvalues VALUES (17,NULL,'Title','10 Anoas');
+INSERT INTO mdvalues VALUES (18,NULL,'Title','Part IV : Recommendations and  General Research  Needs');
+INSERT INTO mdvalues VALUES (19,NULL,'mgseqno','mgsearch.10');
+INSERT INTO mdvalues VALUES (19,NULL,'Title','4 Yak');
+INSERT INTO mdvalues VALUES (21,NULL,'mgseqno','mgsearch.11');
+INSERT INTO mdvalues VALUES (21,NULL,'Title','5 Yakows');
+INSERT INTO mdvalues VALUES (22,NULL,'Title','Part I : Domesticated Bovine Species');
+INSERT INTO mdvalues VALUES (23,NULL,'Title','11 The Bearded Pig');
+INSERT INTO mdvalues VALUES (26,NULL,'mgseqno','mgsearch.22');
+INSERT INTO mdvalues VALUES (26,NULL,'Title','14 Pigmy Hog');
+INSERT INTO mdvalues VALUES (24,NULL,'Title','Part II : Wild Bovine Species');
+INSERT INTO mdvalues VALUES (25,NULL,'Title','Part IlI : Pig and Piglike Species');
+INSERT INTO mdvalues VALUES (27,NULL,'Title','2 Banteng Cattle Hybrids');
+INSERT INTO mdvalues VALUES (28,NULL,'mgseqno','mgsearch.15');
+INSERT INTO mdvalues VALUES (28,NULL,'Title','8 Kouprey');
+INSERT INTO mdvalues VALUES (29,NULL,'Title','Research Contacts');
+INSERT INTO mdvalues VALUES (30,NULL,'Title','19 Grasscutter');
+INSERT INTO mdvalues VALUES (31,NULL,'Title','26 Mouse Deer');
+INSERT INTO mdvalues VALUES (32,NULL,'mgseqno','mgsearch.42');
+INSERT INTO mdvalues VALUES (32,NULL,'Title','5 Chicken');
+INSERT INTO mdvalues VALUES (33,NULL,'mgseqno','mgsearch.56');
+INSERT INTO mdvalues VALUES (33,NULL,'Title','17 Coypu');
+INSERT INTO mdvalues VALUES (34,NULL,'mgseqno','mgsearch.35');
+INSERT INTO mdvalues VALUES (34,NULL,'Title','Introduction');
+INSERT INTO mdvalues VALUES (35,NULL,'Title','34 Black Iguana');
+INSERT INTO mdvalues VALUES (36,NULL,'Title','1 Microcattle');
+INSERT INTO mdvalues VALUES (37,NULL,'Title','20 Guinea Pig');
+INSERT INTO mdvalues VALUES (38,NULL,'mgseqno','mgsearch.71');
+INSERT INTO mdvalues VALUES (38,NULL,'Title','31 Duikers');
+INSERT INTO mdvalues VALUES (39,NULL,'mgseqno','mgsearch.43');
+INSERT INTO mdvalues VALUES (39,NULL,'Title','6 Ducks');
+INSERT INTO mdvalues VALUES (40,NULL,'mgseqno','mgsearch.77');
+INSERT INTO mdvalues VALUES (40,NULL,'Title','35 Bees');
+INSERT INTO mdvalues VALUES (41,NULL,'Title','Board on Science and Technology for International Development (BOSTID)');
+INSERT INTO mdvalues VALUES (42,NULL,'Title','33 Green Iguana');
+INSERT INTO mdvalues VALUES (43,NULL,'Title','C BIOGRAPHICAL SKETCHES OF PANEL MEMBERS');
+INSERT INTO mdvalues VALUES (45,NULL,'Title','Part VII Others');
+INSERT INTO mdvalues VALUES (44,NULL,'Title','Acknowledgments');
+INSERT INTO mdvalues VALUES (46,NULL,'mgseqno','mgsearch.49');
+INSERT INTO mdvalues VALUES (46,NULL,'Title','12 Turkey');
+INSERT INTO mdvalues VALUES (47,NULL,'Title','8 Guinea Fowl');
+INSERT INTO mdvalues VALUES (48,NULL,'mgseqno','mgsearch.57');
+INSERT INTO mdvalues VALUES (48,NULL,'Title','18 Giant Rat');
+INSERT INTO mdvalues VALUES (49,NULL,'mgseqno','mgsearch.34');
+INSERT INTO mdvalues VALUES (49,NULL,'Title','Preface');
+INSERT INTO mdvalues VALUES (50,NULL,'mgseqno','mgsearch.40');
+INSERT INTO mdvalues VALUES (50,NULL,'Title','4 Micropigs');
+INSERT INTO mdvalues VALUES (52,NULL,'Title','25 Other Rodents');
+INSERT INTO mdvalues VALUES (53,NULL,'Title','Part I : Microbreeds');
+INSERT INTO mdvalues VALUES (54,NULL,'Title','Part II : Poultry');
+INSERT INTO mdvalues VALUES (55,NULL,'Title','32 Other Small Antelope');
+INSERT INTO mdvalues VALUES (56,NULL,'mgseqno','mgsearch.63');
+INSERT INTO mdvalues VALUES (56,NULL,'Title','24 Vizcacha');
+INSERT INTO mdvalues VALUES (57,NULL,'Title','29 South America\'s Microdeer');
+INSERT INTO mdvalues VALUES (58,NULL,'Title','30 Water Deer');
+INSERT INTO mdvalues VALUES (59,NULL,'Title','Part V : Deer and Antelope');
+INSERT INTO mdvalues VALUES (60,NULL,'Title','Microlivestock   LittleKnown Small Animals with a Promising Economic Future');
+INSERT INTO mdvalues VALUES (61,NULL,'mgseqno','mgsearch.39');
+INSERT INTO mdvalues VALUES (61,NULL,'Title','3 Microsheep');
+INSERT INTO mdvalues VALUES (62,NULL,'mgseqno','mgsearch.60');
+INSERT INTO mdvalues VALUES (62,NULL,'Title','21 Hutia');
+INSERT INTO mdvalues VALUES (63,NULL,'Title','Part VI : Lizards');
+INSERT INTO mdvalues VALUES (64,NULL,'mgseqno','mgsearch.48');
+INSERT INTO mdvalues VALUES (64,NULL,'Title','11 Quail');
+INSERT INTO mdvalues VALUES (65,NULL,'mgseqno','mgsearch.46');
+INSERT INTO mdvalues VALUES (65,NULL,'Title','9 Muscovy');
+INSERT INTO mdvalues VALUES (66,NULL,'mgseqno','mgsearch.62');
+INSERT INTO mdvalues VALUES (66,NULL,'Title','23 Paca');
+INSERT INTO mdvalues VALUES (67,NULL,'mgseqno','mgsearch.78');
+INSERT INTO mdvalues VALUES (67,NULL,'Title','Appendixes');
+INSERT INTO mdvalues VALUES (68,NULL,'Title','Part IV : Rodents');
+INSERT INTO mdvalues VALUES (69,NULL,'mgseqno','mgsearch.47');
+INSERT INTO mdvalues VALUES (69,NULL,'Title','10 Pigeon');
+INSERT INTO mdvalues VALUES (70,NULL,'Title','14 Domestic Rabbit');
+INSERT INTO mdvalues VALUES (71,NULL,'Title','Part III : Rabbits');
+INSERT INTO mdvalues VALUES (72,NULL,'mgseqno','mgsearch.55');
+INSERT INTO mdvalues VALUES (72,NULL,'Title','16 Capybara');
+INSERT INTO mdvalues VALUES (73,NULL,'mgseqno','mgsearch.38');
+INSERT INTO mdvalues VALUES (73,NULL,'Title','2 Microgoats');
+INSERT INTO mdvalues VALUES (74,NULL,'mgseqno','mgsearch.44');
+INSERT INTO mdvalues VALUES (74,NULL,'Title','7 Geese');
+INSERT INTO mdvalues VALUES (75,NULL,'mgseqno','mgsearch.68');
+INSERT INTO mdvalues VALUES (75,NULL,'Title','28 Musk Deer');
+INSERT INTO mdvalues VALUES (76,NULL,'Title','B Research Contacts');
+INSERT INTO mdvalues VALUES (77,NULL,'Title','A Selected Readings');
+INSERT INTO mdvalues VALUES (78,NULL,'mgseqno','mgsearch.54');
+INSERT INTO mdvalues VALUES (78,NULL,'Title','15 Agouti');
+INSERT INTO mdvalues VALUES (80,NULL,'mgseqno','mgsearch.67');
+INSERT INTO mdvalues VALUES (80,NULL,'Title','27 Muntjac');
+INSERT INTO mdvalues VALUES (79,NULL,'mgseqno','mgsearch.61');
+INSERT INTO mdvalues VALUES (79,NULL,'Title','22 Mara');
+INSERT INTO mdvalues VALUES (81,NULL,'Title','13 Potential New Poultry');
+INSERT INTO mdvalues VALUES (82,NULL,'Title','Now you must decide');
+INSERT INTO mdvalues VALUES (83,NULL,'Title','When are the plants big enough for food and for shelter?');
+INSERT INTO mdvalues VALUES (84,NULL,'Title','Land that is wet');
+INSERT INTO mdvalues VALUES (85,NULL,'mgseqno','mgsearch.101');
+INSERT INTO mdvalues VALUES (85,NULL,'Title','How to begin');
+INSERT INTO mdvalues VALUES (86,NULL,'Title','What do you need to start?');
+INSERT INTO mdvalues VALUES (87,NULL,'Title','Where can you farm snails?');
+INSERT INTO mdvalues VALUES (88,NULL,'Title','How much snail meat can you expect to get?');
+INSERT INTO mdvalues VALUES (91,NULL,'Title','Where can you get snails?');
+INSERT INTO mdvalues VALUES (92,NULL,'Title','Farming snails 1: Learning about snails; Building a pen; Food and shelter plants');
+INSERT INTO mdvalues VALUES (93,NULL,'Title','Building a pen');
+INSERT INTO mdvalues VALUES (94,NULL,'Title','Plant the food and shelter plants');
+INSERT INTO mdvalues VALUES (95,NULL,'Title','Choose a place on your land to put your snail farm');
+INSERT INTO mdvalues VALUES (96,NULL,'mgseqno','mgsearch.86');
+INSERT INTO mdvalues VALUES (96,NULL,'Title','Introduction');
+INSERT INTO mdvalues VALUES (97,NULL,'Title','Plants for food and plants for shelter');
+INSERT INTO mdvalues VALUES (98,NULL,'Title','Where do you keep snails when you farm them?');
+INSERT INTO mdvalues VALUES (99,NULL,'Title','What kind of snails can you farm?');
+INSERT INTO mdvalues VALUES (100,NULL,'Title','Clean the site and prepare the ground');
+INSERT INTO mdvalues VALUES (101,NULL,'Title','The soil and the water in the soil');
+INSERT INTO mdvalues VALUES (103,NULL,'mgseqno','mgsearch.96');
+INSERT INTO mdvalues VALUES (103,NULL,'Title','Wind');
+INSERT INTO mdvalues VALUES (102,NULL,'Title','Learning about snails');
+INSERT INTO mdvalues VALUES (104,NULL,'Title','How many snails will you need when you first begin?');
+INSERT INTO mdvalues VALUES (105,NULL,'Title','Taking better care of your snails');
+INSERT INTO mdvalues VALUES (106,NULL,'Title','When to harvest your snails');
+INSERT INTO mdvalues VALUES (107,NULL,'Title','Taking care of your snails');
+INSERT INTO mdvalues VALUES (108,NULL,'Title','When to take out the fully grown snails');
+INSERT INTO mdvalues VALUES (109,NULL,'Title','Further improvement');
+INSERT INTO mdvalues VALUES (110,NULL,'Title','When to build the second pen and how to build it');
+INSERT INTO mdvalues VALUES (112,NULL,'Title','How to harvest your snails');
+INSERT INTO mdvalues VALUES (113,NULL,'Title','Putting your snails in the pen');
+INSERT INTO mdvalues VALUES (114,NULL,'Title','Take away the weeds and creeping grass');
+INSERT INTO mdvalues VALUES (115,NULL,'Title','Farming snails 2: Choosing snails; Care and harvesting; Further improvement');
+INSERT INTO mdvalues VALUES (116,NULL,'Title','Building the second pen');
+INSERT INTO mdvalues VALUES (118,NULL,'Title','Harvesting your snails');
+INSERT INTO mdvalues VALUES (117,NULL,'Title','Preparing snails for cooking');
+INSERT INTO mdvalues VALUES (121,NULL,'Title','Wet the food and shelter plants and moisten the ground');
+INSERT INTO mdvalues VALUES (120,NULL,'Title','Putting snails in your second pen');
+INSERT INTO mdvalues VALUES (119,NULL,'Title','Building a bigger pen');
+INSERT INTO mdvalues VALUES (123,NULL,'mgseqno','mgsearch.110');
+INSERT INTO mdvalues VALUES (123,NULL,'Title','Preface');
+INSERT INTO mdvalues VALUES (122,NULL,'Title','What to do with the snails you harvest');
+INSERT INTO mdvalues VALUES (124,NULL,'Title','Feeding your snails other kinds of food');
+INSERT INTO mdvalues VALUES (125,NULL,'Title','Building the third pen');
+INSERT INTO mdvalues VALUES (126,NULL,'Title','Choosing your snails');
+INSERT INTO mdvalues VALUES (127,NULL,'Title','Tend the food and shelter plants');
+INSERT INTO mdvalues VALUES (128,NULL,'mgseqno','mgsearch.111');
+INSERT INTO mdvalues VALUES (128,NULL,'Title','The snails');
+INSERT INTO mdvalues VALUES (130,NULL,'Title','Household and Intrahousehold Resource Allocation');
+INSERT INTO mdvalues VALUES (129,NULL,'mgseqno','mgsearch.166');
+INSERT INTO mdvalues VALUES (129,NULL,'Title','Family Law');
+INSERT INTO mdvalues VALUES (132,NULL,'Title','Gender Inequalities Persist');
+INSERT INTO mdvalues VALUES (131,NULL,'Title','Land and Property Rights');
+INSERT INTO mdvalues VALUES (135,NULL,'Title','Definitions and Data Notes');
+INSERT INTO mdvalues VALUES (134,NULL,'Title','Strengthening International Policies to Meet New Challenges');
+INSERT INTO mdvalues VALUES (133,NULL,'mgseqno','mgsearch.178');
+INSERT INTO mdvalues VALUES (133,NULL,'Title','Notes');
+INSERT INTO mdvalues VALUES (137,NULL,'Title','Linkages between Education Health, and Nutritious');
+INSERT INTO mdvalues VALUES (136,NULL,'Title','Women\'s bargaining position in relation to household');
+INSERT INTO mdvalues VALUES (138,NULL,'mgseqno','mgsearch.147');
+INSERT INTO mdvalues VALUES (138,NULL,'Title','Education');
+INSERT INTO mdvalues VALUES (139,NULL,'Title','Development in practice: Toward Gender Equality');
+INSERT INTO mdvalues VALUES (140,NULL,'Title','Working in Collaboration');
+INSERT INTO mdvalues VALUES (141,NULL,'mgseqno','mgsearch.177');
+INSERT INTO mdvalues VALUES (141,NULL,'Title','Conclusions');
+INSERT INTO mdvalues VALUES (142,NULL,'mgseqno','mgsearch.135');
+INSERT INTO mdvalues VALUES (142,NULL,'Title','Foreword');
+INSERT INTO mdvalues VALUES (143,NULL,'Title','Why Do Gender Inequalities Persist?');
+INSERT INTO mdvalues VALUES (144,NULL,'Title','Employment Work');
+INSERT INTO mdvalues VALUES (145,NULL,'mgseqno','mgsearch.139');
+INSERT INTO mdvalues VALUES (145,NULL,'Title','Data Notes');
+INSERT INTO mdvalues VALUES (146,NULL,'Title','Formal Sector Employment');
+INSERT INTO mdvalues VALUES (147,NULL,'Title','Labor Market Policies and Employment Law');
+INSERT INTO mdvalues VALUES (148,NULL,'Title','Financial Laws and Regulations');
+INSERT INTO mdvalues VALUES (149,NULL,'Title','Acknowledgments');
+INSERT INTO mdvalues VALUES (150,NULL,'Title','Informal Sector');
+INSERT INTO mdvalues VALUES (151,NULL,'mgseqno','mgsearch.179');
+INSERT INTO mdvalues VALUES (151,NULL,'Title','References');
+INSERT INTO mdvalues VALUES (152,NULL,'Title','Involving Beneficiaries in Public Policy');
+INSERT INTO mdvalues VALUES (154,NULL,'Title','Access to Extension Services');
+INSERT INTO mdvalues VALUES (153,NULL,'Title','Inflation tends to hit women harder than men.');
+INSERT INTO mdvalues VALUES (155,NULL,'Title','Chapter three');
+INSERT INTO mdvalues VALUES (156,NULL,'Keyword','achieve gender equality');
+INSERT INTO mdvalues VALUES (157,NULL,'Title','Using Targeting Measures to Narrow the Gender');
+INSERT INTO mdvalues VALUES (160,NULL,'mgseqno','mgsearch.140');
+INSERT INTO mdvalues VALUES (160,NULL,'Title','Summary');
+INSERT INTO mdvalues VALUES (159,NULL,'Title','Access to Lund and Property');
+INSERT INTO mdvalues VALUES (158,NULL,'mgseqno','mgsearch.160');
+INSERT INTO mdvalues VALUES (158,NULL,'Title','Conclusion');
+INSERT INTO mdvalues VALUES (161,NULL,'mgseqno','mgsearch.144');
+INSERT INTO mdvalues VALUES (161,NULL,'Title','Conclusion');
+INSERT INTO mdvalues VALUES (162,NULL,'Title','Gender Inequalities Hamper Growth');
+INSERT INTO mdvalues VALUES (165,NULL,'Title','Household and Labor Market Linkages');
+INSERT INTO mdvalues VALUES (164,NULL,'mgseqno','mgsearch.150');
+INSERT INTO mdvalues VALUES (164,NULL,'Title','Chapter two');
+INSERT INTO mdvalues VALUES (163,NULL,'mgseqno','mgsearch.145');
+INSERT INTO mdvalues VALUES (163,NULL,'Title','Chapter one');
+INSERT INTO mdvalues VALUES (167,NULL,'mgseqno','mgsearch.138');
+INSERT INTO mdvalues VALUES (167,NULL,'Title','Definitions');
+INSERT INTO mdvalues VALUES (166,NULL,'Title','Equalizing Opportunities by Modifying, the Legal Framework');
+INSERT INTO mdvalues VALUES (168,NULL,'mgseqno','mgsearch.148');
+INSERT INTO mdvalues VALUES (168,NULL,'Title','Health');
+INSERT INTO mdvalues VALUES (169,NULL,'Title','Progress to Date');
+INSERT INTO mdvalues VALUES (170,NULL,'Title','Sectoral Investments');
+INSERT INTO mdvalues VALUES (171,NULL,'Title','Access to Financial Markets');
+INSERT INTO mdvalues VALUES (172,NULL,'Title','Public Policies Matter');
+INSERT INTO mdvalues VALUES (174,NULL,'Title','Macroeconomic: Policies');
+INSERT INTO mdvalues VALUES (173,NULL,'Title','Generating and Analyzing GenderDesegregated Data');
+INSERT INTO mdvalues VALUES (175,NULL,'Title','Strategies for the Future');
+INSERT INTO mdvalues VALUES (1,NULL,'mgseqno','mgsearch.20');
+INSERT INTO mdvalues VALUES (2,NULL,'mgseqno','mgsearch.26');
+INSERT INTO mdvalues VALUES (3,NULL,'mgseqno','mgsearch.13');
+INSERT INTO mdvalues VALUES (5,NULL,'mgseqno','mgsearch.25');
+INSERT INTO mdvalues VALUES (4,NULL,'mgseqno','mgsearch.5');
+INSERT INTO mdvalues VALUES (6,NULL,'mgseqno','mgsearch.7');
+INSERT INTO mdvalues VALUES (7,NULL,'mgseqno','mgsearch.21');
+INSERT INTO mdvalues VALUES (8,NULL,'mgseqno','mgsearch.14');
+INSERT INTO mdvalues VALUES (9,NULL,'mgseqno','mgsearch.2');
+INSERT INTO mdvalues VALUES (9,NULL,'mgseqno','mgsearch.2');
+INSERT INTO mdvalues VALUES (11,NULL,'mgseqno','mgsearch.3');
+INSERT INTO mdvalues VALUES (10,NULL,'mgseqno','mgsearch.29');
+INSERT INTO mdvalues VALUES (10,NULL,'mgseqno','mgsearch.29');
+INSERT INTO mdvalues VALUES (12,NULL,'mgseqno','mgsearch.9');
+INSERT INTO mdvalues VALUES (13,NULL,'mgseqno','mgsearch.16');
+INSERT INTO mdvalues VALUES (14,NULL,'mgseqno','mgsearch.23');
+INSERT INTO mdvalues VALUES (15,NULL,'mgseqno','mgsearch.4');
+INSERT INTO mdvalues VALUES (16,NULL,'mgseqno','mgsearch.28');
+INSERT INTO mdvalues VALUES (16,NULL,'mgseqno','mgsearch.28');
+INSERT INTO mdvalues VALUES (17,NULL,'mgseqno','mgsearch.17');
+INSERT INTO mdvalues VALUES (18,NULL,'mgseqno','mgsearch.24');
+INSERT INTO mdvalues VALUES (18,NULL,'mgseqno','mgsearch.24');
+INSERT INTO mdvalues VALUES (19,NULL,'mgseqno','mgsearch.10');
+INSERT INTO mdvalues VALUES (22,NULL,'mgseqno','mgsearch.6');
+INSERT INTO mdvalues VALUES (22,NULL,'mgseqno','mgsearch.6');
+INSERT INTO mdvalues VALUES (23,NULL,'mgseqno','mgsearch.19');
+INSERT INTO mdvalues VALUES (26,NULL,'mgseqno','mgsearch.22');
+INSERT INTO mdvalues VALUES (24,NULL,'mgseqno','mgsearch.12');
+INSERT INTO mdvalues VALUES (25,NULL,'mgseqno','mgsearch.18');
+INSERT INTO mdvalues VALUES (27,NULL,'mgseqno','mgsearch.8');
+INSERT INTO mdvalues VALUES (28,NULL,'mgseqno','mgsearch.15');
+INSERT INTO mdvalues VALUES (29,NULL,'mgseqno','mgsearch.27');
+INSERT INTO mdvalues VALUES (30,NULL,'mgseqno','mgsearch.58');
+INSERT INTO mdvalues VALUES (31,NULL,'mgseqno','mgsearch.66');
+INSERT INTO mdvalues VALUES (32,NULL,'mgseqno','mgsearch.42');
+INSERT INTO mdvalues VALUES (33,NULL,'mgseqno','mgsearch.56');
+INSERT INTO mdvalues VALUES (34,NULL,'mgseqno','mgsearch.35');
+INSERT INTO mdvalues VALUES (35,NULL,'mgseqno','mgsearch.75');
+INSERT INTO mdvalues VALUES (36,NULL,'mgseqno','mgsearch.37');
+INSERT INTO mdvalues VALUES (37,NULL,'mgseqno','mgsearch.59');
+INSERT INTO mdvalues VALUES (38,NULL,'mgseqno','mgsearch.71');
+INSERT INTO mdvalues VALUES (39,NULL,'mgseqno','mgsearch.43');
+INSERT INTO mdvalues VALUES (40,NULL,'mgseqno','mgsearch.77');
+INSERT INTO mdvalues VALUES (41,NULL,'mgseqno','mgsearch.82');
+INSERT INTO mdvalues VALUES (41,NULL,'mgseqno','mgsearch.82');
+INSERT INTO mdvalues VALUES (42,NULL,'mgseqno','mgsearch.74');
+INSERT INTO mdvalues VALUES (43,NULL,'mgseqno','mgsearch.81');
+INSERT INTO mdvalues VALUES (43,NULL,'mgseqno','mgsearch.81');
+INSERT INTO mdvalues VALUES (45,NULL,'mgseqno','mgsearch.76');
+INSERT INTO mdvalues VALUES (44,NULL,'mgseqno','mgsearch.33');
+INSERT INTO mdvalues VALUES (46,NULL,'mgseqno','mgsearch.49');
+INSERT INTO mdvalues VALUES (47,NULL,'mgseqno','mgsearch.45');
+INSERT INTO mdvalues VALUES (48,NULL,'mgseqno','mgsearch.57');
+INSERT INTO mdvalues VALUES (49,NULL,'mgseqno','mgsearch.34');
+INSERT INTO mdvalues VALUES (50,NULL,'mgseqno','mgsearch.40');
+INSERT INTO mdvalues VALUES (52,NULL,'mgseqno','mgsearch.64');
+INSERT INTO mdvalues VALUES (53,NULL,'mgseqno','mgsearch.36');
+INSERT INTO mdvalues VALUES (54,NULL,'mgseqno','mgsearch.41');
+INSERT INTO mdvalues VALUES (55,NULL,'mgseqno','mgsearch.72');
+INSERT INTO mdvalues VALUES (56,NULL,'mgseqno','mgsearch.63');
+INSERT INTO mdvalues VALUES (57,NULL,'mgseqno','mgsearch.69');
+INSERT INTO mdvalues VALUES (58,NULL,'mgseqno','mgsearch.70');
+INSERT INTO mdvalues VALUES (59,NULL,'mgseqno','mgsearch.65');
+INSERT INTO mdvalues VALUES (60,NULL,'mgseqno','mgsearch.32');
+INSERT INTO mdvalues VALUES (60,NULL,'mgseqno','mgsearch.32');
+INSERT INTO mdvalues VALUES (61,NULL,'mgseqno','mgsearch.39');
+INSERT INTO mdvalues VALUES (62,NULL,'mgseqno','mgsearch.60');
+INSERT INTO mdvalues VALUES (63,NULL,'mgseqno','mgsearch.73');
+INSERT INTO mdvalues VALUES (64,NULL,'mgseqno','mgsearch.48');
+INSERT INTO mdvalues VALUES (65,NULL,'mgseqno','mgsearch.46');
+INSERT INTO mdvalues VALUES (66,NULL,'mgseqno','mgsearch.62');
+INSERT INTO mdvalues VALUES (67,NULL,'mgseqno','mgsearch.78');
+INSERT INTO mdvalues VALUES (68,NULL,'mgseqno','mgsearch.53');
+INSERT INTO mdvalues VALUES (69,NULL,'mgseqno','mgsearch.47');
+INSERT INTO mdvalues VALUES (70,NULL,'mgseqno','mgsearch.52');
+INSERT INTO mdvalues VALUES (71,NULL,'mgseqno','mgsearch.51');
+INSERT INTO mdvalues VALUES (72,NULL,'mgseqno','mgsearch.55');
+INSERT INTO mdvalues VALUES (73,NULL,'mgseqno','mgsearch.38');
+INSERT INTO mdvalues VALUES (74,NULL,'mgseqno','mgsearch.44');
+INSERT INTO mdvalues VALUES (75,NULL,'mgseqno','mgsearch.68');
+INSERT INTO mdvalues VALUES (76,NULL,'mgseqno','mgsearch.80');
+INSERT INTO mdvalues VALUES (77,NULL,'mgseqno','mgsearch.79');
+INSERT INTO mdvalues VALUES (78,NULL,'mgseqno','mgsearch.54');
+INSERT INTO mdvalues VALUES (80,NULL,'mgseqno','mgsearch.67');
+INSERT INTO mdvalues VALUES (79,NULL,'mgseqno','mgsearch.61');
+INSERT INTO mdvalues VALUES (81,NULL,'mgseqno','mgsearch.50');
+INSERT INTO mdvalues VALUES (82,NULL,'mgseqno','mgsearch.100');
+INSERT INTO mdvalues VALUES (83,NULL,'mgseqno','mgsearch.106');
+INSERT INTO mdvalues VALUES (83,NULL,'mgseqno','mgsearch.106');
+INSERT INTO mdvalues VALUES (84,NULL,'mgseqno','mgsearch.95');
+INSERT INTO mdvalues VALUES (85,NULL,'mgseqno','mgsearch.101');
+INSERT INTO mdvalues VALUES (86,NULL,'mgseqno','mgsearch.87');
+INSERT INTO mdvalues VALUES (87,NULL,'mgseqno','mgsearch.91');
+INSERT INTO mdvalues VALUES (88,NULL,'mgseqno','mgsearch.99');
+INSERT INTO mdvalues VALUES (88,NULL,'mgseqno','mgsearch.99');
+INSERT INTO mdvalues VALUES (89,NULL,'classified','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (89,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (90,NULL,'mgseqno','mgsearch.94');
+INSERT INTO mdvalues VALUES (91,NULL,'mgseqno','mgsearch.90');
+INSERT INTO mdvalues VALUES (92,NULL,'mgseqno','mgsearch.85');
+INSERT INTO mdvalues VALUES (92,NULL,'mgseqno','mgsearch.85');
+INSERT INTO mdvalues VALUES (93,NULL,'mgseqno','mgsearch.104');
+INSERT INTO mdvalues VALUES (94,NULL,'mgseqno','mgsearch.105');
+INSERT INTO mdvalues VALUES (95,NULL,'mgseqno','mgsearch.102');
+INSERT INTO mdvalues VALUES (95,NULL,'mgseqno','mgsearch.102');
+INSERT INTO mdvalues VALUES (96,NULL,'mgseqno','mgsearch.86');
+INSERT INTO mdvalues VALUES (97,NULL,'mgseqno','mgsearch.93');
+INSERT INTO mdvalues VALUES (97,NULL,'mgseqno','mgsearch.93');
+INSERT INTO mdvalues VALUES (98,NULL,'mgseqno','mgsearch.97');
+INSERT INTO mdvalues VALUES (98,NULL,'mgseqno','mgsearch.97');
+INSERT INTO mdvalues VALUES (99,NULL,'mgseqno','mgsearch.89');
+INSERT INTO mdvalues VALUES (100,NULL,'mgseqno','mgsearch.103');
+INSERT INTO mdvalues VALUES (100,NULL,'mgseqno','mgsearch.103');
+INSERT INTO mdvalues VALUES (101,NULL,'mgseqno','mgsearch.92');
+INSERT INTO mdvalues VALUES (103,NULL,'mgseqno','mgsearch.96');
+INSERT INTO mdvalues VALUES (102,NULL,'mgseqno','mgsearch.88');
+INSERT INTO mdvalues VALUES (104,NULL,'mgseqno','mgsearch.98');
+INSERT INTO mdvalues VALUES (104,NULL,'mgseqno','mgsearch.98');
+INSERT INTO mdvalues VALUES (105,NULL,'mgseqno','mgsearch.131');
+INSERT INTO mdvalues VALUES (106,NULL,'mgseqno','mgsearch.124');
+INSERT INTO mdvalues VALUES (107,NULL,'mgseqno','mgsearch.115');
+INSERT INTO mdvalues VALUES (108,NULL,'mgseqno','mgsearch.114');
+INSERT INTO mdvalues VALUES (108,NULL,'mgseqno','mgsearch.114');
+INSERT INTO mdvalues VALUES (109,NULL,'mgseqno','mgsearch.128');
+INSERT INTO mdvalues VALUES (110,NULL,'mgseqno','mgsearch.121');
+INSERT INTO mdvalues VALUES (110,NULL,'mgseqno','mgsearch.121');
+INSERT INTO mdvalues VALUES (112,NULL,'mgseqno','mgsearch.125');
+INSERT INTO mdvalues VALUES (113,NULL,'mgseqno','mgsearch.113');
+INSERT INTO mdvalues VALUES (114,NULL,'mgseqno','mgsearch.118');
+INSERT INTO mdvalues VALUES (114,NULL,'mgseqno','mgsearch.118');
+INSERT INTO mdvalues VALUES (115,NULL,'mgseqno','mgsearch.109');
+INSERT INTO mdvalues VALUES (115,NULL,'mgseqno','mgsearch.109');
+INSERT INTO mdvalues VALUES (116,NULL,'mgseqno','mgsearch.120');
+INSERT INTO mdvalues VALUES (118,NULL,'mgseqno','mgsearch.123');
+INSERT INTO mdvalues VALUES (117,NULL,'mgseqno','mgsearch.127');
+INSERT INTO mdvalues VALUES (121,NULL,'mgseqno','mgsearch.116');
+INSERT INTO mdvalues VALUES (121,NULL,'mgseqno','mgsearch.116');
+INSERT INTO mdvalues VALUES (120,NULL,'mgseqno','mgsearch.122');
+INSERT INTO mdvalues VALUES (119,NULL,'mgseqno','mgsearch.130');
+INSERT INTO mdvalues VALUES (123,NULL,'mgseqno','mgsearch.110');
+INSERT INTO mdvalues VALUES (122,NULL,'mgseqno','mgsearch.126');
+INSERT INTO mdvalues VALUES (122,NULL,'mgseqno','mgsearch.126');
+INSERT INTO mdvalues VALUES (124,NULL,'mgseqno','mgsearch.119');
+INSERT INTO mdvalues VALUES (124,NULL,'mgseqno','mgsearch.119');
+INSERT INTO mdvalues VALUES (125,NULL,'mgseqno','mgsearch.129');
+INSERT INTO mdvalues VALUES (126,NULL,'mgseqno','mgsearch.112');
+INSERT INTO mdvalues VALUES (127,NULL,'mgseqno','mgsearch.117');
+INSERT INTO mdvalues VALUES (128,NULL,'mgseqno','mgsearch.111');
+INSERT INTO mdvalues VALUES (130,NULL,'mgseqno','mgsearch.152');
+INSERT INTO mdvalues VALUES (130,NULL,'mgseqno','mgsearch.152');
+INSERT INTO mdvalues VALUES (129,NULL,'mgseqno','mgsearch.166');
+INSERT INTO mdvalues VALUES (132,NULL,'mgseqno','mgsearch.146');
+INSERT INTO mdvalues VALUES (131,NULL,'mgseqno','mgsearch.164');
+INSERT INTO mdvalues VALUES (135,NULL,'mgseqno','mgsearch.137');
+INSERT INTO mdvalues VALUES (134,NULL,'mgseqno','mgsearch.176');
+INSERT INTO mdvalues VALUES (134,NULL,'mgseqno','mgsearch.176');
+INSERT INTO mdvalues VALUES (133,NULL,'mgseqno','mgsearch.178');
+INSERT INTO mdvalues VALUES (137,NULL,'mgseqno','mgsearch.153');
+INSERT INTO mdvalues VALUES (137,NULL,'mgseqno','mgsearch.153');
+INSERT INTO mdvalues VALUES (136,NULL,'mgseqno','mgsearch.167');
+INSERT INTO mdvalues VALUES (136,NULL,'mgseqno','mgsearch.167');
+INSERT INTO mdvalues VALUES (138,NULL,'mgseqno','mgsearch.147');
+INSERT INTO mdvalues VALUES (139,NULL,'mgseqno','mgsearch.134');
+INSERT INTO mdvalues VALUES (139,NULL,'mgseqno','mgsearch.134');
+INSERT INTO mdvalues VALUES (140,NULL,'mgseqno','mgsearch.175');
+INSERT INTO mdvalues VALUES (141,NULL,'mgseqno','mgsearch.177');
+INSERT INTO mdvalues VALUES (142,NULL,'mgseqno','mgsearch.135');
+INSERT INTO mdvalues VALUES (143,NULL,'mgseqno','mgsearch.142');
+INSERT INTO mdvalues VALUES (144,NULL,'mgseqno','mgsearch.149');
+INSERT INTO mdvalues VALUES (145,NULL,'mgseqno','mgsearch.139');
+INSERT INTO mdvalues VALUES (146,NULL,'mgseqno','mgsearch.155');
+INSERT INTO mdvalues VALUES (147,NULL,'mgseqno','mgsearch.165');
+INSERT INTO mdvalues VALUES (147,NULL,'mgseqno','mgsearch.165');
+INSERT INTO mdvalues VALUES (148,NULL,'mgseqno','mgsearch.168');
+INSERT INTO mdvalues VALUES (149,NULL,'mgseqno','mgsearch.136');
+INSERT INTO mdvalues VALUES (150,NULL,'mgseqno','mgsearch.156');
+INSERT INTO mdvalues VALUES (151,NULL,'mgseqno','mgsearch.179');
+INSERT INTO mdvalues VALUES (152,NULL,'mgseqno','mgsearch.173');
+INSERT INTO mdvalues VALUES (152,NULL,'mgseqno','mgsearch.173');
+INSERT INTO mdvalues VALUES (154,NULL,'mgseqno','mgsearch.159');
+INSERT INTO mdvalues VALUES (153,NULL,'mgseqno','mgsearch.170');
+INSERT INTO mdvalues VALUES (153,NULL,'mgseqno','mgsearch.170');
+INSERT INTO mdvalues VALUES (155,NULL,'mgseqno','mgsearch.161');
+INSERT INTO mdvalues VALUES (157,NULL,'mgseqno','mgsearch.172');
+INSERT INTO mdvalues VALUES (157,NULL,'mgseqno','mgsearch.172');
+INSERT INTO mdvalues VALUES (160,NULL,'mgseqno','mgsearch.140');
+INSERT INTO mdvalues VALUES (159,NULL,'mgseqno','mgsearch.158');
+INSERT INTO mdvalues VALUES (158,NULL,'mgseqno','mgsearch.160');
+INSERT INTO mdvalues VALUES (161,NULL,'mgseqno','mgsearch.144');
+INSERT INTO mdvalues VALUES (162,NULL,'mgseqno','mgsearch.151');
+INSERT INTO mdvalues VALUES (165,NULL,'mgseqno','mgsearch.154');
+INSERT INTO mdvalues VALUES (164,NULL,'mgseqno','mgsearch.150');
+INSERT INTO mdvalues VALUES (163,NULL,'mgseqno','mgsearch.145');
+INSERT INTO mdvalues VALUES (167,NULL,'mgseqno','mgsearch.138');
+INSERT INTO mdvalues VALUES (166,NULL,'mgseqno','mgsearch.163');
+INSERT INTO mdvalues VALUES (166,NULL,'mgseqno','mgsearch.163');
+INSERT INTO mdvalues VALUES (168,NULL,'mgseqno','mgsearch.148');
+INSERT INTO mdvalues VALUES (169,NULL,'mgseqno','mgsearch.141');
+INSERT INTO mdvalues VALUES (170,NULL,'mgseqno','mgsearch.171');
+INSERT INTO mdvalues VALUES (171,NULL,'mgseqno','mgsearch.157');
+INSERT INTO mdvalues VALUES (172,NULL,'mgseqno','mgsearch.162');
+INSERT INTO mdvalues VALUES (174,NULL,'mgseqno','mgsearch.169');
+INSERT INTO mdvalues VALUES (173,NULL,'mgseqno','mgsearch.174');
+INSERT INTO mdvalues VALUES (173,NULL,'mgseqno','mgsearch.174');
+INSERT INTO mdvalues VALUES (175,NULL,'mgseqno','mgsearch.143');
+
+--
+-- Table structure for table `metadata`
+--
+
+CREATE TABLE metadata (
+  MetadataRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  MetaID varchar(64) default NULL,
+  GroupID varchar(64) default NULL,
+  PRIMARY KEY  (MetadataRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `metadata`
+--
+
+INSERT INTO metadata VALUES (1,'20050201:1','DM20','1.6.2');
+INSERT INTO metadata VALUES (2,'20050201:1','DM26','1.8.1');
+INSERT INTO metadata VALUES (3,'20050201:1','DM13','1.5.1');
+INSERT INTO metadata VALUES (4,'20050201:1','DM5','1.3');
+INSERT INTO metadata VALUES (5,'20050201:1','DM25','1.8');
+INSERT INTO metadata VALUES (6,'20050201:1','DM7','1.4.1');
+INSERT INTO metadata VALUES (7,'20050201:1','DM21','1.6.3');
+INSERT INTO metadata VALUES (8,'20050201:1','DM14','1.5.2');
+INSERT INTO metadata VALUES (9,'20050201:1','DM2','1');
+INSERT INTO metadata VALUES (10,'20050201:1','DM29','1.10');
+INSERT INTO metadata VALUES (11,'20050201:1','DM3','1.1');
+INSERT INTO metadata VALUES (12,'20050201:1','DM9','1.4.3');
+INSERT INTO metadata VALUES (13,'20050201:1','DM16','1.5.4');
+INSERT INTO metadata VALUES (14,'20050201:1','DM23','1.6.5');
+INSERT INTO metadata VALUES (15,'20050201:1','DM4','1.2');
+INSERT INTO metadata VALUES (16,'20050201:1','DM28','1.9');
+INSERT INTO metadata VALUES (17,'20050201:1','DM17','1.5.5');
+INSERT INTO metadata VALUES (18,'20050201:1','DM24','1.7');
+INSERT INTO metadata VALUES (19,'20050201:1','DM10','1.4.4');
+INSERT INTO metadata VALUES (20,'20050201:1','DM1','default');
+INSERT INTO metadata VALUES (21,'20050201:1','DM11','1.4.5');
+INSERT INTO metadata VALUES (22,'20050201:1','DM6','1.4');
+INSERT INTO metadata VALUES (23,'20050201:1','DM19','1.6.1');
+INSERT INTO metadata VALUES (24,'20050201:1','DM12','1.5');
+INSERT INTO metadata VALUES (25,'20050201:1','DM18','1.6');
+INSERT INTO metadata VALUES (26,'20050201:1','DM22','1.6.4');
+INSERT INTO metadata VALUES (27,'20050201:1','DM8','1.4.2');
+INSERT INTO metadata VALUES (28,'20050201:1','DM15','1.5.3');
+INSERT INTO metadata VALUES (29,'20050201:1','DM27','1.8.2');
+INSERT INTO metadata VALUES (30,'20050201:2','DM28','1.7.5');
+INSERT INTO metadata VALUES (31,'20050201:2','DM36','1.8.1');
+INSERT INTO metadata VALUES (32,'20050201:2','DM12','1.5.1');
+INSERT INTO metadata VALUES (33,'20050201:2','DM26','1.7.3');
+INSERT INTO metadata VALUES (34,'20050201:2','DM5','1.3');
+INSERT INTO metadata VALUES (35,'20050201:2','DM45','1.9.2');
+INSERT INTO metadata VALUES (36,'20050201:2','DM7','1.4.1');
+INSERT INTO metadata VALUES (37,'20050201:2','DM29','1.7.6');
+INSERT INTO metadata VALUES (38,'20050201:2','DM41','1.8.6');
+INSERT INTO metadata VALUES (39,'20050201:2','DM13','1.5.2');
+INSERT INTO metadata VALUES (40,'20050201:2','DM47','1.10.1');
+INSERT INTO metadata VALUES (41,'20050201:2','DM52','1.12');
+INSERT INTO metadata VALUES (42,'20050201:2','DM44','1.9.1');
+INSERT INTO metadata VALUES (43,'20050201:2','DM51','1.11.3');
+INSERT INTO metadata VALUES (44,'20050201:2','DM3','1.1');
+INSERT INTO metadata VALUES (45,'20050201:2','DM46','1.10');
+INSERT INTO metadata VALUES (46,'20050201:2','DM19','1.5.8');
+INSERT INTO metadata VALUES (47,'20050201:2','DM15','1.5.4');
+INSERT INTO metadata VALUES (48,'20050201:2','DM27','1.7.4');
+INSERT INTO metadata VALUES (49,'20050201:2','DM4','1.2');
+INSERT INTO metadata VALUES (50,'20050201:2','DM10','1.4.4');
+INSERT INTO metadata VALUES (51,'20050201:2','DM1','default');
+INSERT INTO metadata VALUES (52,'20050201:2','DM34','1.7.11');
+INSERT INTO metadata VALUES (53,'20050201:2','DM6','1.4');
+INSERT INTO metadata VALUES (54,'20050201:2','DM11','1.5');
+INSERT INTO metadata VALUES (55,'20050201:2','DM42','1.8.7');
+INSERT INTO metadata VALUES (56,'20050201:2','DM33','1.7.10');
+INSERT INTO metadata VALUES (57,'20050201:2','DM39','1.8.4');
+INSERT INTO metadata VALUES (58,'20050201:2','DM40','1.8.5');
+INSERT INTO metadata VALUES (59,'20050201:2','DM35','1.8');
+INSERT INTO metadata VALUES (60,'20050201:2','DM2','1');
+INSERT INTO metadata VALUES (61,'20050201:2','DM9','1.4.3');
+INSERT INTO metadata VALUES (62,'20050201:2','DM30','1.7.7');
+INSERT INTO metadata VALUES (63,'20050201:2','DM43','1.9');
+INSERT INTO metadata VALUES (64,'20050201:2','DM18','1.5.7');
+INSERT INTO metadata VALUES (65,'20050201:2','DM16','1.5.5');
+INSERT INTO metadata VALUES (66,'20050201:2','DM32','1.7.9');
+INSERT INTO metadata VALUES (67,'20050201:2','DM48','1.11');
+INSERT INTO metadata VALUES (68,'20050201:2','DM23','1.7');
+INSERT INTO metadata VALUES (69,'20050201:2','DM17','1.5.6');
+INSERT INTO metadata VALUES (70,'20050201:2','DM22','1.6.1');
+INSERT INTO metadata VALUES (71,'20050201:2','DM21','1.6');
+INSERT INTO metadata VALUES (72,'20050201:2','DM25','1.7.2');
+INSERT INTO metadata VALUES (73,'20050201:2','DM8','1.4.2');
+INSERT INTO metadata VALUES (74,'20050201:2','DM14','1.5.3');
+INSERT INTO metadata VALUES (75,'20050201:2','DM38','1.8.3');
+INSERT INTO metadata VALUES (76,'20050201:2','DM50','1.11.2');
+INSERT INTO metadata VALUES (77,'20050201:2','DM49','1.11.1');
+INSERT INTO metadata VALUES (78,'20050201:2','DM24','1.7.1');
+INSERT INTO metadata VALUES (79,'20050201:2','DM31','1.7.8');
+INSERT INTO metadata VALUES (80,'20050201:2','DM37','1.8.2');
+INSERT INTO metadata VALUES (81,'20050201:2','DM20','1.5.9');
+INSERT INTO metadata VALUES (82,'20050201:3','DM17','1.2.12');
+INSERT INTO metadata VALUES (83,'20050201:3','DM23','1.5.1');
+INSERT INTO metadata VALUES (84,'20050201:3','DM12','1.2.7');
+INSERT INTO metadata VALUES (85,'20050201:3','DM18','1.3');
+INSERT INTO metadata VALUES (86,'20050201:3','DM4','1.1.1');
+INSERT INTO metadata VALUES (87,'20050201:3','DM8','1.2.3');
+INSERT INTO metadata VALUES (88,'20050201:3','DM16','1.2.11');
+INSERT INTO metadata VALUES (89,'20050201:3','DM1','default');
+INSERT INTO metadata VALUES (90,'20050201:3','DM11','1.2.6');
+INSERT INTO metadata VALUES (91,'20050201:3','DM7','1.2.2');
+INSERT INTO metadata VALUES (92,'20050201:3','DM2','1');
+INSERT INTO metadata VALUES (93,'20050201:3','DM21','1.4');
+INSERT INTO metadata VALUES (94,'20050201:3','DM22','1.5');
+INSERT INTO metadata VALUES (95,'20050201:3','DM19','1.3.1');
+INSERT INTO metadata VALUES (96,'20050201:3','DM3','1.1');
+INSERT INTO metadata VALUES (97,'20050201:3','DM10','1.2.5');
+INSERT INTO metadata VALUES (98,'20050201:3','DM14','1.2.9');
+INSERT INTO metadata VALUES (99,'20050201:3','DM6','1.2.1');
+INSERT INTO metadata VALUES (100,'20050201:3','DM20','1.3.2');
+INSERT INTO metadata VALUES (101,'20050201:3','DM9','1.2.4');
+INSERT INTO metadata VALUES (102,'20050201:3','DM5','1.2');
+INSERT INTO metadata VALUES (103,'20050201:3','DM13','1.2.8');
+INSERT INTO metadata VALUES (104,'20050201:3','DM15','1.2.10');
+INSERT INTO metadata VALUES (105,'20050201:4','DM24','1.7.3');
+INSERT INTO metadata VALUES (106,'20050201:4','DM17','1.5.1');
+INSERT INTO metadata VALUES (107,'20050201:4','DM8','1.3');
+INSERT INTO metadata VALUES (108,'20050201:4','DM7','1.2.3');
+INSERT INTO metadata VALUES (109,'20050201:4','DM21','1.7');
+INSERT INTO metadata VALUES (110,'20050201:4','DM14','1.4.1');
+INSERT INTO metadata VALUES (111,'20050201:4','DM1','default');
+INSERT INTO metadata VALUES (112,'20050201:4','DM18','1.5.2');
+INSERT INTO metadata VALUES (113,'20050201:4','DM6','1.2.2');
+INSERT INTO metadata VALUES (114,'20050201:4','DM11','1.3.3');
+INSERT INTO metadata VALUES (115,'20050201:4','DM2','1');
+INSERT INTO metadata VALUES (116,'20050201:4','DM13','1.4');
+INSERT INTO metadata VALUES (117,'20050201:4','DM20','1.6');
+INSERT INTO metadata VALUES (118,'20050201:4','DM16','1.5');
+INSERT INTO metadata VALUES (119,'20050201:4','DM23','1.7.2');
+INSERT INTO metadata VALUES (120,'20050201:4','DM15','1.4.2');
+INSERT INTO metadata VALUES (121,'20050201:4','DM9','1.3.1');
+INSERT INTO metadata VALUES (122,'20050201:4','DM19','1.5.3');
+INSERT INTO metadata VALUES (123,'20050201:4','DM3','1.1');
+INSERT INTO metadata VALUES (124,'20050201:4','DM12','1.3.4');
+INSERT INTO metadata VALUES (125,'20050201:4','DM22','1.7.1');
+INSERT INTO metadata VALUES (126,'20050201:4','DM5','1.2.1');
+INSERT INTO metadata VALUES (127,'20050201:4','DM10','1.3.2');
+INSERT INTO metadata VALUES (128,'20050201:4','DM4','1.2');
+INSERT INTO metadata VALUES (129,'20050201:5','DM34','1.7.5');
+INSERT INTO metadata VALUES (130,'20050201:5','DM20','1.6.2');
+INSERT INTO metadata VALUES (131,'20050201:5','DM32','1.7.3');
+INSERT INTO metadata VALUES (132,'20050201:5','DM14','1.5.1');
+INSERT INTO metadata VALUES (133,'20050201:5','DM46','1.8');
+INSERT INTO metadata VALUES (134,'20050201:5','DM44','1.7.15');
+INSERT INTO metadata VALUES (135,'20050201:5','DM5','1.3');
+INSERT INTO metadata VALUES (136,'20050201:5','DM35','1.7.6');
+INSERT INTO metadata VALUES (137,'20050201:5','DM21','1.6.3');
+INSERT INTO metadata VALUES (138,'20050201:5','DM15','1.5.2');
+INSERT INTO metadata VALUES (139,'20050201:5','DM2','1');
+INSERT INTO metadata VALUES (140,'20050201:5','DM43','1.7.14');
+INSERT INTO metadata VALUES (141,'20050201:5','DM45','1.7.16');
+INSERT INTO metadata VALUES (142,'20050201:5','DM3','1.1');
+INSERT INTO metadata VALUES (143,'20050201:5','DM10','1.3.5');
+INSERT INTO metadata VALUES (144,'20050201:5','DM17','1.5.4');
+INSERT INTO metadata VALUES (145,'20050201:5','DM7','1.3.2');
+INSERT INTO metadata VALUES (146,'20050201:5','DM23','1.6.5');
+INSERT INTO metadata VALUES (147,'20050201:5','DM33','1.7.4');
+INSERT INTO metadata VALUES (148,'20050201:5','DM36','1.7.7');
+INSERT INTO metadata VALUES (149,'20050201:5','DM4','1.2');
+INSERT INTO metadata VALUES (150,'20050201:5','DM24','1.6.6');
+INSERT INTO metadata VALUES (151,'20050201:5','DM47','1.9');
+INSERT INTO metadata VALUES (152,'20050201:5','DM41','1.7.12');
+INSERT INTO metadata VALUES (153,'20050201:5','DM38','1.7.9');
+INSERT INTO metadata VALUES (154,'20050201:5','DM27','1.6.9');
+INSERT INTO metadata VALUES (155,'20050201:5','DM29','1.7');
+INSERT INTO metadata VALUES (156,'20050201:5','DM1','default');
+INSERT INTO metadata VALUES (157,'20050201:5','DM40','1.7.11');
+INSERT INTO metadata VALUES (158,'20050201:5','DM28','1.6.10');
+INSERT INTO metadata VALUES (159,'20050201:5','DM26','1.6.8');
+INSERT INTO metadata VALUES (160,'20050201:5','DM8','1.3.3');
+INSERT INTO metadata VALUES (161,'20050201:5','DM12','1.4');
+INSERT INTO metadata VALUES (162,'20050201:5','DM19','1.6.1');
+INSERT INTO metadata VALUES (163,'20050201:5','DM13','1.5');
+INSERT INTO metadata VALUES (164,'20050201:5','DM18','1.6');
+INSERT INTO metadata VALUES (165,'20050201:5','DM22','1.6.4');
+INSERT INTO metadata VALUES (166,'20050201:5','DM31','1.7.2');
+INSERT INTO metadata VALUES (167,'20050201:5','DM6','1.3.1');
+INSERT INTO metadata VALUES (168,'20050201:5','DM16','1.5.3');
+INSERT INTO metadata VALUES (169,'20050201:5','DM9','1.3.4');
+INSERT INTO metadata VALUES (170,'20050201:5','DM39','1.7.10');
+INSERT INTO metadata VALUES (171,'20050201:5','DM25','1.6.7');
+INSERT INTO metadata VALUES (172,'20050201:5','DM30','1.7.1');
+INSERT INTO metadata VALUES (173,'20050201:5','DM42','1.7.13');
+INSERT INTO metadata VALUES (174,'20050201:5','DM37','1.7.8');
+INSERT INTO metadata VALUES (175,'20050201:5','DM11','1.3.6');
+INSERT INTO metadata VALUES (176,'20050201:6','DM1','default');
+
+--
+-- Table structure for table `namespaces`
+--
+
+CREATE TABLE namespaces (
+  NamespaceRef int(11) NOT NULL auto_increment,
+  MetadataRef int(11) default NULL,
+  DocID varchar(64) default NULL,
+  MetaID varchar(64) default NULL,
+  NamespaceID varchar(64) default NULL,
+  NamespaceType varchar(64) default NULL,
+  FileType varchar(64) default NULL,
+  FileLoc tinytext,
+  Creator varchar(128) default NULL,
+  PRIMARY KEY  (NamespaceRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `namespaces`
+--
+
+INSERT INTO namespaces VALUES (1,1,NULL,NULL,'1','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (2,2,NULL,NULL,'2','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (3,3,NULL,NULL,'3','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (4,4,NULL,NULL,'4','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (5,5,NULL,NULL,'5','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (6,6,NULL,NULL,'6','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (7,7,NULL,NULL,'7','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (8,8,NULL,NULL,'8','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (9,9,NULL,NULL,'9','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (10,10,NULL,NULL,'10','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (11,11,NULL,NULL,'11','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (12,12,NULL,NULL,'12','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (13,13,NULL,NULL,'13','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (14,14,NULL,NULL,'14','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (15,15,NULL,NULL,'15','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (16,16,NULL,NULL,'16','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (17,17,NULL,NULL,'17','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (18,18,NULL,NULL,'18','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (19,19,NULL,NULL,'19','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (20,20,NULL,NULL,'20','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (21,21,NULL,NULL,'21','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (22,22,NULL,NULL,'22','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (23,23,NULL,NULL,'23','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (24,24,NULL,NULL,'24','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (25,25,NULL,NULL,'25','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (26,26,NULL,NULL,'26','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (27,27,NULL,NULL,'27','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (28,28,NULL,NULL,'28','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (29,29,NULL,NULL,'29','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (30,30,NULL,NULL,'30','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (31,31,NULL,NULL,'31','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (32,32,NULL,NULL,'32','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (33,33,NULL,NULL,'33','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (34,34,NULL,NULL,'34','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (35,35,NULL,NULL,'35','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (36,36,NULL,NULL,'36','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (37,37,NULL,NULL,'37','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (38,38,NULL,NULL,'38','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (39,39,NULL,NULL,'39','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (40,40,NULL,NULL,'40','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (41,41,NULL,NULL,'41','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (42,42,NULL,NULL,'42','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (43,43,NULL,NULL,'43','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (44,44,NULL,NULL,'44','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (45,45,NULL,NULL,'45','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (46,46,NULL,NULL,'46','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (47,47,NULL,NULL,'47','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (48,48,NULL,NULL,'48','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (49,49,NULL,NULL,'49','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (50,50,NULL,NULL,'50','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (51,51,NULL,NULL,'51','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (52,52,NULL,NULL,'52','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (53,53,NULL,NULL,'53','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (54,54,NULL,NULL,'54','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (55,55,NULL,NULL,'55','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (56,56,NULL,NULL,'56','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (57,57,NULL,NULL,'57','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (58,58,NULL,NULL,'58','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (59,59,NULL,NULL,'59','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (60,60,NULL,NULL,'60','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (61,61,NULL,NULL,'61','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (62,62,NULL,NULL,'62','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (63,63,NULL,NULL,'63','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (64,64,NULL,NULL,'64','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (65,65,NULL,NULL,'65','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (66,66,NULL,NULL,'66','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (67,67,NULL,NULL,'67','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (68,68,NULL,NULL,'68','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (69,69,NULL,NULL,'69','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (70,70,NULL,NULL,'70','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (71,71,NULL,NULL,'71','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (72,72,NULL,NULL,'72','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (73,73,NULL,NULL,'73','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (74,74,NULL,NULL,'74','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (75,75,NULL,NULL,'75','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (76,76,NULL,NULL,'76','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (77,77,NULL,NULL,'77','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (78,78,NULL,NULL,'78','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (79,79,NULL,NULL,'79','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (80,80,NULL,NULL,'80','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (81,81,NULL,NULL,'81','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (82,82,NULL,NULL,'82','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (83,83,NULL,NULL,'83','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (84,84,NULL,NULL,'84','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (85,85,NULL,NULL,'85','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (86,86,NULL,NULL,'86','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (87,87,NULL,NULL,'87','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (88,88,NULL,NULL,'88','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (89,89,NULL,NULL,'89','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (90,90,NULL,NULL,'90','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (91,91,NULL,NULL,'91','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (92,92,NULL,NULL,'92','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (93,93,NULL,NULL,'93','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (94,94,NULL,NULL,'94','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (95,95,NULL,NULL,'95','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (96,96,NULL,NULL,'96','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (97,97,NULL,NULL,'97','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (98,98,NULL,NULL,'98','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (99,99,NULL,NULL,'99','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (100,100,NULL,NULL,'100','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (101,101,NULL,NULL,'101','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (102,102,NULL,NULL,'102','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (103,103,NULL,NULL,'103','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (104,104,NULL,NULL,'104','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (105,105,NULL,NULL,'105','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (106,106,NULL,NULL,'106','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (107,107,NULL,NULL,'107','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (108,108,NULL,NULL,'108','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (109,109,NULL,NULL,'109','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (110,110,NULL,NULL,'110','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (111,111,NULL,NULL,'111','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (112,112,NULL,NULL,'112','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (113,113,NULL,NULL,'113','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (114,114,NULL,NULL,'114','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (115,115,NULL,NULL,'115','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (116,116,NULL,NULL,'116','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (117,117,NULL,NULL,'117','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (118,118,NULL,NULL,'118','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (119,119,NULL,NULL,'119','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (120,120,NULL,NULL,'120','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (121,121,NULL,NULL,'121','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (122,122,NULL,NULL,'122','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (123,123,NULL,NULL,'123','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (124,124,NULL,NULL,'124','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (125,125,NULL,NULL,'125','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (126,126,NULL,NULL,'126','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (127,127,NULL,NULL,'127','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (128,128,NULL,NULL,'128','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (129,129,NULL,NULL,'129','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (130,130,NULL,NULL,'130','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (131,131,NULL,NULL,'131','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (132,132,NULL,NULL,'132','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (133,133,NULL,NULL,'133','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (134,134,NULL,NULL,'134','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (135,135,NULL,NULL,'135','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (136,136,NULL,NULL,'136','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (137,137,NULL,NULL,'137','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (138,138,NULL,NULL,'138','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (139,139,NULL,NULL,'139','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (140,140,NULL,NULL,'140','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (141,141,NULL,NULL,'141','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (142,142,NULL,NULL,'142','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (143,143,NULL,NULL,'143','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (144,144,NULL,NULL,'144','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (145,145,NULL,NULL,'145','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (146,146,NULL,NULL,'146','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (147,147,NULL,NULL,'147','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (148,148,NULL,NULL,'148','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (149,149,NULL,NULL,'149','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (150,150,NULL,NULL,'150','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (151,151,NULL,NULL,'151','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (152,152,NULL,NULL,'152','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (153,153,NULL,NULL,'153','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (154,154,NULL,NULL,'154','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (155,155,NULL,NULL,'155','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (156,156,NULL,NULL,'156','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (157,157,NULL,NULL,'157','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (158,158,NULL,NULL,'158','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (159,159,NULL,NULL,'159','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (160,160,NULL,NULL,'160','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (161,161,NULL,NULL,'161','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (162,162,NULL,NULL,'162','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (163,163,NULL,NULL,'163','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (164,164,NULL,NULL,'164','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (165,165,NULL,NULL,'165','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (166,166,NULL,NULL,'166','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (167,167,NULL,NULL,'167','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (168,168,NULL,NULL,'168','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (169,169,NULL,NULL,'169','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (170,170,NULL,NULL,'170','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (171,171,NULL,NULL,'171','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (172,172,NULL,NULL,'172','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (173,173,NULL,NULL,'173','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (174,174,NULL,NULL,'174','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (175,175,NULL,NULL,'175','gsdl3',NULL,NULL,NULL);
+
+--
+-- Table structure for table `structure`
+--
+
+CREATE TABLE structure (
+  StructureRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  StructureID varchar(64) default NULL,
+  StructureType varchar(64) default NULL,
+  Label varchar(128) default NULL,
+  PRIMARY KEY  (StructureRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `structure`
+--
+
+INSERT INTO structure VALUES (1,'20050201:1','Section','Section','Section');
+INSERT INTO structure VALUES (2,'20050201:1','All','Whole Document','All');
+INSERT INTO structure VALUES (3,'20050201:2','Section','Section','Section');
+INSERT INTO structure VALUES (4,'20050201:2','All','Whole Document','All');
+INSERT INTO structure VALUES (5,'20050201:3','Section','Section','Section');
+INSERT INTO structure VALUES (6,'20050201:3','All','Whole Document','All');
+INSERT INTO structure VALUES (7,'20050201:4','Section','Section','Section');
+INSERT INTO structure VALUES (8,'20050201:4','All','Whole Document','All');
+INSERT INTO structure VALUES (9,'20050201:5','Section','Section','Section');
+INSERT INTO structure VALUES (10,'20050201:5','All','Whole Document','All');
+INSERT INTO structure VALUES (11,'20050201:6','All','Whole Document','All');
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/Build.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/Build.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/Build.java	(revision 14150)
@@ -0,0 +1,75 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class Build
+{
+  BuildManager manager;
+
+  public Build(List inputRoots, String site, String collection, String outputDir)
+  { this.manager = new BuildManager(inputRoots, site, collection, outputDir);
+  }
+
+  public void run()
+  { this.manager.run();
+  }
+
+  public static void main(String args[])
+  { int a = 0;
+    List inputDirs = new ArrayList();
+    String collection = null;
+    String site = null;
+    String outputDir  = null;
+
+    while (a < args.length)
+    { System.out.println(args[a]);
+      if (args[a].equals("-inputdir")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  inputDirs.add(args[a+1]);
+	  a ++;	
+	}
+      }
+      else if (args[a].equals("-collect")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  collection = args[a+1];
+	  a ++;
+	}
+      }
+      else if (args[a].equals("-site")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  site = args[a+1];
+	  a ++;
+	}
+      }	  
+      else if (args[a].equals("-outputDir")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{ outputDir = args[a+1];
+	  a ++;
+	}
+      }
+      a ++;
+    }
+
+    if (site == null) {
+	System.err.println("you must specify the site name");
+	System.exit(0);
+    }
+    if (collection == null && inputDirs.size() == 0) {
+	System.err.println("At least one input directory must be given, or a collection name");
+	System.exit(0);
+    }
+    
+    Build build = new Build(inputDirs, site, collection, outputDir);
+    build.run();
+
+    return;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/BuildManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/BuildManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/BuildManager.java	(revision 14150)
@@ -0,0 +1,154 @@
+package org.greenstone.gsdl3.gs3build;
+
+import org.greenstone.gsdl3.gs3build.doctypes.*;
+import org.greenstone.gsdl3.gs3build.indexers.*;
+import org.greenstone.gsdl3.gs3build.notifier.NotifierManager;
+import org.greenstone.gsdl3.gs3build.extractor.*;
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.collection.*;
+
+import java.io.File;
+import java.util.List;
+
+public class BuildManager 
+{
+    CollectionManager collectionManager;
+    RecogniserManager recogniserManager;
+    IndexerManager    indexerManager;
+    ClassifierManager classifierManager;
+    ExtractorManager  extractorManager;
+    NotifierManager   notifierManager;
+    DocumentList      docList;
+    List              inputRoots;
+    String            outputDir;
+    String            archiveDir;
+    
+    public BuildManager(List inputRoots, String site, String collection, String outputDir) 
+    {
+	// set up the collection manager
+	this.collectionManager = new CollectionManager(site, collection);
+	this.collectionManager.setBuildManager(this);
+	
+	// set up the list of input directories
+	if (inputRoots.size() == 0) {
+	    inputRoots.add(this.collectionManager.getImportDirectory());
+	    System.out.println("Adding collection import directory to input list " + this.collectionManager.getImportDirectory());
+	}
+	this.inputRoots = inputRoots;
+	
+	// set up the output directories (building and archive)
+	this.outputDir  = outputDir != null ? outputDir : this.collectionManager.getBuildDirectory();
+	File outputDirFile = new File(this.outputDir);
+	if (!outputDirFile.isDirectory()) {
+	    System.err.println("Unable to open directory " + this.outputDir + " for writing");
+	    System.exit(1);
+	}
+	System.out.println("Output directory set to " + this.outputDir);
+	
+	// set up the archive directory - this will definitely exist
+	this.archiveDir = this.collectionManager.getArchiveDirectory();
+
+	// the list that documents get added to when they are recognised
+	this.docList = new DocumentList(new DocumentIDFactory(this.collectionManager), 
+					this.collectionManager.getDatabase());
+	
+	// set up the various managers
+	this.recogniserManager = new RecogniserManager(this.docList);
+	this.extractorManager  = new ExtractorManager(this.docList);
+	this.classifierManager = new ClassifierManager(this.docList, collectionManager.getDatabase());
+	this.indexerManager = new IndexerManager(this.docList);
+	this.notifierManager = new NotifierManager();
+	
+	// configure the collection - this will add classifiers, indexers, recognisers to the various managers.
+	this.collectionManager.configureCollection();
+	
+	// add in expanders by hand cos they are not in the config file yet
+	// TODO
+	// add in extractors by hand cos they are not in the config file yet
+	this.extractorManager.addExtractor(new IndexExtractor());
+	this.extractorManager.addExtractor(new MetaXMLExtractor());
+	
+	// add in enrichers by hand cos they are not in the config file yet
+	// TODO
+		
+	// prepare a file crawler on the etc directory, and a crawl observer to respond to
+	// the appropriate files.
+	FileCrawler fileCrawler = new FileCrawler(new File(this.collectionManager.getEtcDirectory()), 
+						  new CollectionCrawlObserver(this));
+
+	fileCrawler.crawl();    
+    }
+  
+    public String getLastBuildSequence()
+    { 
+	return this.collectionManager.getCollectionMetadata("gsdl3", "buildsequence");
+    }
+
+    public ClassifierManager getClassifierManager()
+    { 
+	return this.classifierManager;
+    }
+
+    public IndexerManager getIndexerManager()
+    { 
+	return this.indexerManager;
+    }
+
+    // TODO: validation phase
+    public RecogniserManager getRecogniserManager()
+    {
+	return this.recogniserManager;
+    }
+    
+    public void addIndexer(IndexerInterface iface)
+    { 
+	iface.configure(IndexerManager.outputDir, this.outputDir);
+	this.indexerManager.addIndexer(iface);
+    }
+
+    public void run()
+    { 
+	// Initialise collection manager - note start of build
+	this.collectionManager.startBuild();
+	
+	// TODO: add expansion (e.g. Zip files)
+    
+	// Crawl the file tree - will recognise documents
+	for (int i = 0; i < this.inputRoots.size(); i ++) {
+	    FileCrawler fileCrawler = new FileCrawler(new File((String) this.inputRoots.get(i)), recogniserManager);
+
+	    fileCrawler.crawl();
+	}
+
+	// Extract phase, etc.
+	this.extractorManager.extractDocuments();
+	this.classifierManager.classifyDocuments();
+	this.indexerManager.indexDocuments();
+
+	// Timestamp management - update all timestamps on modified dates...
+	// 
+	// This should only occur at the end of building in case the build is cancelled...
+	docList.updateTimestamps(this.collectionManager.getBuildTimestamp());
+	
+	// TODO: validation phase
+	this.notifierManager.detectEvents(this.collectionManager);
+	
+	if (this.archiveDir != null) {
+	    this.docList.writeDocuments(new File(this.archiveDir));
+	    //this.docList.writeSQLDocuments(this.collectionManager.getDatabase());
+	}
+	
+	this.collectionManager.setCollectionMetadata("gsdl3", "documentCount", Integer.toString(this.docList.getCount()));
+    
+	// TODO: write out collection configuration file (service lists, etc.)
+    
+	this.collectionManager.endBuild();
+    }
+  
+    public void addInputDirectory(String inputDir)
+    {
+	this.inputRoots.add(inputDir);
+    }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 14150)
@@ -0,0 +1,595 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.util.Date;
+import java.util.Calendar;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.GregorianCalendar;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+
+import java.net.URL;
+import java.net.URLEncoder;
+
+import java.sql.*;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+import org.greenstone.gsdl3.gs3build.collection.*;
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.indexers.*;
+import org.greenstone.gsdl3.gs3build.doctypes.RecogniserInterface;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+/**
+ *  Store and hold collection-level configuration information for a collection.
+ *  This should be used by BuildManager to work out which classes, etc. to load
+ *  at build time, and as a repository for the collection-level metadata, and
+ *  a means of loading and saving the same to a file or database, as is seen
+ *  fit in the final development of gs3.
+ */
+
+public class CollectionManager
+{
+  GregorianCalendar  lastBuildDate;  // pretty obvious
+  String             adminEmail;     // the email address of the administrator of the
+		                     // collection
+  int                buildDocNo;     // used to generate document identifiers
+  CollectionMetadata metadata;       // collection-level metadata
+  GS3SQLConnection   database;       // the database to store everything in
+  public String             collectionHome;
+  public String             siteHome;
+  public String             collectionName;
+  public String             qualifiedCollectionName; // used as the database name 
+  String             notifyHost;
+
+  BuildManager       buildManager;
+   
+  class CollectionClassifier
+  { URL    file;
+    String type;
+    List fields;
+    String sort;
+    String className;
+
+    public CollectionClassifier(URL parentURL, String type, String className, Node node)
+    { this.type = type;
+      this.className = className;
+      this.fields = new ArrayList();
+
+      NodeList children = node.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++) {
+	Node child = children.item(c);
+
+	if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  String name = child.getNodeName();
+	  
+	  if (name.equals("file")) {
+	    NamedNodeMap atts = children.item(c).getAttributes();
+	    Node attribute = atts.getNamedItem("URL");
+	    String urlString = attribute.getNodeValue();
+	    if (urlString == null)
+	      continue;
+
+	    System.out.println("Path is" + parentURL.getPath());
+	    System.out.println("Host is" + parentURL.getHost());
+	    System.out.println(urlString);
+
+	    try {
+	      URL url = new URL(parentURL, urlString);
+	      this.file = url;
+	      System.out.println(url);
+	    }
+	    catch (java.net.MalformedURLException malEx) {
+	      System.out.println(malEx);
+	    }
+	  }
+	  else if (name.equals("field")) {
+	    String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	    this.fields.add(fieldName.toString());
+	  }
+	  else if (name.equals("sort")) {
+	    String sortName = DOMUtils.getNodeChildText(children.item(c));
+	    this.sort = sortName;
+	  }
+	}
+      }
+    }
+
+    public ClassifierInterface getClassifier()
+    { ClassifierInterface classifier = null;
+
+      if (this.type == null) {
+	return null;
+      }
+      System.out.println("Creating a classifier of type " + this.type.toLowerCase());
+      
+      if (this.type.toLowerCase().equals("hierarchy")) {
+	System.out.println("  hierarchy file is " + this.file);
+	classifier = new HierarchyClassifier(this.className, this.file, this.fields, this.sort);
+      }
+      else if (this.type.toLowerCase().equals("azlist")) {
+	classifier = new AZListClassifier(this.className, this.fields);
+      }
+
+      return classifier;
+    }
+  }
+   
+  /**
+   *  Create the collection manager for a given collection
+   *
+   * @param site the name of the site
+   *  @param collection <code>String</code> the name of the collection
+   */
+  public CollectionManager(String site, String collection) {
+    
+    String gsdl3Root = System.getProperty("GSDL3HOME");
+    if (gsdl3Root == null) {
+      System.out.println("Error: Unable to locate GSDL3HOME");
+      System.exit(1);
+      //return;
+    }
+    
+    this.siteHome = GSFile.siteHome(gsdl3Root, site);
+    File site_dir = new File(this.siteHome);
+    System.out.println(site_dir);
+    if (!site_dir.exists()) {
+      System.out.println("Error: Non-existant site ("+site+") specified");
+      System.exit(1);
+    }
+    site_dir = null;
+    this.collectionHome = GSFile.collectionBaseDir(this.siteHome, collection);
+    
+    File collect_dir = new File(this.collectionHome);
+    if (!collect_dir.exists()) {
+      System.out.println("Error: Non-existant collection ("+collection+") specified in site "+site);
+      System.exit(1);
+    }
+    collect_dir = null;
+
+    this.collectionName = collection;
+    this.qualifiedCollectionName = site+"_"+collection;
+
+    this.database = GS3SQLConnectionFactory.getGS3SQLConnection(this.qualifiedCollectionName);
+    /*    if (this.database != null) {
+      this.database.clearCollection(collection);
+      this.database = null;
+    }
+    */
+    if (this.database == null) {
+	this.database = GS3SQLConnectionFactory.getGS3SQLConnection("test");
+	if (this.database == null) {
+	    System.err.println("Can't connect to the mysql database. Please make sure your mysql is running, and that the correct passwords (if any) are specified in "+gsdl3Root+"/WEB-INF/classes/global.properties");
+	    System.exit(1);
+	}
+	boolean success = this.database.initCollection(this.qualifiedCollectionName);
+	if (!success) {
+	    System.err.println("couldn't init collection " + this.qualifiedCollectionName);
+	    System.exit(1);
+	}
+    }
+
+    this.metadata = new CollectionMetadata();
+
+    File buildDirectory = new File(getBuildDirectory());
+    if (!buildDirectory.exists()) {
+      buildDirectory.mkdir();
+    }
+    if (!buildDirectory.isDirectory()) {
+	System.err.println("Unable to open directory " + buildDirectory + " for writing");
+	System.exit(1);
+    }
+
+    File archiveDirectory = new File(getArchiveDirectory());
+    if (!archiveDirectory.exists()) {
+      archiveDirectory.mkdir();
+    }
+    if (!archiveDirectory.isDirectory()) {
+	System.err.println("Unable to open directory " + archiveDirectory + " for writing");
+	System.exit(1);
+    }
+
+    this.buildDocNo = 1;
+
+    try {
+      GS3SQLSelect select = new GS3SQLSelect("build");
+      select.addField("*");
+      Statement statement = this.database.createStatement();
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	System.out.println("Reading all keys");
+	do {
+	  String key = results.getString("buildKey");
+	  String value = results.getString("buildValue");
+
+	  if (key.equals("NextSeqNo")) {
+	    this.buildDocNo = Integer.parseInt(value);
+	  }
+	  else if (key.equals("lastBuildDate")) {
+	    int year, month, day;
+	    year = Integer.parseInt(value.substring(0, 4));
+	    month = Integer.parseInt(value.substring(4, 6)) - 1; // -1 because Gregorian Calendar perversely treats January as 0, etc.
+	    day = Integer.parseInt(value.substring(6, 8));
+	    
+	    this.lastBuildDate = new GregorianCalendar(year, month, day);
+	  }
+	} while (results.next());
+      }
+      statement.close();
+    }
+    catch (SQLException ex)
+      { System.out.println(ex);
+      }
+  }
+
+  public void setBuildManager(BuildManager build_man) 
+  { this.buildManager = build_man;
+  }
+
+  private void configureBrowsers(Node node, File etcFile)
+  { CollectionClassifier classifier = null;
+    URL etcURL = null;
+
+    etcURL = URLTools.getFileURL(etcFile);
+
+    NodeList children = node.getChildNodes();
+    for (int c = 0; c < children.getLength(); c ++)
+    { // assume that non-element children are irrelevant
+      if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE)
+      { continue;
+      }
+
+      String name = children.item(c).getNodeName();
+      System.out.println(name);
+
+      if (name.equals(GSXML.CLASSIFIER_ELEM))
+      { NamedNodeMap atts = children.item(c).getAttributes();
+
+        // get the type attribute
+	Node attribute = atts.getNamedItem(GSXML.TYPE_ATT);
+	if (attribute == null) {
+	  continue;
+	}
+	String type = attribute.getNodeValue();
+
+        // get the type attribute
+	attribute = atts.getNamedItem(GSXML.NAME_ATT);
+	if (attribute == null) {
+	  continue;
+	}
+	String className = attribute.getNodeValue();
+
+	classifier = new CollectionClassifier(etcURL, type, className, children.item(c));
+	
+	System.out.println("Found classifier " + type);
+	
+	// attach the classifier
+	ClassifierInterface classify = classifier.getClassifier();
+	this.buildManager.getClassifierManager().addClassifier(classify);
+      }  
+    }
+  }
+
+  public void configureCollection()
+  { File collectionConfig = new File(GSFile.collectionConfigFile(this.collectionHome));
+
+    // get the File and read it in
+    try
+    {
+      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder = factory.newDocumentBuilder();
+      Document document = builder.parse(collectionConfig);
+
+      // TODO: report an error
+      if (document == null)
+      {	
+      }
+
+      // now parse the manager file...
+      Element rootElement = document.getDocumentElement();
+
+      if (rootElement.getTagName() != GSXML.COLLECTION_CONFIG_ELEM)
+      {	// TODO: throw exception
+      }
+
+      System.out.println("Configuring collection");
+
+      NodeList children = rootElement.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++)
+      {	// assume that non-element children are irrelevant
+	if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE)
+	{ continue;
+	}
+
+	String name = children.item(c).getNodeName();
+
+	// the name is a plugin element
+
+	if (name.equals(GSXML.RECOGNISE_ELEM)) {
+	    System.out.println("recognise elem");
+	    
+	    NodeList doc_types = ((Element)children.item(c)).getElementsByTagName(GSXML.DOC_TYPE_ELEM);
+	    for(int i=0; i<doc_types.getLength(); i++) {
+		Element doc_type = (Element)doc_types.item(i);
+		String type = doc_type.getAttribute(GSXML.NAME_ATT);
+		System.err.println("adding recogniser, type "+type);
+		RecogniserInterface ri = this.buildManager.getRecogniserManager().addRecogniser(type);
+		if (ri != null) {
+		    ri.setCollectionManager(this);
+		    ri.configure(doc_type);
+		}
+	    }
+	} else if (name.equals(GSXML.SEARCH_ELEM)) {
+	  // pick up attributes from the <search> tag now...
+	  NamedNodeMap searchAttributes = children.item(c).getAttributes();
+	  Node searchAttribute = searchAttributes.getNamedItem(GSXML.TYPE_ATT);
+	  String searchType = null;
+	  if (searchAttribute != null) {
+	      searchType = searchAttribute.getNodeValue();
+	  } else {
+	      System.out.println("no "+GSXML.TYPE_ATT+" attribute found for the "+GSXML.SEARCH_ELEM+" element, assuming mg");
+	      searchType = MGIndexer.MG_INDEX_TYPE;
+	  }
+	  
+	  searchAttribute = searchAttributes.getNamedItem(GSXML.NAME_ATT);
+	  String searchName = null;
+	  if (searchAttribute != null) {
+	      searchName = searchAttribute.getNodeValue();
+	  }
+	  if (searchName == null) {
+	      searchName = "idx"; // need to modify this if we have two search elements with no names
+	  }
+	  // create the pertinent indexer...
+	  IndexerInterface indexer = IndexerFactory.makeIndexer(searchType, searchName);
+
+	  if (indexer == null) {
+	    continue;
+	  }
+
+	  // configure the indexer
+	  indexer.configure(children.item(c));
+
+	  // install it into the build manager
+	  this.buildManager.addIndexer(indexer);
+	}
+	else if (name.equals(GSXML.BROWSE_ELEM))
+	{ this.configureBrowsers(children.item(c), collectionConfig);
+	}
+	else if (name.equals(GSXML.NOTIFY_ELEM))
+	{
+	    this.notifyHost = ((Element) children.item(c)).getAttribute(GSXML.NOTIFY_HOST_ATT);
+	}
+	// TODO: other elements - make a factory-method approach here...
+	else
+	{
+	}
+      }
+    }
+    catch (FactoryConfigurationError e) {
+      System.out.println(e);
+    }
+    catch (ParserConfigurationException ex) {
+      System.out.println(ex);
+    }
+    catch (SAXException ex) {
+      System.out.println(ex);
+    }
+    catch (IOException ex)
+    {
+      System.out.println(ex);
+    }
+
+    System.out.println("<<<Obtaining database>>>>");
+  }
+
+  public String getEtcDirectory()
+  { return GSFile.collectionEtcDir(this.collectionHome);
+  }
+
+  public String getImportDirectory()
+  { return GSFile.collectionImportDir(this.collectionHome);
+  }
+
+  public String getBuildDirectory()
+  { return GSFile.collectionBuildDir(this.collectionHome);
+  }
+
+  public String getArchiveDirectory()
+  { return GSFile.collectionArchiveDir(this.collectionHome);
+  }
+
+  public GS3SQLConnection getDatabase()
+  {
+    return this.database;
+  }
+
+  public long getBuildTimestamp()
+  { return this.lastBuildDate.getTime().getTime();
+  }
+
+  public Date getBuildDate()
+  { return this.lastBuildDate.getTime();
+  }
+
+  public void startBuild()
+  { GregorianCalendar today = new GregorianCalendar();
+  
+    if (this.lastBuildDate != null)
+    { // if the build date is different to the last build date, then reset the build
+      // document number
+      if (today.get(Calendar.YEAR) != this.lastBuildDate.get(Calendar.YEAR) ||
+	  today.get(Calendar.MONTH) != this.lastBuildDate.get(Calendar.MONTH) ||
+	  today.get(Calendar.DAY_OF_MONTH) != this.lastBuildDate.get(Calendar.DAY_OF_MONTH))
+      { this.buildDocNo = 1;
+      }
+      else 
+      { System.out.println("Continuing build sequence from " + this.buildDocNo);
+      }
+    }
+    this.lastBuildDate = today;
+  }
+
+  public void endBuild()
+  {
+    // here we write out the build config file
+    // create the xml for the buildConfig
+    XMLConverter converter = new XMLConverter();
+    Document doc = converter.newDOM();
+    Element build_config = doc.createElement(GSXML.COLLECTION_BUILD_ELEM);
+    Element meta_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    build_config.appendChild(meta_list);
+    Element service_list = doc.createElement(GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+    build_config.appendChild(service_list);
+    
+    // ask the indexers and classifiers to add stuff into the service rack list
+    this.buildManager.getIndexerManager().addServiceDescriptions(service_list);
+    this.buildManager.getClassifierManager().addServiceDescriptions(service_list);
+    // get the String
+    String build_config_string = converter.getString(build_config);
+    // write it to the file
+    try {
+      File build_config_file = new File(GSFile.collectionBuildConfigFileBuilding(this.collectionHome));
+      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(build_config_file), "UTF-8"));
+      writer.write(build_config_string, 0, build_config_string.length());
+      writer.close();
+    } catch (Exception e) {
+      System.err.println("CollectionManager.endBuild() Error while trying to output the buildConfig.xml file.");
+      System.err.println(e.getMessage());
+    }
+    
+    // Update build date information
+    try {
+      GS3SQLDelete remove = new GS3SQLDelete("build");
+      //    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("buildKey", "=", "NextSeqNo"));
+      //    rem
+      Statement statement = this.database.createStatement();
+      statement.execute(remove.toString());
+      
+      GS3SQLInsert insert = new GS3SQLInsert("build");
+      insert.addValue("buildKey", "NextSeqNo");
+      insert.addValue("buildValue", Integer.toString(this.buildDocNo));
+      statement.execute(insert.toString());
+    
+      insert = new GS3SQLInsert("build");
+      insert.addValue("buildKey", "lastBuildDate");
+      insert.addValue("buildValue", getDateString(this.lastBuildDate));
+      statement.execute(insert.toString());
+      statement.close();
+    } catch (SQLException e) {
+	System.err.println("CollectionManager.endBuild(): Can't update build information: "+e);
+    }
+    
+    // Do tail of build output
+    Date startDate = this.lastBuildDate.getTime();
+    Date date = new Date();
+
+    long startTime = startDate.getTime();
+    long endTime   = date.getTime();
+
+    long difference = ((endTime - startTime) + 500) / 1000;
+
+    System.out.println("Build completed");
+    System.out.println("---------------");
+    System.out.println("Total Documents: " + this.getCollectionMetadata("gsdl3", "documentCount"));
+    System.out.println("Total Time :     " + (difference / 60) + " min. " + (difference % 60) + " secs.");
+  }
+
+  private static String getDateString(GregorianCalendar date)
+  { StringBuffer dateString = new StringBuffer();
+  
+    int value;
+    dateString.append(date.get(Calendar.YEAR));
+
+    // the use of month is a little odd, hence the following
+    // code.  Calendar.MONTH yields 0 = January, 1 = February,
+    // etc. hence there is a '+1' added to the month to make 
+    // it into January = 1, etc., and the padding is altered
+    // correspondingly.
+    value = date.get(Calendar.MONTH);
+    if (value < 9)
+    { dateString.append("0");
+    }
+    dateString.append(value + 1);
+    value = date.get(Calendar.DAY_OF_MONTH);
+    if (value < 10)
+      dateString.append("0");
+    dateString.append(value);
+
+    return dateString.toString();
+  }
+
+
+  public String getNextDocumentID()
+  { StringBuffer ID = new StringBuffer(getDateString(this.lastBuildDate));
+
+    int value = this.buildDocNo;
+    this.buildDocNo ++;
+    
+    ID.append(":");
+    ID.append(Integer.toString(value));
+    return ID.toString();
+  }
+  
+  public int getDocumentNumber()
+  { this.buildDocNo ++;
+    return this.buildDocNo - 1;
+  }
+
+  /**
+   *  Get the collection metadata item in the given namespace
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   */
+  public String getCollectionMetadata(String namespace, String label)
+  { return this.metadata.getCollectionMetadata(namespace, label).get(0).toString();
+  }
+
+  /**
+   *  Set the collection metadata item in the given namespace
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label
+   *  @param <code>String</code> the value
+   */
+  public void setCollectionMetadata(String namespace, String label, String value)
+  { this.metadata.setCollectionMetadata(namespace, label, value);
+  }
+  
+  public String getCollectionName() {
+      return collectionName;
+  }
+
+/**
+ * @return
+ */
+public String getNotifyHost() {
+    return notifyHost;
+}
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/Create.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/Create.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/Create.java	(revision 14150)
@@ -0,0 +1,116 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+import org.greenstone.gsdl3.util.GSFile;
+
+public class Create
+{
+    String gsdl3_home = null;
+    String gsdl3_srchome = null;
+    String site = null;
+    String site_home = null;
+    String collection = null;
+
+    public Create(String gsdl3_home, String gsdl3_srchome, String site, 
+		  String collection)
+    { 
+	this.gsdl3_home = gsdl3_home;
+	this.gsdl3_srchome = gsdl3_srchome;
+	this.site = site;
+	this.site_home = GSFile.siteHome(gsdl3_home, site);
+	this.collection = collection;
+
+	// check the site's collect directory
+	File site_collect = new File(GSFile.collectDir(this.site_home));
+	if (!site_collect.isDirectory()) {
+	    System.err.println("Site's collect directory "+site_collect.getPath()+" doesn't exist. Please create it and try again");
+	    System.exit(1);
+	}
+
+       	
+    }
+    
+    public void makeCollection() {
+	
+	String coll_home = GSFile.collectionBaseDir(this.site_home, this.collection);
+	File coll_dir_file = new File(coll_home);
+	if (coll_dir_file.exists()) {
+	    System.err.println("Collection directory ("+coll_dir_file.getPath()+") already exists, not creating it.");
+	    System.exit(1);
+	}
+	createDirectory(coll_dir_file);
+	createDirectory(new File(GSFile.collectionImportDir(coll_home)));
+	createDirectory(new File(GSFile.collectionEtcDir(coll_home)));
+
+	File config_template = new File(gsdl3_srchome + File.separator+ "resources"+File.separator + "xml"+File.separator+"collectionConfig.xml");
+	if (!config_template.isFile()) {
+	    System.err.println("Couldn't find the config file template "+config_template.getPath());
+	    System.exit(1);
+	}
+
+	File coll_config = new File(GSFile.collectionConfigFile(coll_home));
+	GSFile.copyFile(config_template, coll_config);
+
+	System.err.println("New collection created in "+coll_home);
+	System.err.println("Put source documents into import, set the collection's configuration in etc/collectionConfig.xml, then run \"gs3-build.(sh|bat) "+ this.site+" "+this.collection+"\" to build the collection");
+    }
+
+    protected void createDirectory(File dir) {
+	dir.mkdir();
+	if (!dir.isDirectory()) {
+	    System.err.println("Couldn't create directory "+dir.getPath());
+	    System.exit(1);
+	}
+    }
+
+    public static void main(String args[])
+    { 
+	String gsdl3_home = System.getProperty("GSDL3HOME");
+	String gsdl3_srchome = System.getProperty("GSDL3SRCHOME");
+	if (gsdl3_home == null) {
+	    System.out.println("Error: Unable to locate GSDL3HOME");
+	    System.exit(1);
+	}
+	if (gsdl3_srchome == null) {
+	    System.out.println("Error: Unable to locate GSDL3SRCHOME");
+	    System.exit(1);
+	}
+
+	int a = 0;
+	String collection = null;
+	String site = null;
+	
+	while (a < args.length) {
+	    if (args[a].equals("-collect")) {
+		if (a < args.length - 1 &&
+		    args[a+1].charAt(0) != '-') {
+		    
+			collection = args[a+1];
+			a ++;
+		}
+	    } else if (args[a].equals("-site")) {
+		if (a < args.length - 1 &&
+		    args[a+1].charAt(0) != '-') {
+		    
+		    site = args[a+1];
+		    a ++;
+		}
+	    }
+	    a ++;
+	}
+	
+	if (site == null) {
+	    
+	    System.err.println("you must specify the site name");
+	    System.exit(0);
+	}
+	if (collection == null) {
+	    System.err.println("you must specify a collection");
+	    System.exit(0);
+	}
+    
+	Create c = new Create(gsdl3_home, gsdl3_srchome, site, collection);
+	c.makeCollection();
+	return;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java	(revision 14150)
@@ -0,0 +1,8 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+public interface FileCrawlObserver
+{
+	public void processFile(File file);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawler.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawler.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawler.java	(revision 14150)
@@ -0,0 +1,39 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+public class FileCrawler
+{
+  File root;
+  FileCrawlObserver observer;
+
+  public FileCrawler(File file, FileCrawlObserver observer)
+  { this.root = file;
+    this.observer = observer;
+  }
+
+  public void crawl()
+  { this.crawlDirectory(this.root);
+  }
+
+  public boolean crawlDirectory(File file)
+  {
+    File [] children = file.listFiles();
+
+    if (children == null) {
+      return true;
+    }
+
+    for (int c = 0; c < children.length; c ++) {
+      if (children[c].isDirectory()) {
+	crawlDirectory(children[c]);
+      }
+      else
+      {
+	// TODO: process file
+	this.observer.processFile(children[c]);
+      }
+    }
+    return true;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/GSDataDump.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/GSDataDump.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/GSDataDump.java	(revision 14150)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.gs3build.doctypes.*;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnectionFactory;
+
+public class GSDataDump
+{ 
+  public static void main(String args[])
+  {
+    String database = args[0];
+
+    GS3SQLConnection connection = GS3SQLConnectionFactory.getGS3SQLConnection(database);
+    DocumentList docList = DocumentList.readSQLDocuments(connection);
+    docList.writeDocuments(new File(database));
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java	(revision 14150)
@@ -0,0 +1,356 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.util.GSXML;
+
+public class AZListClassifier extends AbstractClassifier
+{
+  GS3SQLConnection database;
+  MultiMap         alphaMap;
+  List             fields;
+  String           name;
+
+  class AZDocumentItem
+  { public DocumentID documentId;
+    public String     title;
+
+    public AZDocumentItem(DocumentID documentId, String title)
+    { this.documentId = documentId;
+      this.title = title;
+    }
+
+    public String getTitle()
+    { return this.title;
+    }
+
+    public DocumentID getID()
+    { return this.documentId;
+    }
+
+    public String toString()
+    { return this.title;
+    }
+  }
+
+  private void sortList(String key)
+  { List list = this.alphaMap.getAll(key);
+    if (list.size() > 1) {
+      int first, last;
+
+      first = 0;
+      last = list.size() - 1;
+      while (first != last)
+      { int at = (first + last) / 2;
+
+        if (list.get(at).toString().compareTo(list.get(list.size() - 1).toString()) > 0) {
+	  last = at;
+        }
+	else {
+	  first = at + 1;
+	}
+      }
+
+      Object newObject = list.get(list.size()-1);
+      last = list.size() - 1;
+      while (last > first) {
+	list.set(last, list.get(last-1));
+	last --;
+      }
+      list.set(first, newObject);
+    }
+  }
+
+  public AZListClassifier(String name, List fields)
+  { this.fields   = fields;
+    this.alphaMap = new MultiMap();
+    this.name     = name;
+    //  System.out.println("AZList :"+fields.size());
+  }
+
+  public AZListClassifier()
+  { this.fields = new ArrayList();
+  }
+
+  public void configure(List parameters)
+  {
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+    }
+  }
+
+  public void setDatabase(GS3SQLConnection connection)
+  { this.database = connection;
+  }
+
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+			
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      // ...and send them to the classifier
+      Iterator thisValue = values.iterator();
+      while (thisValue.hasNext()) 
+      { String value = thisValue.next().toString();
+
+	String classText = null;
+
+	int c = 0;
+	while (c < value.length() && classText == null)
+	{ if (Character.isDigit(value.charAt(c))) 
+	  { classText = "0-9";
+	  }
+	  else if (Character.isLetter(value.charAt(c))) 
+	  { classText = value.substring(c, c+1).toLowerCase();
+	  }
+  	  c ++;
+	}
+
+	if (classText != null) {
+	    //	  System.out.println(classText + " " + documentID);
+	  this.alphaMap.put(classText, new AZDocumentItem(documentID, value));
+	  this.sortList(classText);
+	}
+      }
+    }
+
+    return true;    
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void endClassifierPass(int pass)
+  {
+  }
+
+  public void completeClassification()
+  { this.writeSQL(this.database);
+  }
+
+  public int writeSQLClassifyNode(GS3SQLConnection connection, String parentId, int orderRef,
+				  String label, String name, String description, int noOfLeafDocs)
+  { GS3SQLAction action;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+  
+    int classifyRef;
+
+    // check for existing node
+    select = new GS3SQLSelect("classifiers");
+    select.addField("ClassifyRef");
+    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ClassifyID", "=", label);
+    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+    select.setWhere(where);
+
+
+    try {
+      Statement statement = connection.createStatement();
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	GS3SQLUpdate update = new GS3SQLUpdate("classifiers");
+	update.setWhere(where);
+	action = update;
+ 
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      else {
+	insert = new GS3SQLInsert("classifiers");
+
+	insert.addValue("ParentID", parentId);
+
+	action = insert;
+	classifyRef = -1;
+      }
+      action.addValue("ClassifyID", label);
+      action.addValue("Name", name);
+      action.addValue("Description", description);
+      action.addValue("ClassifyOrder", Integer.toString(orderRef), GS3SQLField.INTEGER_TYPE);
+      action.addValue("NumLeafDocs", Integer.toString(noOfLeafDocs), GS3SQLField.INTEGER_TYPE);
+
+      // do the update/insert
+      statement.execute(action.toString());
+
+
+      // get the ClassifyRef if we don't already have it (have done a
+      // insert action above)...
+      if (classifyRef == -1) {
+	results = statement.executeQuery(select.toString());
+	if (!results.first()) {
+	  return -1;
+	}
+	
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      
+      statement.close();
+    }  catch (SQLException sqlEx) {
+	System.err.println("AZListClassifier.writeSQLClassifyNode(): "+sqlEx);
+	return -1;
+    }
+  
+
+    return classifyRef;
+  }
+
+  public boolean writeSQL(GS3SQLConnection connection)
+  { GS3SQLAction action;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+ 
+    Iterator keys;
+ 
+    String prefix = this.name;
+    if (prefix==null || prefix.equals("")) {
+	prefix = "CLAZ"+this.fields.get(0).toString();
+	// TODO: write this name back to collectionConfig.xml
+	this.name = prefix;
+    }
+
+    int leafCount = 0;
+    keys = this.alphaMap.keySet().iterator();
+    while (keys.hasNext()) {
+      Object key = keys.next();
+
+      List childDocs = (List) this.alphaMap.getAll(key);
+      if (childDocs != null) {
+	leafCount += childDocs.size();
+      }
+    }
+
+    int parentClassify = this.writeSQLClassifyNode(connection, "", 0, prefix, "", "Classifier", leafCount);
+
+    if (parentClassify < 0) 
+    { return false;
+    }
+
+    try {
+	Statement statement = connection.createStatement();
+    
+    List children;
+    
+    // TODO: cope with change rather than create from scratch...
+    int nodeOrder = 1;
+    keys = this.alphaMap.keySet().iterator();
+    while (keys.hasNext()) {
+      Object key = keys.next();
+
+      List childDocs = (List) this.alphaMap.getAll(key);
+
+      if (childDocs != null) {
+	String className = prefix + "." + key.toString();
+
+	int classifyRef = this.writeSQLClassifyNode(connection, prefix, nodeOrder, className, key.toString(), key.toString(), childDocs.size());
+
+	// note the child documents...
+        Iterator iterator = childDocs.iterator();
+	int childOrder = 1;
+	//St
+	while (iterator.hasNext()) {
+	  AZDocumentItem documentItem = (AZDocumentItem) iterator.next();
+	  DocumentID docId = documentItem.getID();
+
+	  insert = new GS3SQLInsert("classdocuments");
+	  insert.addValue("ClassifyRef", Integer.toString(classifyRef), GS3SQLField.INTEGER_TYPE);
+	  insert.addValue("DocID", docId.toString());
+	  insert.addValue("DocOrder", Integer.toString(childOrder), GS3SQLField.INTEGER_TYPE);	  
+
+	  statement.execute(insert.toString());
+
+	  childOrder ++;
+	}
+
+	nodeOrder ++;
+      }
+    }
+
+   
+    /*
+    else {
+      // TODO: clear 'dead' child classifications
+
+      // delete child documents
+      GS3SQLDelete delete = new GS3SQLDelete("classdocuments");
+      delete.setWhere(where);
+
+      statement.execute(delete.toString());
+    }
+
+    // post the child nodes...
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      if (!childNode.writeSQL(connection)) {
+	return false;
+      }
+    }
+    */
+    statement.close();
+    } catch (SQLException e) {
+	System.err.println("AZListClassifier.writeSQL(): "+e);
+	return false;
+    }
+
+    return true;
+  }
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+
+}
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java	(revision 14150)
@@ -0,0 +1,61 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+
+public abstract class AbstractClassifier implements ClassifierInterface
+{ public AbstractClassifier()
+  {
+  }
+
+  public void configure(URL parentURL, org.w3c.dom.Node node)
+  { NodeList children = node.getChildNodes();
+    for (int c = 0; c < children.getLength(); c ++) {
+      Node child = children.item(c);
+      
+      if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	String name = child.getNodeName();
+	
+	if (name.equals("file")) {
+	  NamedNodeMap atts = children.item(c).getAttributes();
+	  Node attribute = atts.getNamedItem("URL");
+	  String urlString = attribute.getNodeValue();
+	  if (urlString == null)
+	    continue;
+	  
+	  try {
+	    URL url = new URL(parentURL, urlString);
+	    //	    this.file = url;
+	    System.out.println(url);
+	  }
+	  catch (java.net.MalformedURLException malEx) {
+	    System.out.println(malEx);
+	  }
+	}
+	else if (name.equals("field")) {
+	  String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	  System.out.println("Adding field " + fieldName);
+	  //	  this.fields.add(fieldName.toString());
+	}
+	else if (name.equals("sort")) {
+	  String sortName = DOMUtils.getNodeChildText(children.item(c));
+	  //	  this.sort = sortName;
+	}
+      }
+    }
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java	(revision 14150)
@@ -0,0 +1,428 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public abstract class AbstractHierarchyNode
+{
+  String prefix;        // a basic prefix used in this hierarchy
+  String descriptor;    // the textual descriptor used on screen or long-hand
+  String name;          // the index number, letter assignment or other item
+                        //   used to identify the position of the item in the
+                        //   hierarchy
+  String id;            // an identifier used by the GLI for maintenance
+                        //   purposes; this plays no active role in the
+                        //   rebuilding process (at the moment)
+  List childNodes;      // the child classification nodes of this node
+  List childDocs;       // the child documents of this node
+  List matches;         // the other metadata values that may be matched
+                        //   against the classifier
+  AbstractHierarchyNode parent; // the parent of the node
+
+  class AbstractHierarchyDocument
+  { DocumentID documentId;
+    String     sortKey;
+
+    public AbstractHierarchyDocument(DocumentID documentId, String sortKey)
+    { this.documentId = documentId;
+      this.sortKey    = sortKey;
+    }
+
+    public DocumentID getID()
+    { return this.documentId;
+    }
+
+    public String toString()
+    { return this.sortKey;
+    }
+  } 
+
+  /**
+   *  Simple node
+   */
+  public AbstractHierarchyNode()
+  { this.descriptor = "";
+    this.name       = "";
+    this.id         = "";
+    this.prefix     = "";
+    this.childNodes = new ArrayList();
+    this.childDocs  = new ArrayList();
+    this.parent     = null;
+    this.matches    = new ArrayList();
+  }
+    
+  public AbstractHierarchyNode(String prefix, String name, String id, String descriptor)
+  { this.descriptor = descriptor;
+    this.name       = name;
+    this.id         = id;
+    this.prefix     = prefix;
+    this.childNodes = new ArrayList();
+    this.childDocs  = new ArrayList();
+    this.parent     = null;
+    this.matches    = new ArrayList();
+    this.matches.add(this.id);
+    this.matches.add(this.name);
+  }
+    
+  public void addChild(AbstractHierarchyNode child)
+  { this.childNodes.add(child);
+    child.setParent(this);
+
+    /**
+    if (this.id == null) {
+      System.out.println(child.id.toString() + " added to root");
+    }
+    else {
+      System.out.println(child.id.toString() + " added to " + this.id);
+    }
+    */
+  }
+
+  public boolean add(AbstractHierarchyNode child)
+  { if (this.id != null && this.id.length() > 0 &&
+        !child.id.startsWith(this.id + "."))
+    { return false;
+    }
+
+    Iterator subNodes = this.childNodes.iterator();
+    while (subNodes.hasNext()) {
+      AbstractHierarchyNode subNode = (AbstractHierarchyNode) subNodes.next();
+      
+      if (subNode.add(child)) {
+	return true;
+      }
+    }
+    
+    this.addChild(child);
+    return true;
+  }
+
+  public void addDocument(DocumentID document, String sortKey)
+  { if (sortKey == null && false) {
+    System.err.println("attempt to classify by an unset key");
+      return;
+    }
+    this.childDocs.add(new AbstractHierarchyDocument(document, sortKey));
+
+    System.out.println("Adding " + document.toString() + " to " + this.name + " " + this.childDocs.size());
+    if (sortKey != null) {
+      int first = 0;
+      int last  = this.childDocs.size() - 1;
+
+      while (first != last) {
+	int at = (first + last) / 2;
+	
+	System.out.println(this.name + " " + at + " " + this.childDocs.size() + " " + this.childDocs.get(at) + " " + sortKey);
+	if (this.childDocs.get(at) == null ||
+	    this.childDocs.get(at).toString() == null) {
+	  last = at;
+	  System.out.println("COntinuing");
+	  continue;
+	}
+
+	if (this.childDocs.get(at).toString().compareTo(sortKey) > 0) {
+	  last = at;
+	}
+	else {
+	  first = at + 1;
+	}
+      }
+
+      Object newItem = this.childDocs.get(this.childDocs.size() - 1);
+      last = this.childDocs.size() - 1;
+      while (last > first) {
+	this.childDocs.set(last, this.childDocs.get(last-1));
+	last --;
+      }
+      this.childDocs.set(first, newItem);
+    }
+
+    //  System.out.println("Added " + document.toString() + " to " + this.name + " " + this.childDocs.size());
+    //    for (int a = 0; a < this.childDocs.size(); a ++) {
+    //System.out.println(this.childDocs.get(a));
+    //}
+  }
+
+  public void setParent(AbstractHierarchyNode parent)
+  { this.parent = parent;
+  }
+    
+  public AbstractHierarchyNode getParent()
+  { return this.parent;
+  }
+
+  public String getParentId()
+  { 
+    if (this.id == null) {
+      return "";
+    }
+
+    int dotAt = this.id.lastIndexOf('.');
+    if (dotAt < 0) {
+      return "";
+    }
+    return this.id.substring(0, dotAt);
+  }  
+
+  public void setDescriptor(String descriptor)
+  { this.descriptor = descriptor;
+  }
+
+  public void setID(String id)
+  { this.id = id;
+    this.matches.add(id);
+  }
+
+  public String getID()
+  { return this.id;
+  }
+
+  public int noOfChildDocs()
+  { return this.childDocs.size();
+  }
+
+  public int noOfLeafDocs()
+  { int total = this.childDocs.size();
+
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      total += childNode.noOfLeafDocs();
+    }
+
+    return total;
+  }
+
+  /**
+   *  Set the name for this hierarchy node - i.e. its brief description
+   *
+   *  @param <code>String</code> the new node name.
+   */
+  public void setName(String name)
+  { this.name = name;
+    this.matches.add(name);
+  }
+
+  /**
+   *  Get the name for this hierarchy node - i.e. its brief description
+   *
+   *  @return <code>String</code> the node's name.
+   */
+  public String getName()
+  { return this.name;
+  }
+
+  /**
+   *  Add another string which can be used to describe this hierarchy node - i.e. one which if
+   *  found in a document would indicate that that document belongs to this hierarchy node.
+   *
+   *  @param <code>String</code> the string to match.
+   */
+  public void addMatch(String match)
+  { this.matches.add(match);
+  }
+
+  /**
+   *  Check if a given string matches any of the values given for this hierarchy node...
+   *
+   *  @param <code>String</code> the descriptive string to be compared against the hierarchy
+   *  @return <code>boolean</code> <code>true</code> whether the string matches this hierarchy
+   */
+  public boolean isMatch (String toMatch)
+  { Iterator thisMatch = this.matches.iterator();
+			
+    while (thisMatch.hasNext())
+    { String thisMatchText = thisMatch.next().toString();
+      
+      if (thisMatchText.equals(toMatch))
+      { return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   *  Take a document, and find the classifications that it matches against in
+   *  the current hierarchy.  
+   *
+   *  @param <code>DocumentID</code> the id of the document being classified
+   *  @param <code>List</code> the values against which the classifier should
+   *         test for the document being a match - i.e. the pertinent document
+   *         property values.
+   *  @param <code>ClassifierObserverInterface</code>
+   *         object modifies the document with information about the
+   *         classifications that it fell within.
+   */
+  abstract public void getClassifications(DocumentID documentID, List values, String sortKey,
+					  ClassifierObserverInterface observer);
+
+  public boolean writeSQL(GS3SQLConnection connection)
+  {
+    int classifyRef;
+    GS3SQLAction action = null;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+
+    Statement statement;
+    // can we connect to the database?
+    try {
+	statement = connection.createStatement();
+    } catch (SQLException e) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	return false;
+    }
+    
+    // Get own full id
+    String fullId = this.id.length() > 0 ? this.prefix+"."+this.id : this.prefix;
+
+    // check for an existing instance of this classifier
+    select = new GS3SQLSelect("classifiers");
+    select.addField("ClassifyRef");
+    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ClassifyID", "=", fullId);
+    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+    select.setWhere(where);
+
+
+    // update or insert the classifier as required
+    try {
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	GS3SQLUpdate update = new GS3SQLUpdate("classifiers");
+	update.setWhere(where);
+	action = update;
+ 
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      else {
+	insert = new GS3SQLInsert("classifiers");
+
+	if (this.id == null || this.id.length() == 0) {
+	  insert.addValue("ParentID", "");
+	}
+	else {
+	  String parentId = this.getParentId();
+
+	  if (parentId.length() > 0) {
+	    insert.addValue("ParentID", this.prefix+"."+this.getParentId());
+	  }
+	  else {
+	    insert.addValue("ParentID", this.prefix);
+	  }
+	}
+
+	action = insert;
+	classifyRef = -1;
+      }
+      action.addValue("ClassifyID", fullId);
+      action.addValue("Name", this.name);
+      action.addValue("Description", this.descriptor);
+
+      String tailId = "0";
+      if (this.id.length() > 0) {
+	int lastDot = this.id.lastIndexOf('.');
+	if (lastDot >= 0) {
+	  tailId = this.id.substring(lastDot+1);
+	}
+	else {
+	  tailId = this.id;
+	}
+      }
+      action.addValue("ClassifyOrder", tailId, GS3SQLField.INTEGER_TYPE);
+      action.addValue("NumLeafDocs", Integer.toString(this.noOfLeafDocs()), GS3SQLField.INTEGER_TYPE);
+
+      statement.execute(action.toString());
+    }
+    catch (SQLException sqlEx) {
+      if (action == null) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx);
+      }
+      else {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx + " " + action.toString());
+      }
+      return false;
+    }
+
+    // get the ClassifyRef if we don't already have it (have done a
+    // insert action above)...
+    if (classifyRef == -1) {
+      try {
+        ResultSet results = statement.executeQuery(select.toString());
+	if (!results.first()) {
+	    statement.close();
+	    return false;
+	}
+	
+	classifyRef = results.getInt("ClassifyRef");
+	
+      }
+      catch (SQLException sqlEx) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx);
+	return false;
+      }
+    }
+    else {
+      // TODO: clear 'dead' child classifications
+
+      // delete child documents
+      GS3SQLDelete delete = new GS3SQLDelete("classdocuments");
+      delete.setWhere(where);
+      try {
+	  statement.execute(delete.toString());
+      } catch (SQLException e) {
+	  System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	  return false;
+      }
+    }
+    
+    // post the child nodes...
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      if (!childNode.writeSQL(connection)) {
+	  System.out.println("Failed to write " );
+	  return false;
+      }
+    }
+
+    // note the child documents...
+    iterator = this.childDocs.iterator();
+    int order = 1;
+    while (iterator.hasNext()) {
+      AbstractHierarchyDocument hierarchyDoc = (AbstractHierarchyDocument) iterator.next();
+      DocumentID docId = hierarchyDoc.getID();
+
+      insert = new GS3SQLInsert("classdocuments");
+      insert.addValue("ClassifyRef", Integer.toString(classifyRef), GS3SQLField.INTEGER_TYPE);
+      insert.addValue("DocID", docId.toString());
+      insert.addValue("DocOrder", Integer.toString(order), GS3SQLField.INTEGER_TYPE);
+      try {
+	  statement.execute(insert.toString());
+      } catch (SQLException e) {
+	  System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	  return false;
+      }
+      order ++;
+    }
+    // close the statment
+    try {
+	statement.close();
+    } catch (SQLException e) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+    }
+
+    return true;
+  }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java	(revision 14150)
@@ -0,0 +1,44 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public interface ClassifierInterface
+{
+  /**
+   *  Initialise the classifier as required.  The parameters given are specific
+   *  to the classifier type, so this interface does not guarantee any interpretations
+   *  of any given parameter.
+   *  
+   *  @param <code>List</code> the parameters with which to configure the classifier.
+   */
+  public void configure(List parameters);
+  public void configure(java.net.URL baseURL, org.w3c.dom.Node node);
+
+  public void setDatabase(org.greenstone.gsdl3.gs3build.database.GS3SQLConnection connection);
+
+  /**
+   *  Classify an individual document - the document should be informed of any
+   *  classification nodes which it is assigned to, so that it can obtain knowledge
+   *  of any "parents".  These are assigned into the "classified" metadata item in
+   *  the "gsdl3" namespace...
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document);
+  
+  public void startClassifierPass(int pass);
+  
+  public int getClassifierPasses();
+  
+  public void endClassifierPass(int pass);
+  
+  /**
+   *  Called at the end of the classification sequence - the classifier should
+   *  save its state at this point...
+   */
+  public void completeClassification();
+
+    public boolean addClassifierDescription(Element classifier_list);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java	(revision 14150)
@@ -0,0 +1,155 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+import java.util.Iterator;
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+
+public class ClassifierManager
+{
+  DocumentList           documents;
+  ClassifierInterface [] list;
+  int                    size;
+  int                    used;
+  GS3SQLConnection       database;
+
+  /**
+   *  Instantiate a classifier manager that will classify the
+   *  documents in a given <code>DocumentList</code> object.
+   *  The list of documents should not change during the
+   *  <code>classifyDocuments</code> function, and the documents
+   *  themselves may be altered at that time.
+   *
+   *  @param <code>DocumentList</code> a provider of documents for
+   *         classification.
+   */
+  public ClassifierManager(DocumentList documentList, GS3SQLConnection database)
+  { this.list = new ClassifierInterface[10];
+    this.size = 10;
+    this.used = 0;
+    this.documents = documentList;
+    this.database  = database;
+  }
+
+  /**
+   *  Add a classifier to the end of the list of classifiers...
+   *
+   *  @param <code>ClassifierInterface</code> the classifier.
+   */
+  public void addClassifier(ClassifierInterface classifier)
+  { this.ensureSize(this.used + 1);
+    this.list[this.used] = classifier;
+    this.used ++;
+
+    classifier.setDatabase(this.database);
+  }
+
+  /**
+   *  Classify an individual document - the document itself may be written
+   *  to in consequence, in which case the document <strong>must</strong>
+   *  be rewritten to disk.  This function does <strong>not</strong> do the
+   *  writing itself.
+   *
+   *  @param <code>DocumentID</code> the document's identifier.
+   *  @param <code>DocumentInterface</code> the document itself.
+   */
+  public void classifyDocument(DocumentID docId, DocumentInterface document)
+  { for (int i = 0; i < this.used; i ++)
+    {	this.list[i].classifyDocument(docId, document);
+    }
+  }
+
+  /**
+   *  Classify all the documents that the document list is able to provide...
+   */
+  public void classifyDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.list[i].getClassifierPasses(); p ++)
+      {	this.list[i].startClassifierPass(p);
+
+	Iterator iterator = this.documents.iterator();
+
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+
+	  if (document.isIndexed()) {
+	    if (!this.list[i].classifyDocument(document.getID(), document)) {
+	      System.out.println("Ending document");
+	    }
+	    
+	    if (document.isChanged()) {
+	      //	      System.out.println("Writing document " + document.getID());
+	      this.documents.storeChangedDocument(document);
+	    }
+	  }
+	}
+	this.list[i].endClassifierPass(p);
+      }
+      this.list[i].completeClassification();
+    }
+  }
+
+  public void ensureSize(int size)
+  { while (size >= this.size)
+    { ClassifierInterface newList [] = new ClassifierInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+
+  public static ClassifierInterface loadClassifier(String classifierName)
+  { System.out.println("Loading classifier " + classifierName);
+    try {
+      ClassifierInterface classifier = 
+	(ClassifierInterface) Class.forName("org.greenstone.gsdl3.gs3build.classifier."+classifierName+"Classifier").newInstance();
+      return classifier;
+    }
+    catch (ClassNotFoundException ex) {
+      System.err.println("Attempt to load classifier " + classifierName + " which does not exist");
+    }
+    catch (InstantiationException instEx) {
+      System.err.println("Unable to create classifier " + classifierName);
+    }
+    catch (IllegalAccessException accessEx) {
+      System.err.println("Unable to instantiate classifier " + classifierName + " due to permissions problems " + accessEx.toString());
+    }
+    return null;
+  }
+    
+  public static ClassifierInterface loadClassifier(String classifierName, List params)
+  { ClassifierInterface classifier = loadClassifier(classifierName);
+    if (classifier != null) {
+      classifier.configure(params);
+    }
+    return classifier;
+  }
+
+  public boolean addServiceDescriptions(Element service_class_list) {
+    if (this.used==0) {
+      // no classiifers
+      return true;
+    }
+    Document owner_doc = service_class_list.getOwnerDocument();
+    Element browse_service = owner_doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    browse_service.setAttribute(GSXML.NAME_ATT, "GS3Browse");
+    service_class_list.appendChild(browse_service);
+    
+    Element classifier_list = owner_doc.createElement(GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+    browse_service.appendChild(classifier_list);
+    boolean success = true;
+    for (int i = 0; i < this.used; i ++) {
+	if (!this.list[i].addClassifierDescription(classifier_list)) {
+	    success = false;
+	}
+    }
+    return success;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java	(revision 14150)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+public interface ClassifierObserverInterface
+{
+  public void recordClassification(String label);
+  public java.util.List getClassifications();
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java	(revision 14150)
@@ -0,0 +1,309 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+public class GS2HierarchyClassifier implements ClassifierInterface
+{ 
+  class GS2HierarchyClassifierObserver implements ClassifierObserverInterface
+  { List              classifications;
+    DocumentInterface document;
+
+    public GS2HierarchyClassifierObserver(DocumentInterface document)
+    { this.document = document;
+    }
+
+    public void recordClassification(String label)
+    { this.document.addDocumentMetadata("gsdl3", "classified", label);
+      this.document.setChanged(true);
+      //      System.out.println("Assigned document " + this.document.getID().toString() + " to " + label);
+    }
+
+    public List getClassifications()
+    { return this.classifications;
+    }
+  }
+
+  class GS2HierarchyNode extends AbstractHierarchyNode
+  { /**
+     *  Simple node
+     */
+    public GS2HierarchyNode()
+    { super();
+    }
+    
+    public GS2HierarchyNode(String prefix, String name, String id, String descriptor)
+    { super(prefix, name, id, descriptor);
+    }
+    
+    /**
+     *  Take a document, and find the classifications that it matches against in
+     *  the current hierarchy.  
+     *
+     *  @param <code>DocumentID</code> the id of the document being classified
+     *  @param <code>List</code> the values against which the classifier should
+     *         test for the document being a match - i.e. the pertinent document
+     *         property values.
+     *  @param <code>ClassifierObserverInterface</code>
+     *         object modifies the document with information about the
+     *         classifications that it fell within.
+     */
+    public void getClassifications(DocumentID documentID, List values, String sortKey,
+				   ClassifierObserverInterface observer)
+    { // Go through the list of values, and if this node matches, record
+      // the fact in the classifier observer...
+      Iterator valueList = values.iterator();
+
+      while (valueList.hasNext())
+      {	if (this.isMatch(valueList.next().toString()))
+	{ observer.recordClassification(this.id);
+	  this.addDocument(documentID, sortKey);
+	}
+      }
+     
+      // recurse into the child nodes for them to do the same...
+      Iterator childList = this.childNodes.iterator();
+
+      while (childList.hasNext())
+      { ((GS2HierarchyNode) childList.next()).getClassifications(documentID, values, sortKey, observer);
+      }
+    }
+  }
+
+  /**
+   *  An inner class to handle Classifier files
+   */
+  class GS2HierarchyHandler extends GS2TextFileHandler
+  { GS2HierarchyNode rootNode = null;
+
+    GS2HierarchyHandler(String hierarchyName, String content)
+    { super(content);
+ 
+      String prefix = "CL"+hierarchyName;
+      this.rootNode = new GS2HierarchyNode(prefix, hierarchyName, "", hierarchyName);
+
+      while (this.hasMoreLines()) {
+	String description;
+	String name, id;
+
+	this.getLine();
+	
+	name = this.getEntry(true);
+	if (name == null || name.length() == 0) {
+	  continue;
+	}
+
+	id = this.getEntry(true);
+	if (id == null || id.length() == 0) {
+	  continue;
+	}
+
+	description = this.getEntry(true);
+	if (description == null || description.length() == 0) {
+	  continue;
+	}
+	
+	// Initialise the tree
+	GS2HierarchyNode node = new GS2HierarchyNode(prefix, name, id, description);
+
+	this.rootNode.add(node);
+      }
+    }
+
+    public GS2HierarchyNode getHierarchy()
+    { return this.rootNode;
+    }
+  }
+
+  private          GS2HierarchyNode hierarchy;
+  List             fields;
+  String           sortBy;
+  GS3SQLConnection database;
+
+  public GS2HierarchyClassifier()
+  { // a 'do nothing' constructor which takes more information from configure...
+    this.fields = new ArrayList();
+    this.database = null;
+  }
+
+  public GS2HierarchyClassifier(File file)
+  {	
+    String documentText = 
+      DocumentLoader.getAsString(file);
+	
+    if (documentText == null) {
+      System.err.println("GS2HierarchyClassifier: Unable to load any content for " + file);
+      return;
+    }
+
+    GS2HierarchyHandler handler = new GS2HierarchyHandler("CL", documentText);
+
+    this.hierarchy = handler.getHierarchy();
+    this.fields = new ArrayList();
+    //    System.out.println("loaded hierarchy for "+ file.toString());
+    this.database = null;
+  }
+
+  public void setDatabase(GS3SQLConnection database)
+  { this.database = database;
+  }
+
+  public void configure(URL url, org.w3c.dom.Node node)
+  {
+  }
+
+  public void configure(List parameters)
+  {
+    File file = null;
+    
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+      else if (param.equals("-hfile")) {
+	// this parameter had better have been expanded previously to a full path,
+	// as we're not going to do that here!!!
+	if (iterator.hasNext()) {
+	  file = new File(iterator.next().toString());
+	}
+      }
+      else if (param.equals("-sort")) {
+	if (iterator.hasNext()) {
+	  String sort = iterator.next().toString();
+	  if (sort != null && sort.length() > 0) {
+	    this.sortBy = sort;
+	  }
+	}
+      }
+    }
+
+    if (file == null) {
+      return;
+    }
+
+    String documentText = 
+      DocumentLoader.getAsString(file);
+	
+    if (documentText == null) {
+      System.err.println("GS2HierarchyClassifier: Unable to load any content for " + file);
+      return;
+    }
+
+    GS2HierarchyHandler handler = new GS2HierarchyHandler(this.fields.get(0).toString(), documentText);
+
+    this.hierarchy = handler.getHierarchy();    
+  }
+
+  /**
+   *  Classify a particular document - the document id will be used to record its
+   *  membership of any classifier nodes, and the document interface will also be
+   *  used to assign back knowledge of any classifiers to which the document is
+   *  assigned.
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>DocumentInterface</code> the document interface
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { if (this.hierarchy == null) {
+      return true;
+    }
+    
+    // the observer records the assignment of documents to classifications
+    GS2HierarchyClassifierObserver classifyObserver = new GS2HierarchyClassifierObserver(document);
+		
+    // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      //      System.out.println("Matching " + values.get(0).toString());
+
+      if (documentID == null) {
+	System.out.println("Bad documentID");
+	continue;
+      }
+
+      // get the sort key for the metadata item if possible
+      String sortKey = null;
+
+      if (this.sortBy != null) {
+	List sortKeys = document.getDocumentMetadataItem(this.sortBy);
+	if (sortKeys != null && sortKeys.size() > 0) {
+	  sortKey = sortKeys.get(0).toString();
+	}
+      }
+
+      // ...and send them to the classifier
+      this.hierarchy.getClassifications(documentID, values, sortKey, classifyObserver);
+    }
+    return true;
+  }
+
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+  
+  public void endClassifierPass(int pass)
+  {
+  }
+  
+  public void completeClassification()
+  { // TODO: store the classifications to file...
+
+    // The hierarchy member may be null if the classifier did not initialise correctly...
+    if (this.hierarchy != null) {
+      this.hierarchy.writeSQL(this.database);
+    }
+  }
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	//classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier.setAttribute(GSXML.NAME_ATT, "name"); // what is teh name???
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java	(revision 14150)
@@ -0,0 +1,344 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.xerces.parsers.SAXParser;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+
+public class HierarchyClassifier extends AbstractClassifier
+{ 
+  class HierarchyClassifierObserver implements ClassifierObserverInterface
+  { List              classifications;
+    DocumentInterface document;
+
+    public HierarchyClassifierObserver(DocumentInterface document)
+    { this.document = document;
+    }
+
+    public void recordClassification(String label)
+    { this.document.addDocumentMetadata("gsdl3", "classified", label);
+    }
+
+    public List getClassifications()
+    { return this.classifications;
+    }
+  }
+
+  class HierarchyNode extends AbstractHierarchyNode
+  { /**
+     *  Simple node
+     */
+    public HierarchyNode()
+    { super();
+    }
+
+    public HierarchyNode(String prefix, String name, String id, String descriptor)
+    { super(prefix, name, id, descriptor);
+    }
+    
+    public void getClassifications(DocumentID documentID, List values, String sortKey,
+				   ClassifierObserverInterface observer)
+    { Iterator valueList = values.iterator();
+
+      while (valueList.hasNext())
+      {	if (this.isMatch(valueList.next().toString()))
+	{ observer.recordClassification(this.name);
+	  this.addDocument(documentID, sortKey);
+	}
+      }
+     
+      // recurse into the child nodes for them to do the same...
+      Iterator childList = this.childNodes.iterator();
+
+      while (childList.hasNext())
+      { ((HierarchyNode) childList.next()).getClassifications(documentID, values, sortKey, observer);
+      }
+    }
+  }
+
+  /**
+   *  An inner class to handle Classifier files
+   */
+  class HierarchyHandler extends DefaultHandler
+  { StringBuffer description;
+    StringBuffer name;
+    StringBuffer path;
+    StringBuffer match;
+    String       prefix;
+    boolean inElement;
+    AbstractHierarchyNode rootNode = null;
+    AbstractHierarchyNode currentNode = null;
+
+    HierarchyHandler(String classifierName)
+    { super();
+ 
+      this.name        = null;
+      this.description = null;
+      this.path        = null;
+      this.match       = null;
+      this.prefix      = classifierName;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("Hierarchy"))
+      {	// create a new node in the hierarchy
+	HierarchyNode node = new HierarchyNode(prefix, "", "", "Classifier");
+
+	this.rootNode = node;
+	this.currentNode = node;
+      }
+      else if (localName.equals("Classification"))
+      {	// create a new node in the hierarchy
+	HierarchyNode node = new HierarchyNode(this.prefix, "", "", "");
+
+	// Initialise the tree
+	if (this.currentNode == null)
+	{ this.rootNode = node;
+	}
+        // Add the node
+	else
+	{ this.currentNode.addChild(node);
+	}
+	this.currentNode = node;
+      }
+      else if (localName.equals("Name"))
+      {	this.name = new StringBuffer();
+      }
+      else if (localName.equals("Path"))
+      {	this.path   = new StringBuffer();
+      }
+      else if (localName.equals("Description"))
+      {	this.description = new StringBuffer();
+      }
+      else if (localName.equals("Matches"))
+      {	this.match = new StringBuffer();
+      }
+      else if (localName.equals("Children"))
+      {	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("Classification"))
+      {	if (this.currentNode.getParent() == null) {
+	  // TODO: Error state
+          // this.rootNodes.add(this.currentNode);
+        }
+	this.currentNode = this.currentNode.getParent();
+      }
+      else if (localName.equals("Name"))
+      { this.currentNode.setName(XMLTools.cleanString(this.name.toString()));
+      System.out.println("Name is " + this.name);
+        this.name = null;
+      }
+      else if (localName.equals("Path"))
+      {	this.currentNode.setID(XMLTools.cleanString(this.path.toString()));
+        this.path = null;
+      }
+      else if (localName.equals("Description"))
+      { this.currentNode.setDescriptor(XMLTools.cleanString(this.description.toString()));
+        this.description = null;
+      }
+      else if (localName.equals("Matches"))
+      { this.currentNode.addMatch(XMLTools.cleanString(this.match.toString()));
+        this.match = null;
+      }
+      else if (localName.equals("Children"))
+      {	
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { String string = new String(c, start, length);
+      if (this.path != null)
+      { this.path.append(string);
+      }
+      else if (this.name != null)
+      { this.name.append(string);
+      }
+      else if (this.description != null)
+      { this.description.append(string);
+      }
+      else if (this.match != null)
+      { this.match.append(string);
+      }
+    }
+
+    public AbstractHierarchyNode getHierarchy()
+    { return this.rootNode;
+    }
+  }
+  
+  private GS3SQLConnection database;
+  private AbstractHierarchyNode    hierarchy;
+  private List             fields;
+  private String           sortBy;
+  private String           name;
+
+  public HierarchyClassifier(String name, URL basefile, List fields, String sortBy)
+  {	
+    try 
+    { SAXParser parser = new SAXParser();
+      HierarchyHandler handler = new HierarchyHandler(name);
+      /*
+      XMLReader reader = XMLReaderFactory.createXMLReader();
+      reader.setContentHandler(handler);
+      reader.setErrorHandler(handler);*/
+      parser.setContentHandler(handler);
+
+      // A Classifier consists of one file only - get it.
+      /*
+      FileReader fileReader = new FileReader(basefile);
+      reader.parse(new InputSource(fileReader));
+      */
+      parser.parse(basefile.toString());
+
+      this.hierarchy = handler.getHierarchy();
+      this.fields = fields;
+      this.sortBy = sortBy;
+      this.name = name;
+      System.out.println("Counting " + this.fields.size() + " fields");
+    }
+    catch (SAXException saxException)
+    { // TODO: log error
+	System.out.println("unable to process hierarchy " + saxException.toString());
+    }
+    catch (java.io.FileNotFoundException fileException)
+    {
+	System.out.println("unable to process hierarchy " + fileException.toString());
+    }
+    catch (java.io.IOException ioException)
+    {
+	System.out.println("unable to process hierarchy " + ioException.toString());
+    }
+  }
+
+  public void configure(List parameters)
+  {
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+    }
+  }
+
+  /**
+   *  Indicate the database to be used for saving the hierarchy to.
+   *
+   *  @param <code>GS3SQLConnection</code> the database
+   */
+  public void setDatabase(GS3SQLConnection database)
+  { this.database = database;
+  }
+
+  /**
+   *  Classify a particular document - the document id will be used to record its
+   *  membership of any classifier nodes, and the document interface will also be
+   *  used to assign back knowledge of any classifiers to which the document is
+   *  assigned.
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>DocumentInterface</code> the document interface
+   *
+   *  @return <code>boolean</code> a return value - currently always 
+   *          <code>true</code> for success
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { // the observer records the assignment of documents to classifications
+    HierarchyClassifierObserver classifyObserver = new HierarchyClassifierObserver(document);
+
+    // if we have no fields, there is nothing to do...
+    if (this.fields == null)
+    { return true;
+    }
+
+    // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+			
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      if (documentID == null) {
+	System.out.println("Bad documentID");
+	continue;
+      }
+
+      // get the sort key for the metadata item if possible
+      String sortKey = null;
+
+      if (this.sortBy != null) {
+	List sortKeys = document.getDocumentMetadataItem(this.sortBy);
+	if (sortKeys != null && sortKeys.size() > 0) {
+	  sortKey = sortKeys.get(0).toString();
+	}
+      }
+
+      // ...and send them to the classifier
+      this.hierarchy.getClassifications(documentID, values, sortKey, classifyObserver);
+    }
+    return true;
+  }
+
+  /**
+   *  @see ClassifierInterface:getClassifierPasses
+   */
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+  
+  public void endClassifierPass(int pass)
+  {
+  }
+  
+  public void completeClassification()
+  { // TODO: store the classifications to file...
+    if (this.hierarchy != null) {
+      this.hierarchy.writeSQL(this.database);
+    }
+  }
+
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java	(revision 14150)
@@ -0,0 +1,31 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.util.GregorianCalendar;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public class BuildIDFactory implements IDFactoryInterface
+{
+	int sequenceNumber;
+
+	public BuildIDFactory(BuildManager manager)
+	{
+		GregorianCalendar today = new GregorianCalendar();
+
+		String oldBuild = manager.getLastBuildSequence();
+		if (oldBuild != null)
+		{ this.sequenceNumber = Integer.parseInt(oldBuild);
+		}
+		else
+		{	this.sequenceNumber = 0;
+		}
+	}
+
+	public DocumentID getDocumentID(DocumentInterface document)
+	{	String value = new String(Integer.toString(this.sequenceNumber));
+		this.sequenceNumber ++;
+		return new DocumentID(value);
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java	(revision 14150)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+import org.greenstone.gsdl3.gs3build.FileCrawlObserver;
+
+public class CollectionCrawlObserver implements FileCrawlObserver
+{
+  BuildManager manager;
+
+  public CollectionCrawlObserver(BuildManager manager)
+  { this.manager = manager;
+  }
+
+  public void processFile(File file)
+  { if (file.getName().equals("collect.cfg")) {
+      GS2CollectionCfg config = GS2CollectionCfg.getGS2CollectionCfg(this.manager, file);
+    }
+    else if (file.toString().endsWith(".txt")) {
+      // Commented out: this is now done on the line above...
+      // GS2HierarchyClassifier classifier = new GS2HierarchyClassifier(file);
+      // manager.getClassifierManager().addClassifier(classifier);
+    }
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java	(revision 14150)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+/**
+ *  A class to consolidate the metadata assigned to the documents of a collection
+ *  as per the extension to Greenstone 2 done by George Buchanan.
+ */
+
+public class CollectionMetadata
+{ METSDescriptive metadata;
+	
+  public CollectionMetadata()
+  { this.metadata = new METSDescriptive("CM0", "collection");
+  }
+
+  /**
+   *  Accrue the metadata assigned to a particular document
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> the field of metadata
+   *  @param <code>String</code> the actual value assigned
+   */
+  public void addDocumentMetadata(DocumentID documentID, String namespace, String field, String value)
+  { // we don't presently use the document ID in this member function
+    this.metadata.addMetadata(namespace, field, value);
+  }
+
+  /**
+   *  Set the metadata stored on a given metadata item - any other values are
+   *  destroyed.
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   *  @param <code>String</code> the actual value assigned
+   */
+  public void setCollectionMetadata(String namespace, String label, String value)
+  { this.metadata.setMetadata(namespace, label, value);
+  }
+
+  /**
+   *  Get the metadata stored on a given metadata item
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   */
+  public List getCollectionMetadata(String namespace, String label)
+  { return this.metadata.getMetadata(namespace, label);
+  }
+
+  /**
+   *  Write the collection metadata
+   *
+   *  TODO: dummy function - complete
+   */
+  public boolean write(PrintWriter writer)
+  { return false;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java	(revision 14150)
@@ -0,0 +1,159 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.File;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.greenstone.gsdl3.gs3build.classifier.ClassifierInterface;
+import org.greenstone.gsdl3.gs3build.classifier.ClassifierManager;
+
+import org.greenstone.gsdl3.gs3build.indexers.*;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public class GS2CollectionCfg
+{
+  BuildManager buildManager;
+  IndexerInterface indexer;
+
+  class GS2CollectCfgHandler extends GS2TextFileHandler
+  { 
+    
+    GS2CollectCfgHandler(String content, File collectFile, GS2CollectionCfg configurator)
+    { super(content);
+ 
+      while (this.hasMoreLines()) {
+	String lineType;
+
+	this.getLine();
+	
+	lineType = this.getEntry(true);
+	if (lineType == null || lineType.length() == 0) {
+	  continue;
+	}
+
+	if (lineType.equals("classify"))
+	{
+	  String classType = this.getEntry(true);
+	  if (classType == null || classType.length() == 0) {
+	    continue;
+	  }
+	  classType = "GS2" + classType;
+	  
+	  List paramList = new ArrayList();
+	  
+	  // read in the parameters - to allow for better abstraction of other parts,
+	  // the handling of this is a bit 'odd', specifically modifying any file
+	  // references here before passing them on...
+	  String parameter = null;
+	  String lastParam = null;
+	  while (this.hasMore())
+	  { String param = this.getEntry(true);
+	    
+	    if (param != null && param.length() > 0) 
+	    { // expand any hfile references...
+	      if (lastParam != null && lastParam.equals("-hfile")) {
+		param = collectFile.getParent() + File.separator + param;
+	      }
+	      
+	      paramList.add(param);
+	    }
+
+	    lastParam = param;
+	  }
+
+	  ClassifierInterface classifier = ClassifierManager.loadClassifier(classType, paramList);
+	  if (classifier != null) {
+	    configurator.addClassifier(classifier);
+	  }
+	  else {
+	    System.err.println("Classifier not loaded");
+	  }
+	}
+	else if (lineType.equals("buildtype")) {
+	  if (this.hasMore()) {
+	    String type = this.getEntry(true);
+	    if (type != null) {
+	      configurator.setBuildType(type);	      
+	    }
+	  }
+	}
+	else if (lineType.equals("indexes")) {
+	  while (this.hasMore()) {
+	    String index = this.getEntry(true);
+	    if (index != null) {
+	      System.out.println("Adding index " + index);
+	      configurator.addIndex(index);
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  public GS2CollectionCfg(BuildManager manager, File file)
+  {
+    String documentText = 
+      DocumentLoader.getAsString(file);
+
+    if (documentText == null) {
+      return;
+    }
+
+    this.buildManager = manager;
+
+    GS2CollectCfgHandler handler = new GS2CollectCfgHandler(documentText, file, this);
+  }
+  
+  void addClassifier(ClassifierInterface classifier)
+  {
+    ClassifierManager manager = this.buildManager.getClassifierManager();
+
+    if (manager != null) {
+      manager.addClassifier(classifier);
+    }
+    else {
+      System.out.println("Unable to install due to missing manager");
+    }
+  }
+
+  void addIndex(String index)
+  {
+    if (this.indexer == null) {
+      this.setBuildType("mg");
+    }
+    this.indexer.configure(IndexerInterface.GS2_INDEX_LABEL, index);
+  }
+
+  void setBuildType(String type)
+  { if (type.equals("mgpp")) {
+      this.indexer = new MGPPIndexer("mgpp");
+    }
+    else {
+      this.indexer = new MGIndexer("mg");
+    }
+    this.buildManager.addIndexer(this.indexer);
+  }
+
+  public static GS2CollectionCfg getGS2CollectionCfg(BuildManager manager, File cfgFile)
+  { GS2CollectionCfg configurator = new GS2CollectionCfg(manager, cfgFile);
+    return configurator;
+  }
+
+  public static GS2CollectionCfg getGS2CollectionCfg(BuildManager manager, String etcDir)
+  {
+    File file = new File(etcDir, "collect.cfg");
+    
+    if (!file.exists()) {
+      return null;
+    }
+
+    return getGS2CollectionCfg(manager, file);
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java	(revision 14150)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public interface IDFactoryInterface
+{
+	public DocumentID getDocumentID(DocumentInterface document);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java	(revision 14150)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+/**
+ *  Facilitates communication between Greenstone and a database system,
+ *  be it gdbm, an SQL database or other product - what lies underneath
+ *  is down to you...
+ *
+ *  As this is a "write only" interface, the document is only stored as
+ *  required.
+ */
+
+import java.sql.*;
+
+public interface DatabaseAdaptorInterface
+{
+    //  public Connection getConnection();
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java	(revision 14150)
@@ -0,0 +1,54 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+
+public class GS3SQLAction
+{ String      table;
+  List        values;
+
+  public GS3SQLAction(String table)
+  {
+    this.table  = table;
+    this.values = new ArrayList();
+  }
+
+  public void addValue(String field, String value, String type)
+  { this.values.add(new GS3SQLValue(field, value, type));
+  }
+  
+  public void addValue(String field, String value)
+  { this.values.add(new GS3SQLValue(field, value));
+  }
+
+  public void addDate(String field, java.sql.Timestamp date)
+  { this.values.add(new GS3SQLValue(field, getDateString(date)));
+  }
+
+  public static String getDateString(java.sql.Timestamp date)
+  { StringBuffer reply = new StringBuffer();
+    Calendar calendar = new GregorianCalendar();
+    calendar.setTime(date);
+
+    reply.append(Integer.toString(calendar.get(Calendar.YEAR)));
+    reply.append("-");
+    reply.append(Integer.toString(calendar.get(Calendar.MONTH)+1));
+    reply.append("-");
+    reply.append(Integer.toString(calendar.get(Calendar.DAY_OF_MONTH)));
+    reply.append(" ");
+    reply.append(Integer.toString(calendar.get(Calendar.HOUR_OF_DAY)));
+    reply.append(":");
+    reply.append(Integer.toString(calendar.get(Calendar.MINUTE)));
+    reply.append(":");
+    reply.append(Integer.toString(calendar.get(Calendar.SECOND)));
+
+    return reply.toString();
+  }
+  
+  public void addField(String field)
+  { this.values.add(new GS3SQLValue(field, null));
+  }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java	(revision 14150)
@@ -0,0 +1,245 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+
+public class GS3SQLConnection extends SQLConnection
+{
+
+    public GS3SQLConnection(java.sql.Connection connection, String database)
+    { 
+	super(connection, database);
+    }
+    
+
+  public void clearCollection(String collectionName)
+  {
+      dropDatabase(collectionName);
+  }
+    
+    public GS3SQLConnection cloneConnection() {
+	GS3SQLConnection conn = GS3SQLConnectionFactory.getGS3SQLConnection(this.database);
+	return conn;
+    }
+  /**
+   *  Initialise a collection for use.
+   *
+   *  @return <code>boolean</code> <code>true</code> if the collection successfully initialised.
+   */
+  public boolean initCollection(String collectionName)
+  {	
+    /*
+    // Check if the collection already exists
+    // if so, abort
+
+    // do a sample query to check if the collection exists
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+    if (this.execute(select.toString())) {
+      return true;
+    }
+
+    // if there is an error, assume that the database doesn't exist...
+    // and reconnect to a "test" database for now...
+    this.connection = GS3SQLConnectionFactory.getConnection("test");
+    */
+
+      // create the new db
+      if (!createDatabase(collectionName)) {
+          System.err.println("Couldn't create database " + collectionName);
+	  return false;
+      }
+      // now connect to it
+      if (!connectToDatabase(collectionName)) {
+          System.err.println("couldn't connect to the database " + collectionName);
+	  return false;
+      }
+    
+      
+    try {
+      Statement statement = this.connection.createStatement();
+
+      // create build history table
+      GS3SQLCreateTable buildTable = new GS3SQLCreateTable("build");
+      buildTable.addAutoPrimaryKey("buildRef");
+      buildTable.addProperty("buildKey", 64);
+      buildTable.addProperty("buildValue");
+      statement.execute(buildTable.toString());
+
+      // create document table
+      GS3SQLCreateTable docTable = new GS3SQLCreateTable("document");
+      docTable.addProperty("DocID", 64);   // The document identifer, unique in collection
+      docTable.setPrimaryKey("DocID");     // ...which is also the primary key
+      docTable.addProperty("DocType", 64); // The document type - used to regenerate the
+                                           // correct Document object type.
+      docTable.addProperty("AccessionDate", GS3SQLField.DATETIME_TYPE);
+      docTable.addProperty("IndexedDate", GS3SQLField.DATETIME_TYPE);
+      docTable.addProperty("ModifiedDate", GS3SQLField.DATETIME_TYPE);
+      statement.execute(docTable.toString());
+      
+      // create structure table
+      GS3SQLCreateTable structureMap = new GS3SQLCreateTable("structure");
+      structureMap.addAutoPrimaryKey("StructureRef");
+      structureMap.addProperty("DocID", 64);          // a 'foreign key' in effect - but not treated
+                                                      // as such for efficiency reasons
+      structureMap.addProperty("StructureID", 64);    // this identifier
+      structureMap.addProperty("StructureType", 64);  // the type of the structure
+      structureMap.addProperty("Label", 128);         // the label of the structure
+      statement.execute(structureMap.toString());
+    
+      // create section table
+      GS3SQLCreateTable sectionMap = new GS3SQLCreateTable("divisions");
+      sectionMap.addAutoPrimaryKey("DivisionRef");
+      sectionMap.addProperty("StructureRef", GS3SQLField.INTEGER_TYPE);
+      sectionMap.addProperty("DocID", 64);        // a 'foreign key' in effect - but not treated
+                                                  // as such for efficiency reasons
+      sectionMap.addProperty("ParentType", 64);   // the type of the parent - document or section
+      sectionMap.addProperty("ParentRef", 64);    // the parent sql reference identifier
+      sectionMap.addProperty("SectionID", 64);    // this identifier
+      sectionMap.addProperty("DivisionType", 64); // the type of the section
+      sectionMap.addProperty("LabelOrder", 64);   // the order of the section
+      sectionMap.addProperty("ShortLabel", 128);  // the short label of the section
+      sectionMap.addProperty("UserLabel", 255);   // the long label of the section
+      statement.execute(sectionMap.toString());
+
+      // a division->metadata reference mapping
+      GS3SQLCreateTable metaRefs = new GS3SQLCreateTable("divisionmetarefs");
+      metaRefs.addProperty("DocID", 64);
+      metaRefs.addProperty("DivisionRef", GS3SQLField.INTEGER_TYPE);
+      metaRefs.addProperty("DivisionType", 16);   // whether a namespace or meta ref
+      metaRefs.addProperty("MetaID", 64); // the metadata reference itself, as a string
+      statement.execute(metaRefs.toString());
+
+      // a division->file reference mapping
+      GS3SQLCreateTable fileRefs = new GS3SQLCreateTable("divisionfilerefs");
+      fileRefs.addProperty("DocID", 64);
+      fileRefs.addProperty("DivisionRef", GS3SQLField.INTEGER_TYPE);
+      fileRefs.addProperty("DivisionType", 16); // whether section, group or file
+      fileRefs.addProperty("FileID", 32);
+      statement.execute(fileRefs.toString());
+      
+      //   create metadata table section table
+      GS3SQLCreateTable metadataSection = new GS3SQLCreateTable("metadata");
+      metadataSection.addAutoPrimaryKey("MetadataRef");
+      metadataSection.addProperty("DocID", 64);   // this table isn't actually used for much
+      metadataSection.addProperty("MetaID", 64);  // in and of itself...
+      metadataSection.addProperty("GroupID", 64); // the 'name' property
+      statement.execute(metadataSection.toString());
+      
+      //   create namespace table
+      GS3SQLCreateTable namespaces = new GS3SQLCreateTable("namespaces");
+      namespaces.addAutoPrimaryKey("NamespaceRef");
+      namespaces.addProperty("MetadataRef", GS3SQLField.INTEGER_TYPE);
+      namespaces.addProperty("DocID", 64);   // these three fields are the primary key
+      namespaces.addProperty("MetaID", 64);
+      namespaces.addProperty("NamespaceID", 64);
+      namespaces.addProperty("NamespaceType", 64);
+      namespaces.addProperty("FileType", 64);
+      namespaces.addProperty("FileLoc");
+      namespaces.addProperty("Creator", 128);
+      statement.execute(namespaces.toString());
+
+      // create metadata grouping table
+      GS3SQLCreateTable metagroups = new GS3SQLCreateTable("mdgroups");
+      metagroups.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE);
+      metagroups.addAutoPrimaryKey("MdGroupRef");
+      metagroups.addProperty("MdGroupID", 32);
+      statement.execute(metagroups.toString());
+
+      // create metadata qualifiers table
+      GS3SQLCreateTable metaqualifiers = new GS3SQLCreateTable("mdqualifiers");
+      metaqualifiers.addProperty("MdGroupRef", GS3SQLField.INTEGER_TYPE);
+      metaqualifiers.addProperty("QualifierName", 64);
+      metaqualifiers.addProperty("QualifierValue", 64);
+      statement.execute(metaqualifiers.toString());
+      
+      // create metadata values table
+      GS3SQLCreateTable metadata = new GS3SQLCreateTable("mdvalues");
+      metadata.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE); // the parent namespace {one candidate primary key}
+      metadata.addProperty("GroupRef", GS3SQLField.INTEGER_TYPE);     // the parent group     {a sub-key or independent primary key}
+      metadata.addProperty("Label", 256);                             // the key of the metadata item
+      metadata.addProperty("Value");                                  // the value of the metadata item
+      statement.execute(metadata.toString());
+    
+      //   create file section table
+      GS3SQLCreateTable filesec = new GS3SQLCreateTable("filesection");
+      filesec.addAutoPrimaryKey("FileSectionRef");
+      filesec.addProperty("DocID", 64);
+      filesec.addProperty("FileSecID", 64);
+      statement.execute(filesec.toString());
+    
+      //   create file groups table
+      GS3SQLCreateTable filegroups = new GS3SQLCreateTable("filegroups");
+      // TODO: some unique identifier
+      filegroups.addAutoPrimaryKey("FileGroupRef");
+      filegroups.addProperty("DocID", 64);
+      filegroups.addProperty("FileGroupID", 64);
+      filegroups.addProperty("ParentRef", GS3SQLField.INTEGER_TYPE);
+      filegroups.addProperty("ParentType", 16);
+      statement.execute(filegroups.toString());
+    
+      //   create file table
+      GS3SQLCreateTable files = new GS3SQLCreateTable("files");
+      // TODO: the unique identifier from file groups table
+      files.addProperty("FileGroupRef", GS3SQLField.INTEGER_TYPE);
+      files.addProperty("FileLocType", 64);
+      files.addProperty("FileLocation");
+      files.addProperty("MIMEType", 64);
+      files.addProperty("FileID", 32);
+      statement.execute(files.toString());
+      
+      //   create admin table...etc.
+
+      // 
+      // END OF METS TABLES
+      //
+
+      //
+      // BEGINNING OF GSDL TABLES
+      //
+      GS3SQLCreateTable classifiers = new GS3SQLCreateTable("classifiers");
+      classifiers.addAutoPrimaryKey("ClassifyRef");
+      classifiers.addProperty("ClassifyID");
+      classifiers.addProperty("ParentID");
+      classifiers.addProperty("ClassifyOrder");
+      classifiers.addProperty("Name");
+      classifiers.addProperty("Description");
+      classifiers.addProperty("NumLeafDocs", GS3SQLField.INTEGER_TYPE);
+      statement.execute(classifiers.toString());
+
+      GS3SQLCreateTable classDocs = new GS3SQLCreateTable("classdocuments");
+      classDocs.addProperty("ClassifyRef", GS3SQLField.INTEGER_TYPE);
+      classDocs.addProperty("DocID");
+      classDocs.addProperty("DocOrder", GS3SQLField.INTEGER_TYPE);
+      statement.execute(classDocs.toString());
+
+      GS3SQLCreateTable classData = new GS3SQLCreateTable("classdata");
+      classData.addProperty("ClassifyRef", GS3SQLField.INTEGER_TYPE);
+      classData.addProperty("Label");
+      classData.addProperty("Value");
+      statement.execute(classData.toString());
+
+      statement.close();
+      //
+      // END OF GSDL TABLES
+      //
+
+    }
+    catch (SQLException ex)
+    {
+      System.out.println(ex.toString());
+      return false;
+    }
+    return true;
+  }
+
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java	(revision 14150)
@@ -0,0 +1,76 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.io.InputStream;
+import java.util.Properties;
+import org.greenstone.gsdl3.util.GlobalProperties;
+
+public class GS3SQLConnectionFactory
+{
+
+    public static Connection getConnection(String database)
+    {
+	try {	    
+	    String server = GlobalProperties.getProperty("mysql.server", "localhost");
+	    String port = GlobalProperties.getProperty("mysql.tcp.port", "3306");
+	    String password = GlobalProperties.getProperty("mysql.gsdl3admin.password");
+	    
+	    Class.forName("com.mysql.jdbc.Driver").newInstance();
+	    String connection_address = "jdbc:mysql://"+server+":"+port+"/"+database+"?user=gsdl3admin";
+	    if (password != null && !password.equals("")) {
+		connection_address += "&password="+password;
+	    }
+	    Connection connection = DriverManager.getConnection(connection_address);
+	    return connection;
+	}
+	catch (Exception ex) {
+	    System.out.println("GS3SQLConnectionFactory.getConnection() Error:");
+	    System.out.println(ex.getMessage());
+	}
+	return null;
+    }
+    
+    public static SQLConnection getSQLConnection(String database)
+    {
+	Connection c = getConnection(database);
+	if (c== null) {
+	    return null;
+	}
+	return new SQLConnection(c, database);
+    }
+
+    
+    public static GS3SQLConnection getGS3SQLConnection(String database)
+    {
+	Connection c = getConnection(database);
+	if (c== null) {
+	    return null;
+	}
+	return new GS3SQLConnection(c, database);
+    }
+
+    /* returns a Connection to the server, */
+    public static Connection getConnection() 
+    {
+	return getConnection("test");
+    }
+
+    /* returns a Connection to the server, */
+    public static SQLConnection getSQLConnection() 
+    {
+	return getSQLConnection("test");
+    }
+
+    public static GS3SQLConnection getGS3SQLConnection()
+    { 
+	return getGS3SQLConnection("test");
+    }
+
+    public static void main(String args[])
+    { 
+	GS3SQLConnection connection = getGS3SQLConnection();
+	connection.initCollection("maya");
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java	(revision 14150)
@@ -0,0 +1,96 @@
+package org.greenstone.gsdl3.gs3build.database;
+  
+import java.util.List;
+import java.util.ArrayList;
+
+
+public class GS3SQLCreateTable
+{ 
+    String tableName;
+    List   properties;
+    List   primaryKey;
+    List   indexed;
+
+    public GS3SQLCreateTable(String tableName)
+    { 
+	this.tableName = tableName;
+	this.properties = new ArrayList();
+	this.primaryKey = new ArrayList();
+	this.indexed    = new ArrayList();
+    }
+    
+    public void addAutoPrimaryKey(String fieldName)
+    {
+	GS3SQLField field = new GS3SQLField(fieldName, GS3SQLField.AUTOINTEGER_TYPE);
+	this.properties.add(field);
+	this.setPrimaryKey(fieldName);
+    }
+
+    public void addProperty(String fieldname)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname);
+	this.properties.add(field);
+    }
+    
+    public void addProperty(String fieldname, String type)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname, type);
+	this.properties.add(field);
+    }
+
+    public void addProperty(String fieldname, int length)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname, length);
+	this.properties.add(field);
+    }
+
+    public void setPrimaryKey(String fieldname)
+    { 
+	this.primaryKey.clear();
+	this.primaryKey.add(fieldname);
+    }
+
+    public void extendPrimaryKey(String fieldname)
+    { 
+	this.primaryKey.add(fieldname);
+    }
+    
+    public void addIndex(String fieldname)
+    { 
+	this.indexed.add(fieldname);
+    }
+
+    public String toString()
+    {	
+	StringBuffer reply = new StringBuffer();
+	
+	reply.append("CREATE TABLE ");
+	reply.append(this.tableName);
+	reply.append(" (");
+	for (int f = 0; f < this.properties.size(); f ++) {
+	    GS3SQLField field = (GS3SQLField) this.properties.get(f);
+	    if (f != 0) {
+		reply.append(",");
+	    }
+	    reply.append(field.toString());
+	}
+
+	for (int p = 0; p < this.primaryKey.size(); p ++) {
+	    if (p == 0){
+		reply.append(", PRIMARY KEY(");
+	    }
+	    else {
+		reply.append(",");
+	    }
+	    reply.append(this.primaryKey.get(p).toString());
+	    
+	    if (p == this.primaryKey.size() - 1) {
+		reply.append(")");
+	    }
+	}
+	// TODO: add primary key, index etc. notations
+	reply.append(");");
+	
+	return reply.toString();
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java	(revision 14150)
@@ -0,0 +1,35 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLDelete
+{ 
+  GS3SQLWhere where;
+  String      table;
+
+  public GS3SQLDelete(String table)
+  { this.table = table;
+    this.where = null;
+  }
+
+  public void setWhere(GS3SQLWhere where)
+  { this.where = where;
+  }
+
+  public String toString()
+  {
+    StringBuffer reply = new StringBuffer();
+    reply.append("DELETE FROM ");
+    reply.append(this.table);
+
+    if (this.where != null)
+    { reply.append(" WHERE ");
+      reply.append(where.toString());
+    }
+
+    return reply.toString();
+  }
+}
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java	(revision 14150)
@@ -0,0 +1,74 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLField
+{ String name;
+  int    length;
+  String type;
+
+    public final static String VARCHAR_TYPE = "VARCHAR";
+    public final static String INTEGER_TYPE = "INTEGER";
+    public final static String AUTOINTEGER_TYPE = "AUTOINTEGER";
+    public final static String DATE_TYPE = "DATE";
+    public final static String DATETIME_TYPE = "DATETIME";
+
+    public GS3SQLField(String name)
+    { this.name = name;
+      this.length = 1024;
+      this.type   = VARCHAR_TYPE;
+    }
+
+    public GS3SQLField(String name, int length)
+    { this.name = name;
+      this.length = length;
+      this.type   = VARCHAR_TYPE;
+    }
+
+    public GS3SQLField(String name, String type)
+    { this.name = name;
+      this.type   = type;
+      this.length = 1024;
+    }
+
+    public GS3SQLField(String name, String type, int length)
+    { this.name = name;
+      this.length = length;
+      this.type   = type;
+    }
+
+    public String getName()
+    { return this.name;
+    }
+
+    public int getLength()
+    { return this.length;
+    }
+
+    public String toString()
+    { StringBuffer reply = new StringBuffer();
+
+      reply.append(this.name);
+      reply.append(" ");
+      if (this.type.equals(VARCHAR_TYPE))
+      {
+	if (length > 255) {
+	  reply.append("TINYTEXT");
+	}
+	else {
+	  reply.append("VARCHAR(");
+	  reply.append(Integer.toString(this.length));
+	  reply.append(")");
+	}
+      }
+      else if (this.type.equals(AUTOINTEGER_TYPE))
+      { reply.append("INTEGER AUTO_INCREMENT");
+      }
+      else
+      {
+	reply.append(this.type);
+      }
+      return reply.toString();
+    }
+
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java	(revision 14150)
@@ -0,0 +1,54 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLInsert extends GS3SQLAction
+  { public GS3SQLInsert(String table)
+    { super(table);
+    }
+
+    public String toString()
+    {
+      StringBuffer reply = new StringBuffer();
+      reply.append("INSERT INTO ");
+      reply.append(this.table);
+      reply.append(" ");
+      if (this.values.size() >= 1)
+	reply.append("(");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+        reply.append(value.field);
+      }
+      if (this.values.size() >= 1)
+	reply.append(")");
+      reply.append(" VALUES ");
+      if (this.values.size() >= 1)
+	reply.append("(");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.DATETIME_TYPE) ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+        reply.append(value.value);
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.DATETIME_TYPE) ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+      }
+      if (this.values.size() >= 1)
+	reply.append(")");
+      reply.append(";");
+      return reply.toString();
+    }
+  }
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java	(revision 14150)
@@ -0,0 +1,48 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLSelect extends GS3SQLAction
+{ GS3SQLWhere where;
+  boolean     distinct;
+
+  public GS3SQLSelect(String table)
+  { super(table);
+    this.where  = null;
+    this.distinct = false;
+  }
+    
+  public void setWhere(GS3SQLWhere where)
+  { this.where = where;
+  }
+
+  public void setDistinct(boolean distinct)
+  { this.distinct = distinct;
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+  
+    reply.append("SELECT ");
+
+    if (this.distinct)
+    { reply.append("DISTINCT ");
+    }
+ 
+    for (int i = 0; i < this.values.size(); i ++)
+    { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+      if (i > 0)
+	reply.append(", ");
+      reply.append(value.field);
+    }
+
+    reply.append(" FROM ");
+    reply.append(this.table);
+
+    if (this.where != null)
+    { reply.append(" WHERE ");
+      reply.append(where.toString());
+    }
+    return reply.toString();
+  }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java	(revision 14150)
@@ -0,0 +1,50 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public   class GS3SQLUpdate extends GS3SQLAction
+  { GS3SQLWhere where;
+
+    public GS3SQLUpdate(String table)
+    { super(table);
+      this.where  = null;
+    }
+
+    public void setWhere(GS3SQLWhere where)
+    { this.where = where;
+    }
+
+    public String toString()
+    {
+      StringBuffer reply = new StringBuffer();
+      reply.append("UPDATE ");
+      reply.append(this.table);
+      reply.append(" SET ");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+        reply.append(value.field);
+	reply.append("=");
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+	reply.append(value.value);
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+      }
+      if (this.where != null)
+      { reply.append(" WHERE ");
+	reply.append(where.toString());
+      }
+      reply.append(";");
+      return reply.toString();
+    }
+  }
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java	(revision 14150)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLValue
+{ String field;
+  String value;
+  String type;
+
+  public GS3SQLValue(String field, String value)
+  {
+    this.field = field;
+    this.value = value;
+    this.type  = null;
+  }
+
+  public GS3SQLValue(String field, String value, String type)
+  {
+    this.field = field;
+    this.value = value;
+    this.type  = type;
+  }
+
+  public String getType()
+  {
+    return this.type;
+  }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java	(revision 14150)
@@ -0,0 +1,55 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class GS3SQLWhere implements GS3SQLWhereElement
+{ String condition;
+  List   elements;
+
+  public static final String AND_CONDITION = "AND";
+  public static final String OR_CONDITION = "OR";
+
+  public GS3SQLWhere(GS3SQLWhereElement element)
+  { this.condition = AND_CONDITION;
+    this.elements  = new ArrayList();
+    this.elements.add(element);
+  }
+
+  public GS3SQLWhere()
+  { this.condition = AND_CONDITION;
+    this.elements  = new ArrayList();   
+  }
+
+  public GS3SQLWhere(GS3SQLWhereElement element, String condition, GS3SQLWhereElement other)
+  { this.elements = new ArrayList();
+    this.elements.add(element);
+    this.elements.add(other);
+    this.condition = condition;
+  }
+
+  public void add(GS3SQLWhereElement element)
+  { this.elements.add(element);
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    for (int i = 0; i < this.elements.size(); i ++)
+    {
+      if (i != 0) {
+	reply.append(" ");
+	reply.append(this.condition);
+	reply.append(" ");
+      }
+      reply.append(this.elements.get(i).toString());
+    }
+    return reply.toString();
+  }
+
+  public void setCondition(String condition)
+  { this.condition = condition;
+  }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java	(revision 14150)
@@ -0,0 +1,10 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public interface GS3SQLWhereElement
+{ // empty interface for type safety
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java	(revision 14150)
@@ -0,0 +1,47 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class GS3SQLWhereItem implements GS3SQLWhereElement
+{ String field;
+  String comparison;
+  String value;
+  String type;
+
+  public GS3SQLWhereItem(String field, String comparison, String value, String type)
+  { this.field      = field;
+    this.comparison = comparison;
+    this.value      = value;
+    this.type       = type;
+  }
+
+  public GS3SQLWhereItem(String field, String comparison, String value)
+  { this.field      = field;
+    this.comparison = comparison;
+    this.value      = value;
+    this.type       = GS3SQLField.VARCHAR_TYPE;
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    reply.append(this.field);
+    if (!this.comparison.equals("=")) {
+      reply.append(" ");
+    }
+    reply.append(this.comparison);
+    if (!this.comparison.equals("=")) {
+      reply.append(" ");
+    }
+    if (type.equals(GS3SQLField.VARCHAR_TYPE)) {
+	reply.append("\"" + this.value + "\"");
+    }
+    else {
+	reply.append(this.value);
+    }
+    return reply.toString();
+  }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java	(revision 14150)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.*;
+
+public class MySQLAdaptor implements DatabaseAdaptorInterface
+{
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/SQLConnection.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/SQLConnection.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/database/SQLConnection.java	(revision 14150)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+public class SQLConnection
+{
+    protected Connection connection;
+    protected String database;
+    
+    public SQLConnection(java.sql.Connection connection, String database)
+    { 
+	this.connection = connection;
+	this.database = database;
+    }
+    
+    protected void finalize() {
+	if (this.connection!=null) {
+	    try {
+		this.connection.close();		
+	    } catch (Exception e) {}
+	}	
+	this.connection = null;   
+    }
+    
+    public void close() {
+	finalize();
+    }
+    
+    public Statement createStatement() throws SQLException
+    { 
+	return this.connection.createStatement();
+    }
+    
+    public boolean connectToDatabase(String database) {
+	if (this.connection != null) {
+	    try {
+		this.connection.close();
+	    } catch (Exception e) {}
+	}
+	this.connection = GS3SQLConnectionFactory.getConnection(database);
+	if (this.connection == null) {
+	    return false;
+	}
+	this.database = database;
+	return true;
+    }
+    
+    public boolean dropDatabase(String database) {
+	try {
+	    Statement statement = this.connection.createStatement();
+	    statement.execute("DROP DATABASE "+database+";");
+	    statement.close();
+	}
+	catch (SQLException ex){
+	    System.err.println(ex);
+	    return false;
+	}
+	return true;
+	
+    }
+
+    public boolean createDatabase(String database) {
+	try {
+	    String command = "CREATE DATABASE " + database;
+	    Statement statement = this.connection.createStatement();
+	    statement.execute(command);
+	    statement.close();
+	} catch (Exception e) {
+	    System.err.println(e);
+	    return false;
+	}
+	 
+	return true;
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 14150)
@@ -0,0 +1,505 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Date;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import java.sql.Timestamp;
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.metadata.NamespaceFactory;
+import org.greenstone.gsdl3.gs3build.metadata.StructureIdentifierFactory;
+import org.greenstone.gsdl3.gs3build.metadata.GSDL3Namespace;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSHeader;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSNamespace;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+/**
+ *  Provide a base-line functionality for the <code>DocumentInterface</code>
+ *  class.
+ */
+
+public abstract class AbstractDocument implements DocumentInterface
+{
+  METSFileSet        fileSet;
+  METSDescriptiveSet metadata;
+  METSStructureSet   structureSet;
+  METSHeader         header;
+  DocumentID         id;
+  boolean            isModified;
+  StructureIdentifierFactory structureIdFactory;
+  java.sql.Timestamp firstDate;
+  java.sql.Timestamp indexDate;
+  java.sql.Timestamp modifiedDate;
+
+  /**
+   *  <p>Create a very vanilla document with a given document identifier.</p>
+   *  <p>Most commonly used in dealing with loading files using DocumentFactory
+   *  or similar.</p>
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   */
+  public AbstractDocument(DocumentID id)
+  { this.fileSet      = new METSFileSet();
+    this.metadata     = new METSDescriptiveSet();
+    this.header       = new METSHeader();
+    this.structureSet = new METSStructureSet();
+    this.id           = id;
+    this.structureIdFactory = new StructureIdentifierFactory();
+
+    java.util.Date thisDate = new java.util.Date();
+    this.firstDate = new java.sql.Timestamp(thisDate.getTime());
+    this.indexDate = new java.sql.Timestamp(thisDate.getTime());
+    this.modifiedDate = null; // as a signature that the modified date needs finding...
+  }
+  
+  /**
+   *  Create a basic document from a given <code>URL</code.  This is usually the form
+   *  called through the recognisers.
+   *
+   *  @param <code>URL</code> the URL of the first file in the document package
+   */
+  public AbstractDocument(URL url)
+  { this.fileSet      = new METSFileSet();
+    METSFile metsFile = this.fileSet.addFile(url);
+    this.metadata     = new METSDescriptiveSet();
+    this.header       = new METSHeader();
+    this.structureSet = new METSStructureSet();
+    this.id           = null;
+
+    java.util.Date thisDate = new java.util.Date();
+    this.firstDate = new java.sql.Timestamp(thisDate.getTime());
+    this.indexDate = new java.sql.Timestamp(thisDate.getTime());
+
+    METSStructure structure = new METSStructure("All", "All", "Whole Document");
+    METSDivision documentBody = new METSDivision("All", "All", "All", "Whole Document", "Document");
+    structure.addDivision(documentBody);
+    this.structureSet.addStructure(structure);
+    documentBody.addFileReference(metsFile.getID());
+    documentBody.addMetadataReference("DM1");
+  }
+
+  /**
+   *  Set the identified for the document.  Every document should have
+   *  a document number set on its accession, either through metadata
+   *  placed upon it internally or externally, or by assignment through
+   *  a <code>DocumentIDFactory</code>.  Each identifier should be
+   *  unique.
+   *
+   *  @param <code>DocumentID</code> the document identifier - in XML
+   *         terms, the gsdl3:id element.
+   */
+  public void setID(DocumentID id)
+  { this.id = id;
+    this.isModified = true;
+  }
+
+  /**
+   *  Get the document identifier - this should be unique to the document,
+   *  but care must be taken in the configuration of the collection to
+   *  ensure that this is the case.
+   *
+   *  @return <code>DocumentID</code> the identifer
+   */
+  public DocumentID getID()
+  { return this.id;
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  { return true;
+  }
+
+  /**
+   *  Get the date that this file was modified
+   */
+  public long getFilesDatestamp()
+  { return this.fileSet.getModifiedDatestamp(); 
+  }
+
+  /**
+   *  Get the date that this file was modified
+   */
+  public long getModifiedDatestamp()
+  { if (this.modifiedDate == null) {
+      this.setModifiedDatestamp();
+    }
+    return this.modifiedDate.getTime();
+  }
+
+  /**
+   *  Update/set the date of the most recent file modification
+   */
+  public void setModifiedDatestamp()
+  { this.modifiedDate = new java.sql.Timestamp(this.fileSet.getModifiedDatestamp()); 
+  }
+
+  /**
+   *  Get the date that this document was first indexed
+   */
+  public long getAccessionDate()
+  { return this.firstDate.getTime();
+  }
+
+  /**
+   *  Get the date that this document was last indexed
+   */
+  public long getLastIndexedDate()
+  { return this.indexDate.getTime();
+  }
+
+  /**
+   *  Set the last indexed date for this document;
+   */
+  public void setLastIndexedDate(long timestamp)
+  { this.indexDate = new java.sql.Timestamp(timestamp);
+  }
+
+  /**
+   *  Check if this document is in the database already.
+   *
+   *  In this simple implementation, the first file in the document's "default"
+   *  filegroup is taken to be the canonical file for this document - any document
+   *  of the same type with the same canonical file is taken to be a match.
+   *
+   *  @return <code>boolean</code> - if a matching document is found in the
+   *                                 database.
+   */
+  public String getDuplicateID(GS3SQLConnection connection)
+  { //String query = "SELECT * FROM document INNER JOIN filegroups ON document.docId=filegroups.docId WHERE DocType=\"" + HTML_DOCUMENT_TYPE + "\"";
+
+    // Query for documents using the same file...
+    String query = "SELECT DocID FROM files INNER JOIN filegroups ON files.FileGroupRef=filegroups.FileGroupRef WHERE (filegroups.FileGroupId=\"default\" AND files.FileLocation=\"" + this.fileSet.getFile(0).getLocation().toString() + "\")";
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+
+	List docs = new ArrayList();
+
+	if (results.first()) {
+	    do {
+		String value = results.getString("DocID");		
+		docs.add(value);
+	    } while (results.next());
+	    
+	    Iterator docIterator = docs.iterator();
+	    while (docIterator.hasNext()) {
+		String docId = docIterator.next().toString();
+		String innerQuery = "SELECT * FROM document WHERE DocID=\"" + docId + "\"";
+		results = statement.executeQuery(innerQuery);
+		if (results.first()) {
+		    String docType = results.getString("DocType");
+		    if (docType.equals(this.getDocumentType())) {
+			return docId;
+		    }
+		}
+	    }
+	}
+	statement.close();
+    }
+    catch (java.sql.SQLException sqlEx) {
+      System.err.println("AbstractDocument.getDuplicateID(): "+sqlEx);
+    }
+
+    return "";
+  }
+
+  /**
+   *  Obtain the METS header of this document
+   *
+   *  @return <code>METSHeader</code> the header
+   */
+  public METSHeader getHeader()
+  { return this.header;
+  }	
+
+  /**
+   *  Set the METS header for this document.
+   *
+   *  @param <code>METSHeader</code> the header
+   */
+  public void setHeader(METSHeader header)
+  { this.header = header;
+  }
+
+  /**
+   *  A simple implementation of the isDocumentType function that does <b>not</b> consider
+   *  inheritance - it <code>must</code> be extended as required.
+   */
+  public boolean isDocumentType(String type)
+  { return type.equals(this.getDocumentType());
+  }
+
+  public abstract String getDocumentType();
+  
+  public abstract String getDocumentText();
+
+  public abstract String getSectionText(String sectionId);
+
+  public String getMETSType()
+  { return "document";
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void addDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.addDocumentMetadata(label.getNamespace(), label.getLabel(), value);
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void setDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.setDocumentMetadata(label.getNamespace(), label.getLabel(), value);
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void addDocumentMetadata(String namespace, String label, String value)
+  { this.metadata.addMetadata("default", namespace, label, value);
+    this.isModified = true;
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void setDocumentMetadata(String namespace, String label, String value)
+  { this.metadata.setMetadata("default", namespace, label, value);
+    this.isModified = true;
+  }
+
+  /**
+   *  @see DocumentInterface:removeDocumentMetadata
+   */
+  public void removeDocumentMetadata(String namespace, String label)
+  { this.metadata.removeMetadata("default", namespace, label);
+    this.isModified = true;
+  }
+
+  /**
+   *
+   */
+  public void removeAllMetadata(String namespace, String label)
+  { this.metadata.removeAllMetadata(namespace, label);
+    this.isModified = true;
+  }
+
+  /**
+   *  Post metadata to a file in this document - the appropriate changes
+   *  should be made...
+   */
+  public void postFileMetadata(URL fileLocation, String namespace, String label, String value)
+  {
+    // First get the list of file groups, etc. that this file is associated with...
+    List fileGroups = this.fileSet.findGroups(fileLocation);
+
+    // Next, get the METS divisions associated with each file group...
+    List divisions = this.structureSet.findDivisionsForFiles(fileGroups);
+    
+    // Finally, post the metadata to the metadata group associated with each structure
+    Iterator divisionIter = divisions.iterator();
+    while (divisionIter.hasNext())
+    { METSDivision division = (METSDivision) divisionIter.next();
+
+      // get the open namespace for this division
+      METSNamespace namespaceMetadata = division.findNamespace(namespace, true, this.metadata);
+
+      // then post the metadata to it...
+      namespaceMetadata.addMetadata(label, value);
+    }
+  }
+
+  /**
+   *  Get the metadata structure of the document
+   *
+   *  @return <code>METSDescriptive</code> the metadata holder for the document.
+   */
+  public METSDescriptiveSet getDocumentMetadata()
+  { return this.metadata;
+  }
+
+  /**
+   *  Set the metadata structure for this document
+   *
+   *  @param <code>METSDescriptive</code> the new metadata holder for the document.
+   */
+  public void setDocumentMetadata(METSDescriptiveSet metadata)
+  { this.metadata = metadata;
+    this.isModified = true;
+  }
+
+  /**
+   *  Get the metadata structure of the document
+   *
+   *  @return <code>METSStructureSet</code> the metadata holder for the document.
+   */
+  public METSStructureSet getDocumentStructure()
+  { return this.structureSet;
+  }
+
+  public void setDocumentStructure(METSStructureSet structureSet)
+  { this.structureSet = structureSet;
+  }
+
+  /**
+   *  Get the values associated with a particular metadata value.
+   *
+   *  @param <code>String</code> the namespace to find the values in.
+   *  @param <code>String</code> the label to match to find the values.
+   *
+   *  @return <code>List</code> the values.
+   */
+  public List getDocumentMetadataItem(String namespace, String label)
+  { return this.metadata.getMetadata("default", namespace, label);
+  }
+
+  /**
+   *  Get the values associated with a particular metadata value.
+   *
+   *  @param <code>String</code> the namespace and label separated by a
+   *         colon.
+   *
+   *  @return <code>List</code> the values.
+   */
+  public List getDocumentMetadataItem(String namespaceLabel)
+  { String namespace, label;
+  
+    int colonAt = namespaceLabel.indexOf(':');
+    if (colonAt < 0)
+    { namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
+      label     = namespaceLabel;
+    }
+    else
+    { namespace = namespaceLabel.substring(0, colonAt);
+      label     = namespaceLabel.substring(colonAt+1);
+    }
+    return this.metadata.getMetadata("default", namespace, label);
+  }
+
+  /**
+   *  @see DocumentInterface:getDocumentFiles
+   */
+  public METSFileSet getDocumentFiles()
+  { return this.fileSet;
+  }
+
+  public void setDocumentFiles(METSFileSet fileSet)
+  { this.fileSet = fileSet;
+  }
+
+  /**
+   *  This is just a dummy function that does nothing at this level...
+   */
+  public org.w3c.dom.Document getDOMDocument()
+  { return null;
+  }
+  
+    public int getNumSections() {
+	return 3;
+	/*	String query = "SELECT count(*) FROM divisions WHERE DocID="+this.id+" AND ParentType='Division'";
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    
+	    if (results.first()) {
+		int count = results.getInt(0);
+		System.err.println("count = "+count);
+		return count;
+	    }
+	} catch (Exception e) {
+	    System.err.println("AbstractDocument.getNumSections(): "+e);
+	}
+	return -1;*/
+    }
+  /**
+   *  @see DocumentInterface:isMETSCompatible
+   */
+  public boolean isMETSCompatible()
+  { return true;
+  }
+
+  /**
+   *  Use a default document writer - this may be overridden for subclasses...
+   *
+   *  @see DocumentInterface:writeMETSObject
+   */
+  public DocumentWriter getMETSWriter()
+  { return new DocumentWriter();
+  }
+
+  /**
+   *  Use a default SQL document writer - this may be overridden for subclasses...
+   *
+   */
+  public DocumentSQLWriter getSQLWriter()
+  { return new DocumentSQLWriter();
+  }
+
+  /**
+   *  Obtain a document from the SQL database
+   */
+    public static AbstractDocument readSQL(GS3SQLConnection connection, ResultSet sqlResult)
+  { try {
+      DocumentID id = new DocumentID(sqlResult.getString("DocID"));
+      String     type = sqlResult.getString("docType");
+
+      // Use a factory method to create the correct subtype...
+      AbstractDocument document = DocumentFactory.createDocument(type, id);
+      // Append the document date information
+      document.indexDate    = sqlResult.getTimestamp("IndexedDate");
+      document.firstDate    = sqlResult.getTimestamp("AccessionDate");
+      document.modifiedDate = sqlResult.getTimestamp("ModifiedDate");
+
+      // Get the individual components of the document
+      METSFileSet fileSet = METSFileSet.readSQL(document, connection);
+      document.setDocumentFiles(fileSet);
+      METSDescriptiveSet descriptiveSet = METSDescriptiveSet.readSQL(document, connection);
+      document.setDocumentMetadata(descriptiveSet);
+      METSStructureSet structureSet = METSStructureSet.readSQL(document, connection);
+      document.setDocumentStructure(structureSet);
+
+      // indicate that the document is not currently modified
+      document.setChanged(false);
+      return document;
+    }
+    catch (SQLException sqlEx) {
+      System.err.println("AbstractDocument.readSQL(): Failure to load document: " + sqlEx);
+    }
+    return null;
+  }
+
+  /**
+   *  
+   */
+  public boolean isChanged()
+  { return this.isModified;
+  }
+
+  public void setChanged(boolean isModified)
+  { this.isModified = isModified;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 14150)
@@ -0,0 +1,101 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+/**
+   A simple base class for Recognisers
+   A new recogniser should set preferred_mime_type, filename_extensions, and document_type in the constructor. 
+   The DocumentFactory class needs to be modified to map the documentType to 
+   the corresponding document
+   And thats all for simple recognisers.
+   More complicated recognisers can just implement from scratch RecogniserInterface
+*/
+abstract public class AbstractRecogniser implements RecogniserInterface
+{
+
+    DocumentList list_repository = null;
+    String preferred_mime_type = "SET THIS IN THE CONCRETE CLASS"; 
+    ArrayList filename_extensions = null;
+    String document_type = "SET THIS IN THE CONCRETE CLASS";
+    CollectionManager coll_manager = null;
+
+    /** The constructor should set the variables 
+     * preferredMimeType, filename_extensions and documentType
+     */
+    public AbstractRecogniser() {
+
+    }
+    
+    /** set the collection manager */
+    public void setCollectionManager(CollectionManager coll_man) {
+	this.coll_manager = coll_man;
+    }
+    /** configure by default does nothing */
+    public boolean configure(Element config_elem){
+	return true;
+    }
+    public void setListRepository(DocumentList docList) {
+	this.list_repository = docList;
+    }
+
+    protected boolean isAcceptedFilename(String filename) {
+	if (filename_extensions == null) {
+	    return false; // or should this be true??
+	}
+	for (int i=0; i<filename_extensions.size(); i++) {
+	    if (filename.endsWith((String)filename_extensions.get(i))) {
+		return true;
+	    }
+	}
+	return false;
+    }
+    
+    public boolean parseDocument(METSFile file)
+    {
+	String MIMEType = file.getMIMEType();
+	if (MIMEType == null || 
+	    MIMEType.equals(this.preferred_mime_type)) {
+	    URL location = file.getLocation();
+	    return this.parseDocument(location);
+	}
+	return false;
+    }
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if ( isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting "+document_type+" Document " + filename);
+		AbstractDocument doc = DocumentFactory.createDocument(document_type, url);
+		this.list_repository.addDocument(doc);
+		return true;
+	    }
+	} else {
+	    // Get Mime type remotely, and then proceed if required
+	    String mimeType = HTTPTools.getMIMEType(url);
+	    
+	    if (mimeType == this.preferred_mime_type) {
+		System.out.println("Posting "+document_type+" Document " + url.toString());
+		
+		AbstractDocument doc = DocumentFactory.createDocument(document_type, url);
+		this.list_repository.addDocument(doc);
+		return true;
+	    }
+	}
+	return false;
+    }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java	(revision 14150)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public class DocumentFactory {
+    
+    public static AbstractDocument createDocument(String type, DocumentID id) {
+	
+	if (type.equals(HTMLDocument.HTML_DOCUMENT_TYPE)) {
+	    return new HTMLDocument(id);
+	} else if (type.equals(TextDocument.TEXT_DOCUMENT_TYPE)) {
+	    return new TextDocument(id);
+	} else if (type.equals(JPEGDocument.JPEG_DOCUMENT_TYPE)) {
+	    return new JPEGDocument(id);
+	} else if (type.equals(IndexDocument.INDEX_DOCUMENT_TYPE)) {
+	    return new IndexDocument(id);
+	} else if (type.equals(MetadataDocument.METADATA_DOCUMENT_TYPE)) {
+	    return new MetadataDocument(id);
+	} else if (type.equals(METSDocument.METS_DOCUMENT_TYPE)) {
+	    return new METSDocument(id);
+	} else if (type.equals(GMLDocument.GML_DOCUMENT_TYPE)) {
+	    return new GMLDocument(id);
+	}
+	
+	/*
+	  else if (type.equals(ExtXMLDocument.EXTXML_DOCUMENT_TYPE))
+	  { return new ExtXMLDocument(id);
+	  }*/
+	return null;
+    }
+    public static AbstractDocument createDocument(String type, URL url) {
+	
+	if (type.equals(HTMLDocument.HTML_DOCUMENT_TYPE)) {
+	    return new HTMLDocument(url);
+	} else if (type.equals(TextDocument.TEXT_DOCUMENT_TYPE)) {
+	    return new TextDocument(url);
+	} else if (type.equals(JPEGDocument.JPEG_DOCUMENT_TYPE)) {
+	    return new JPEGDocument(url);
+	} else if (type.equals(IndexDocument.INDEX_DOCUMENT_TYPE)) {
+	    return new IndexDocument(url);
+	} else if (type.equals(MetadataDocument.METADATA_DOCUMENT_TYPE)) {
+	    return new MetadataDocument(url);
+	} else if (type.equals(METSDocument.METS_DOCUMENT_TYPE)) {
+	    return new METSDocument(url);
+	} else if (type.equals(GMLDocument.GML_DOCUMENT_TYPE)) {
+	    return new GMLDocument(url);
+	}
+	
+	/*
+	  else if (type.equals(ExtXMLDocument.EXTXML_DOCUMENT_TYPE))
+	  { return new ExtXMLDocument(id);
+	  }*/
+	return null;
+    }
+    
+  public static DocumentInterface readSQLDocument(GS3SQLConnection connection, DocumentID id)
+  {
+    String query = "SELECT * FROM document WHERE DocID=\""+id.toString()+"\";";
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+	
+	DocumentInterface di = null;
+	if (results.first()) {
+	    di = AbstractDocument.readSQL(connection, results);
+	}
+	statement.close();
+	return di;
+    }
+    catch (SQLException sqlEx) {
+      System.err.println("AbstractDocument.readSQLDocument():"+sqlEx);
+    }
+    return null;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java	(revision 14150)
@@ -0,0 +1,14 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public class DocumentID
+{
+	String id;
+
+	public DocumentID(String id)
+	{	this.id = id;
+	}
+
+	public String toString()
+	{	return this.id;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java	(revision 14150)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+
+public class DocumentIDFactory implements DocumentIDFactoryInterface
+{ private CollectionManager collection;
+
+  public DocumentIDFactory(CollectionManager collection)
+  { this.collection = collection;
+  }
+
+  public DocumentID getNewDocumentID(DocumentInterface document)
+  { DocumentID id;
+
+    String idString = this.collection.getNextDocumentID();
+    id = new DocumentID(idString);
+    
+    return id;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java	(revision 14150)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public interface DocumentIDFactoryInterface
+{
+	public DocumentID getNewDocumentID(DocumentInterface document);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 14150)
@@ -0,0 +1,282 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.List;
+import java.util.Map;
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public interface DocumentInterface
+{
+  /**
+   *  Get the immediate document type of the document.  The type may be derived/inherited
+   *  from other DocumentInterface types, but that cannot be checked here.
+   *
+   *  @return <code>String</code> the document type as a string
+   */
+  public String getDocumentType();
+  
+  /**
+   *  Check if this document is of a particular type, or is derived from a particular
+   *  type - i.e. inheritance is considered as well as the immediate type.
+   *
+   *  @param  <code>String</code> the type to check against.
+   *  @return <code>boolean</code> if the document matches the given type.
+   */
+  public boolean isDocumentType(String documentType);
+  
+  public void setID(DocumentID id);
+  public DocumentID getID();
+
+  /**
+   *  Get the METS type of the document.
+   *
+   *  @return <code>String</code> the document type as a string
+   */
+  public String getMETSType();
+  
+  public void setHeader(METSHeader header);
+  public METSHeader getHeader();
+
+  /**
+   *  Whether the document is indexed or not.
+   *
+   *  @return <code>boolean</code> <code>true</code> by default.
+   */
+  public boolean isIndexed();
+
+  /**
+   *  Get the latest date that a component file was modified...
+   */
+  public long getFilesDatestamp();
+
+  /**
+   *  Get the stored modified date for the files datestamp...
+   */
+  public long getModifiedDatestamp();
+
+  /**
+   *
+   */
+  public void setModifiedDatestamp();
+
+  /**
+   *  Get the date that this document was first indexed
+   */
+  public long getAccessionDate();
+
+  /**
+   *  Get the date that this document was last indexed
+   */
+  public long getLastIndexedDate();
+
+  /**
+   *  Set the last indexed date for this document;
+   */
+  public void setLastIndexedDate(long date);
+
+  /**
+   *  Check if the document matches another in the database
+   */
+  public String getDuplicateID(GS3SQLConnection connection);
+    
+  /**
+   *  The plain text of the document.
+   *
+   *  @return <code>String</code> This value may be <code>null</code>
+   *          for documents which have no textual component - e.g.
+   *          an image file
+   */
+  public String getDocumentText();
+
+  /**
+   *  The document as a dom object
+   */
+  public org.w3c.dom.Document getDOMDocument();
+
+  /**
+   *  Get the text of a section of this document
+   *
+   *  @return <code>String</code> the text as a string - for sections
+   *          that have no textual components, this value may be
+   *          <code>null</code>
+   */
+  public String getSectionText(String sectionId);
+	
+  /**
+   *  The metadata for the document, encoded as a map.
+   *
+   *  The returned Map may have the following properties:
+   *  1) It may be <code>null</code> - e.g. for plain text documents
+   *  2) Any value in the map may be a <code>List</code> object containing
+   *     more than one possible value - e.g. they key 'Author' may associated
+   *     with a List of several people.
+   *  3) Any value in the map may itself be a <code>Map</code> where the
+   *     encoding scheme permits groupings of hierarchical metadata items.
+   *  4) Where the namespace for a metadata item is known, its key will
+   *     include the namespace.  Hence Dublin Core 'Author' would be encoded
+   *     as "dc.Author" as the key.
+   *
+   *  @return <code>METSDescriptive</code> the metadata of the document
+   */
+  public METSDescriptiveSet getDocumentMetadata();
+  
+  /**
+   *  The metadata for a particular given upon the document, encoded as a <code>List</code>
+   *
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> the label of the values to obtain
+   *
+   *  @return <code>List</code> the metadata values
+   */
+  public List getDocumentMetadataItem(String namespace, String label);
+  
+  /**
+   *  The metadata for the document, encoded as a map.
+   *
+   *  @param <code>String</code> the namespace and label of the metadata, separated
+   *                             by a colon.  If no namespace is given, it is 
+   *                             defaulted
+   *
+   *  @return <code>List</code> the metadata values
+   */
+  public List getDocumentMetadataItem(String namespaceLabel);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.  This is a "cheap" form which doesn't have a separate
+   *  namespace element.  Either the data is to be stored in the "open"
+   *  Greenstone metadata namespace, or the namespace is encoded within
+   *  the label.
+   *
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures.  The label may commence
+   *                             with a namespace followed by a colon. 
+   *                             <p>e.g. "dc:title" for Dublin Core Title.</p>
+   *  @param <code>String</code> value of the metadata
+   */
+  public void addDocumentMetadata(MetadataLabel label, String value);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.  This is a "cheap" form which doesn't have a separate
+   *  namespace element.  Either the data is to be stored in the "open"
+   *  Greenstone metadata namespace, or the namespace is encoded within
+   *  the label.
+   *
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures.  The label may commence
+   *                             with a namespace followed by a colon. 
+   *                             <p>e.g. "dc:title" for Dublin Core Title.</p>
+   *  @param <code>String</code> value of the metadata
+   */
+  public void setDocumentMetadata(MetadataLabel label, String value);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void addDocumentMetadata(String namespace, String label, String value);
+
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void setDocumentMetadata(String namespace, String label, String value);
+
+  /**
+   *  Facilitate the removal of document metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   */
+  public void removeDocumentMetadata(String namespace, String label);
+  
+  /**
+   *  Facilitate the removal of all metadata of a given label from the document.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   */
+  public void removeAllMetadata(String namespace, String label);
+  
+  /**
+   *  Post metadata to a file in this document - the appropriate changes
+   *  should be made...
+   *
+   *  @param <code>URL</code> the location of the file...
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void postFileMetadata(URL fileLocation, String namespace, String label, String value);
+
+  /**
+   *  The constituent files for the document: each should be wrapped up
+   *  as a URL
+   *
+   *  @return <code>METSFileSet</code> the files which constitute the document
+   */
+  public METSFileSet getDocumentFiles();
+
+  /**
+   *  Set the constituent files for the document.
+   */
+  public void setDocumentFiles(METSFileSet fileSet);
+
+  /**
+   *  Obtain the structural information on the document
+   *
+   *  @return <code>METSStructureSet</code> the structural information on the
+   *          document.
+   */
+  public METSStructureSet getDocumentStructure();
+  
+  /**
+   *  Indicate whether the document can be stored in a native form in
+   *  a METS wrapper
+   *
+   *  @return <code>boolean</code> <code>true</code> if the document
+   *          is just to be wrapped in a METS shell.
+   */
+  public boolean isMETSCompatible();
+  
+  /**
+   *  Write the document into a METS wrapper - for many document types,
+   *  this will not actually be done by the document itself, but rather
+   *  by the default writer
+   */
+  public DocumentWriter getMETSWriter();
+  
+  /**
+   *  Get the writer to send the document to an SQL database
+   */
+  public DocumentSQLWriter getSQLWriter();
+
+  /**
+   *  Check if the document is changed or not
+   */
+  public boolean isChanged();
+
+  /**
+   *  Set the document modified state
+   */
+  public void setChanged(boolean isModified);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java	(revision 14150)
@@ -0,0 +1,535 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+
+import java.io.PrintWriter;
+import java.io.FileWriter;
+import java.io.File;
+import java.io.IOException;
+
+import java.net.URL;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLSelect;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLWhereItem;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLWhere;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLField;
+
+public class DocumentList
+{
+  DocumentInterface []       list;       // what is currently cached
+  int                        size;       // the maximum number in the cache
+  int                        used;       // the actual number in the cache
+  int                        count;      // the total number of known documents
+  DocumentIDFactoryInterface idFactory;  // A manufacturer of novel document IDs
+  GS3SQLConnection           connection; // used to query the SQL database
+
+  private static final int maxSize = 3;
+
+  public DocumentList(GS3SQLConnection connection)
+  { this.idFactory = null;
+    this.list  = new DocumentInterface[3];
+    this.used  = 0;
+    this.size  = 3;
+    this.count = 0;
+    this.connection = connection;
+  }
+
+  public DocumentList(DocumentIDFactoryInterface idFactory, GS3SQLConnection connection)
+  { this.idFactory = idFactory;
+    this.list  = new DocumentInterface[3];
+    this.used  = 0;
+    this.size  = 3;
+    this.count = 0;
+    this.connection = connection;
+  }
+
+  /**
+   *  Obtain the list of <code>DocumentID</code> objects representing the unique
+   *  document identifiers of documents that refer to the file given as a parameter.
+   *
+   *  @param <code>URL</code> the location of the file to match
+   *
+   *  @return <code>List</code> of <code>DocumentID</code> reference identifiers.
+   */
+  public List getDocumentIdsWithFile(URL fileLocation)
+  { List reply = new ArrayList();
+
+    GS3SQLSelect select = new GS3SQLSelect("files");
+    select.addField("*");
+    GS3SQLWhere  where  = new GS3SQLWhere(new GS3SQLWhereItem("FileLocation", "=", fileLocation.toString()));
+    select.setWhere(where);
+
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(select.toString());
+    
+	select = new GS3SQLSelect("filegroups");
+	select.addField("DocID");
+	select.setDistinct(true);
+      
+	where = new GS3SQLWhere();
+	where.setCondition(GS3SQLWhere.OR_CONDITION);
+	
+	GS3SQLWhereItem whereItem = null;
+
+      	results.first();
+	do {
+	  int fileGroupRef = results.getInt("FileGroupRef");
+	  whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(fileGroupRef), GS3SQLField.INTEGER_TYPE);
+	  where.add(whereItem);
+	}
+	while (results.next());
+	select.setWhere(where);
+	
+	results = statement.executeQuery(select.toString());
+	
+	results.first();
+	do {
+	  String docId = results.getString("DocID");
+	  reply.add(docId);
+	} while (results.next());
+	statement.close();
+    }
+    catch (SQLException sqlEx) {
+	System.err.println("DocumentList.getDocumentIdsWithFile(): "+sqlEx);
+    }
+    
+    return reply;
+  }
+
+  /**
+   *  Get a list of documents that match a given set of patterns,
+   *  within a given URL node.
+   *
+   *  @param <code>List</code> the list of patterns to match
+   *  @param <code>String</code> the partial URL of the root node under which o
+   *         match files.  NB: this is a <code>String</code> as the URL may be
+   *         incomplete and not properly match the strict requirements for <code>URL</code>
+   */
+  public List findDocumentIdsUsingFiles(List fileRefs, String withinNode)
+  {
+    StringBuffer queryBuffer = new StringBuffer("SELECT FileGroupRef FROM files WHERE FileLocation ");
+
+    Iterator files = fileRefs.iterator();
+    while (files.hasNext()) {
+      String file = files.next().toString();
+
+      if (withinNode != null) {
+	queryBuffer.append("REGEXP \"^");
+	queryBuffer.append(withinNode);
+	queryBuffer.append(".*");
+      }
+      else {
+	queryBuffer.append("REGEXP \"");
+      }
+      queryBuffer.append(file);
+      queryBuffer.append("\"");
+
+      if (files.hasNext()) {
+	queryBuffer.append(" OR ");
+      }
+    }
+    queryBuffer.append(";");
+    return this.findDocumentIdsUsingFileQuery(queryBuffer.toString());
+  }
+
+  public List findDocumentIdsUsingFile(String fileRef, String withinNode)
+  { String query = "SELECT FileGroupRef FROM files WHERE FileLocation REGEXP \"^"+withinNode+".*"+fileRef+"\";";
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+  /**
+   *  Return a list of document identifiers against a simple pattern.  No root node is given, so
+   *  any file matching the pattern given will be returned. USE WITH CAUTION!!!
+   *
+   *  @param <code>String</code> a fragment of file pathname to match against.
+   *
+   *  @return <code>List</code> of <code>DocumentID</code> objects.
+   */
+  public List findDocumentIdsUsingFile(String fileRef)
+  { 
+    // Get the simple list of file objects & their file group reference
+    String query = "SELECT FileGroupRef FROM files WHERE FileLocation REGEXP \"" + fileRef +"\";";
+
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+  public List findDocumentIdsUsingFileExact(String fileRef)
+  { 
+    // Get the simple list of file objects & their file group reference
+    String query = "SELECT FileGroupRef FROM files WHERE FileLocation=\"" + fileRef +"\";";
+
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+    private List findDocumentIdsUsingFileQuery(String query) {
+    
+	try {
+	
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+
+	if (!results.first()) {
+	    statement.close();
+	    return null;
+	}
+      
+      // get a list of group ids first and turn it into a query on filegroups
+      StringBuffer queryBuffer = new StringBuffer("SELECT * FROM filegroups WHERE ");
+      boolean first = true;
+      
+      do {
+	int groupRef = results.getInt("FileGroupRef");
+	
+	if (first) {
+	  first = false;
+	}
+	else {
+	  queryBuffer.append(" OR ");
+	}
+	queryBuffer.append("FileGroupRef=" + Integer.toString(groupRef));
+      } while (results.next());
+      queryBuffer.append(";");
+      
+      // make a holder for the actual file section identifiers
+      List divisions = new ArrayList();
+      
+      // expand (or, in fact, contract) through the document
+      // structures...recreating new filegroup queries as necessary
+      while (queryBuffer.length() > 0) {
+	results = statement.executeQuery(queryBuffer.toString());
+	
+	if (!results.first()) {
+	    statement.close();
+	    return null;
+	}
+	
+	queryBuffer = new StringBuffer();
+	do {
+	  String type = results.getString("ParentType");
+	  String parentRef = results.getString("ParentRef");
+	  if (type.equals(METSFileGroup.SECTION_PARENT)) {
+	    divisions.add(parentRef);
+	  }
+	  else {
+	    if (queryBuffer.length() > 0) {
+	      queryBuffer.append(" OR ");
+	    }
+	    queryBuffer.append("FileGroupRef=" + parentRef);
+	  }
+	} while (results.next());
+	
+	if (queryBuffer.length() > 0) {
+	  queryBuffer.insert(0, "SELECT * FROM filegroups WHERE ");
+	  queryBuffer.append(";");
+	}
+      }
+      
+      // ok, now find all the sections in which we are interested...
+      queryBuffer.setLength(0);
+      queryBuffer.append("SELECT DISTINCT DocID FROM filesection WHERE ");
+      Iterator iterator = divisions.iterator();
+      first = true;
+      
+      while (iterator.hasNext()) {
+	String ref = iterator.next().toString();
+	
+	if (first) {
+	  first = false;
+	}
+	else {
+	  queryBuffer.append(" OR ");
+	}
+	
+	queryBuffer.append("FileSectionRef="+ref);
+      }
+      queryBuffer.append(";");
+      
+      // execute the division query
+      results = statement.executeQuery(queryBuffer.toString());
+      
+      if (!results.first()) {
+	  statement.close();
+	  return null;
+      }
+
+      List reply = new ArrayList();
+      do {
+	reply.add(results.getString("DocID"));
+      } while (results.next());
+      
+      statement.close();
+      return reply;
+    }
+    catch (SQLException ex) {
+      System.err.println("DocumentList.findDocumentIdsUsingFileQuery()"+ ex);
+    }
+    return null;
+  }
+
+  /**
+   *  Cache a document into the cache, without writing it to the database.
+   *  Used directly by other parts of DocumentList when they know that the
+   *  document is in the database already, or they are going to write it
+   *  themselves...
+   *
+   *  @param <code>DocumentInterface</code> the document to cache
+   */
+  private void cacheDocument(DocumentInterface document)
+  { // increase cache size, etc. as necessary
+    if (this.used == this.size) {
+      if (this.size >= maxSize) {
+	for (int i = 0; i < this.size - 1; i ++) {
+	  this.list[i] = this.list[i+1];
+	}
+	this.used --;
+      }
+      else {
+	this.ensureSize((this.size * 2) > maxSize ? maxSize : (this.size * 2));
+      }
+    }
+    
+    // insert the document itself
+    this.list[this.used] = document;    
+  }
+
+  /**
+   *  Write the document into the document list (cache) and the database.
+   *
+   *  @param <code>DocumentInterface</code> the document itself
+   */
+  public void addDocument(DocumentInterface document)
+  { // initially, test if the document has a duplicate...
+    String duplicateDocID = document.getDuplicateID(this.connection);
+    if (duplicateDocID.length() > 0) {
+      System.out.println("Found duplicate document ");
+      return;
+    }
+
+    // first cache it...
+    this.cacheDocument(document);
+
+    // set the document identifier, if not already set
+    if (document.getID() == null) {
+      System.out.println("Posting new docuument ID");
+      DocumentID id = this.idFactory.getNewDocumentID(document);
+      document.setID(id);
+    }
+
+    // add to the database as well, if it is modified...
+    if (document.isChanged()) {
+      System.out.println("Document was changed");
+      document.getSQLWriter().writeDocument(document, this.connection);
+    }
+
+    // Remember that we've used one more item from the cache.
+    this.used ++;
+
+    // Note additional document
+    this.count ++;
+  }
+
+  /**
+   *  Note that an individual document is modified, and act accordingly
+   *
+   *  @param <code>DocumentInterface</code> the document
+   */
+  public void storeChangedDocument(DocumentInterface document)
+  { document.getSQLWriter().writeDocument(document, this.connection);
+  }
+
+  /**
+   *  Get an iterator across all the documents, not merely those in
+   *  the cache.  Note that this <code>Iterator</code> does <b>not</b>
+   *  support the <code>remove()</code> function, and will raise an
+   *  <code>UnsupportedOperationException</code> if you attempt to do
+   *  so.
+   *
+   *  @return <code>Iterator</code> the iterator across the documents.
+   */
+  public Iterator iterator()
+  { return new DocumentListIterator(connection);
+  }
+
+  /**
+   *  Get the nth member of the <b>cached</b> document list.
+   *
+   *  @deprecated
+   */
+  public DocumentInterface getDocument(int index)
+  { if (index < 0 || index >= this.used)
+    { return null;
+    }
+    return this.list[index];
+  }
+
+  /**
+   *  Simple "obtain a document" function
+   */
+  public DocumentInterface getDocument(DocumentID documentId)
+  {
+    DocumentInterface document = DocumentFactory.readSQLDocument(connection, documentId);
+    if (document != null) {
+      this.cacheDocument(document);
+    }
+    return document;
+  }
+
+  /**
+   *  Update timestamps on an entire document list - done at the beginning of a build cycle
+   *
+   *  @param <code>The date of the new build cycle</code>
+   */
+  public void updateTimestamps(long buildTimeStamp)
+  { Iterator documents = this.iterator();
+    int item = 0;
+
+    while (documents.hasNext())
+    { DocumentInterface document = (DocumentInterface) documents.next();
+      
+      long thisTimeStamp = document.getFilesDatestamp();
+      long lastTimeStamp = document.getModifiedDatestamp();
+
+      if (thisTimeStamp > lastTimeStamp) {
+	System.out.println("Updating timestamps " + thisTimeStamp + " " + lastTimeStamp);
+
+	DocumentSQLWriter.touchDocument(document.getID(), this.connection, buildTimeStamp, thisTimeStamp);
+      }
+    }
+  }
+
+  /**
+   *  A convenience method to map onto the old Vector source code...
+   */
+  protected void ensureSize(int size)
+  { DocumentInterface [] newList = new DocumentInterface[size];
+    System.arraycopy(this.list, 0, newList, 0, this.size);
+    this.list = newList;
+    this.size = size;
+  }
+
+  /**
+   *  Write the documents into a directory as METS/XML
+   */
+  public void writeDocuments(File directory)
+  { Iterator documents = this.iterator();
+    int item = 0;
+
+    while (documents.hasNext())
+    { DocumentInterface document = (DocumentInterface) documents.next();
+      try
+      { item ++;
+	File localFile = new File(directory, "Doc"+Integer.toString(item)+".xml");
+	FileWriter fileWriter = new FileWriter(localFile);
+	PrintWriter writer = new PrintWriter(fileWriter);
+	document.getMETSWriter().writeDocument(document, writer);
+	writer.close();
+	fileWriter.close();
+      }
+      catch (IOException io)
+      {	
+      }
+    }
+  }
+
+  public void writeSQLDocuments(GS3SQLConnection connection)
+  { for (int i = 0; i < this.used; i ++)
+    { this.list[i].getSQLWriter().writeDocument(this.list[i], connection);
+    }
+  }
+
+  public static DocumentList readSQLDocuments(GS3SQLConnection connection)
+  { DocumentList list = new DocumentList(connection);
+
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+    
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet documents = statement.executeQuery(select.toString());
+	if (documents.first()) { 
+	    do {
+		DocumentInterface document = AbstractDocument.readSQL(connection, documents);
+		list.addDocument(document);
+	    }
+	    while (documents.next());
+	}
+	statement.close();
+    }
+    catch (java.sql.SQLException ex) {
+	System.out.println("DocumentList.writeSQLDocuments(): "+ex);
+	return null;
+    }
+
+    return list;
+  }
+
+  public int getCount()
+  { return this.count;
+  }
+
+  public int size()
+  { return this.used;
+  }
+}
+
+class DocumentListIterator implements Iterator
+{
+  private boolean hasNext;
+  private Statement statement;
+  private ResultSet resultSet;
+  private GS3SQLConnection connection;
+
+  public DocumentListIterator(GS3SQLConnection connection)
+  {
+    this.connection = connection;
+
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+
+    try {
+	this.statement = connection.createStatement();
+      this.resultSet = statement.executeQuery(select.toString());
+      this.hasNext = this.resultSet.first();
+    } catch (SQLException ex) {
+      System.err.println("DocumentListIterator(): "+ex);
+      this.hasNext = false;
+    }
+  }
+
+  public boolean hasNext()
+  { return this.hasNext;
+  }
+
+  public Object next()
+  {
+    // get the 'next' document first
+    DocumentInterface document = AbstractDocument.readSQL(connection, this.resultSet);
+
+    // now actually step forward to the next item, so that we know if we have one!
+    try {
+      this.hasNext = this.resultSet.next();
+
+      if (!this.hasNext) {
+	  this.statement.close(); // be a good citizen & close used statement 
+      }
+    } catch (SQLException ex) {
+	System.err.println("DocumentList.iterator.next(): "+ex);
+      this.hasNext = false;
+    }
+    return document;
+  }
+  
+  public void remove() throws UnsupportedOperationException
+  { throw new UnsupportedOperationException("DocumentList does not support iterator removal of documents");
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java	(revision 14150)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.URL;
+
+public class DocumentLoader
+{
+  public static String getAsString(InputStream in)
+  { StringBuffer reply;
+    byte 	data[] = new byte[1024];
+    int		databytes;
+
+    reply	= new StringBuffer();
+    
+    try
+    {
+      do
+      { databytes = in.read(data);
+        if (databytes > 0)
+	{ reply.append(new String(data, 0, databytes));
+	}
+      } while (databytes >= 0);
+    }
+    catch (IOException io)
+    {
+    }
+    
+    return reply.toString();
+  }
+
+  public static String getAsString(File file)
+  { FileInputStream in;
+    String          reply = null;
+
+    try
+    { in	= new FileInputStream(file);
+      if (in == null)
+	{	return null;
+	}
+      reply = getAsString(in);
+      
+      in.close();
+    }
+    catch (IOException io)
+    { return null;
+    }
+    return reply;
+  }
+
+  public static String getAsString(URL url)
+  { if (url.getProtocol().equals("file"))
+    { File file = new File(url.getPath());
+      return getAsString(file);
+    }
+  
+    return null;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java	(revision 14150)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public class DocumentMetadata
+{
+	String name;
+	String value;
+
+	public DocumentMetadata(String name, String value)
+	{	this.name = name;
+		this.value = value;
+	}
+
+	public String getName()
+	{	return this.name;
+	}
+
+	public String getValue()
+	{	return this.value;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java	(revision 14150)
@@ -0,0 +1,98 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.PrintWriter;
+
+import java.sql.*;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class DocumentSQLWriter
+{
+  public DocumentSQLWriter()
+  {
+  }
+
+  public static boolean touchDocument(DocumentID docID, GS3SQLConnection connection, long touchTime, long modTime)
+  { GS3SQLUpdate update = new GS3SQLUpdate("document");
+    update.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", docID.toString())));
+    update.addDate("IndexedDate", new java.sql.Timestamp(touchTime));
+    update.addDate("ModifiedDate", new java.sql.Timestamp(modTime));
+    try {
+	Statement statement = connection.createStatement();
+	statement.execute(update.toString());
+	statement.close();
+    } catch (SQLException e) {
+	System.err.println("DocumentSQLWriter.touchDocument() Error: "+ e);
+	return false;
+    }
+    System.out.println(update.toString());
+    return true;
+  }
+
+  /**
+   * Just write the document to the database, assuming that it is not already there
+   */
+  public boolean writeDocument(DocumentInterface document, GS3SQLConnection connection)
+  {		
+    // TODO: output document in METS format, with Greenstone-specific metadata
+    // placed in a wrapper inside the descriptive metadata block
+
+    // put the document into the database
+    try {
+      if (document.getID() != null)
+      { //tag = XMLTools.addAttribute(tag, "OBJID", document.getID().toString());
+	GS3SQLSelect select = new GS3SQLSelect("document");
+	select.addField("*");
+	select.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", document.getID().toString())));
+	
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(select.toString());
+	if (!results.first()) { // empty result
+	  GS3SQLInsert insert = new GS3SQLInsert("document");
+	  insert.addValue("DocID", document.getID().toString());
+	  insert.addValue("DocType", document.getDocumentType());
+
+	  insert.addDate("AccessionDate", new java.sql.Timestamp(document.getAccessionDate()));
+	  insert.addDate("IndexedDate", new java.sql.Timestamp(document.getLastIndexedDate()));
+	  insert.addDate("ModifiedDate", new java.sql.Timestamp(document.getModifiedDatestamp()));
+
+	  statement.execute(insert.toString());
+	}
+	else {
+	  /* redundant code - not used... */
+	  //	  GS3SQLUpdate update = new GS3SQLUpdate("document");
+	  //	  update.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", document.getID().toString())));
+	  //	  connection.execute(update.toString());
+	}
+	statement.close();
+      }
+    } catch (SQLException ex) {
+	System.out.println("DocumentSQLWriter.writeDocument() Error:"+ex);
+    }
+    //    output.println(tag);
+    
+    // Write the structural metadata:
+    //   if in "Create METS" mode:
+    //     take all the documents as being in the one group, of the "physical"
+    //     document; particular document plugins may extend this behaviour,
+    //     which would suggest placing it in the AbstractDocument or similar
+    //     class?!
+    if (document.getDocumentMetadata() == null) {
+      System.out.println("Null metadata document");
+    }
+    document.getDocumentMetadata().writeSQL(document, connection);
+
+    document.getDocumentFiles().writeSQL(document, connection);
+
+    document.getDocumentStructure().writeSQL(document, connection);
+
+    // Write the body information
+    // TODO: only output the document if there is some
+    // content/formatting change between the original and
+    // what greenstone actually does; if the original is
+    // good, then place a marker saying "get the document
+    // "from the original file"
+    return true;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java	(revision 14150)
@@ -0,0 +1,65 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.PrintWriter;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+public class DocumentWriter
+{
+    public DocumentWriter()
+    {
+    }
+
+    public boolean writeDocument(DocumentInterface document, PrintWriter output)
+    {		
+	// TODO: output <xml header>
+	String xml_header ="<?xml ";
+	xml_header = XMLTools.addAttribute(xml_header, "version", "1.0");
+	xml_header = XMLTools.addAttribute(xml_header, "encoding", "UTF-8");
+	xml_header = XMLTools.addAttribute(xml_header, "standalone", "no");
+	xml_header = xml_header+"?>";
+	output.println(xml_header);
+
+	// TODO: output document in METS format, with Greenstone-specific metadata
+	// placed in a wrapper inside the descriptive metadata block
+	String tag = XMLTools.getOpenTag("mets", "mets");
+	if (document.getID() != null){
+	    tag = XMLTools.addAttribute(tag,"xmlns:mets","http://www.loc/gov/METS/");
+	    tag = XMLTools.addAttribute(tag,"xmlns:xsi","http://www.w3.org/2001/XMLSchema-instance");
+	    tag = XMLTools.addAttribute(tag,"xmlns:gsdl3","http://www.greenstone.org/gsdlmetadata/1.0/");
+	    tag = XMLTools.addAttribute(tag,"xmlns:xlink","http://www.w3.org/TR/xlink/");
+	    tag = XMLTools.addAttribute(tag,"xsi:schemaLocation","http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/mets.xsd http://www.greenstone.org/gsdlmetadata/1.0/ http://www.greenstone.org/gsdlmetadata/1.0/gsdl_metadata_schema.xsd");
+	    tag = XMLTools.addAttribute(tag, "OBJID", document.getID().toString());
+	}
+	output.println(tag);
+	
+	// Writing the structural metadata:
+	//   if in "Create METS" mode:
+	//     take all the documents as being in the one group, of the "physical"
+	//     document; particular document plugins may extend this behaviour,
+	//     which would suggest placing it in the AbstractDocument or similar
+	//     class?!
+	document.getDocumentMetadata().write(output);
+
+	// TODO: only output the document if there is some
+	// content/formatting change between the original and
+	// what greenstone actually does; if the original is
+	// good, then place a marker saying "get the document
+	// "from the original file"
+	if  (document.isMETSCompatible()){
+	    document.getDocumentFiles().write(output);
+	    document.getDocumentStructure().write(output);
+	}
+	else {
+	    DocumentWriter writer = document.getMETSWriter();
+	    
+	    // ensure that this call isn't recursive - 
+	    // TODO: This is a rather naive check and needs improving
+	    if (writer != this) {
+		writer.writeDocument(document, output);
+	    }
+	}
+	output.write("</mets:mets>\n");
+	return true;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java	(revision 14150)
@@ -0,0 +1,51 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class GMLDocument extends AbstractDocument
+{
+  public static final String GML_DOCUMENT_TYPE = "GML";
+
+  public GMLDocument(DocumentID id)
+  { super(id);
+  }
+
+  public GMLDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return GML_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  public METSDescriptiveSet getDocumentMetadata()
+  { return null;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java	(revision 14150)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+public class GMLRecogniser extends  AbstractRecogniser
+{
+    
+    public GMLRecogniser()   { 
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".gml");
+	this.document_type = GMLDocument.GML_DOCUMENT_TYPE;
+    }
+    
+    
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	if (filename != null) {
+	    if (isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting GML Document " + filename);
+		GMLDocument doc = new GMLDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		// System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java	(revision 14150)
@@ -0,0 +1,35 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class GS2METSRecogniser extends AbstractRecogniser
+{
+    public GS2METSRecogniser() 
+    {
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("docmets.xml");
+	this.document_type = METSDocument.METS_DOCUMENT_TYPE;
+    }
+    
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2PerlRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2PerlRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2PerlRecogniser.java	(revision 14150)
@@ -0,0 +1,136 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+import java.io.File;
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.Processing;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+
+public class GS2PerlRecogniser extends AbstractRecogniser
+{
+    public String plugin_name;
+    public GS2PerlRecogniser()
+    { 
+	//this.preferred_mime_type = "application/pdf";
+	//this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	//this.filename_extensions.add(".pdf");
+	//this.filename_extensions.add(".txt");
+	this.document_type = METSDocument.METS_DOCUMENT_TYPE;
+    }
+
+    public boolean configure(Element config_elem) {
+	String mime_type = config_elem.getAttribute("mimeType");
+	if (mime_type.equals("")) {
+	    System.err.println("GS2PerlRecogniser configure Error: no mimetype specified");
+	    return false;
+	}
+	this.preferred_mime_type = mime_type;
+	String plugin = config_elem.getAttribute("plugin");
+	if (plugin.equals("")) {
+	    System.err.println("GS2PerlRecogniser configure Error: no plugin specified");
+	    return false;
+	}
+	this.plugin_name = plugin;
+	
+	String extensions = config_elem.getAttribute("filenameExtensions");
+	if (extensions.equals("")) {
+	    System.err.println("GS2PerlRecogniser configure Error: no filename extensions specified");
+	    return false;
+	}
+	String [] exts = extensions.split("|");
+	for (int i=0; i<exts.length; i++) {
+	    this.filename_extensions.add(exts[i]);
+	}
+	return true;
+	
+    }
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if ( isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting "+document_type+" Document " + filename);
+		METSDocument doc = processFile(url, filename);
+		
+		if (doc != null) {
+		    this.list_repository.addDocument(doc);
+		    return true;
+		}
+		return false;
+	    }
+	} else {
+	    // Get Mime type remotely, and then proceed if required
+	    String mimeType = HTTPTools.getMIMEType(url);
+	    
+	    if (mimeType == this.preferred_mime_type) {
+		System.out.println("Posting "+document_type+" Document " + url.toString());
+		METSDocument doc = processFile(url, null);
+		if (doc != null) {
+		
+		    this.list_repository.addDocument(doc);
+		    return true;
+		}
+		return false;
+	    }
+	}
+	return false;
+    }
+
+
+    protected METSDocument processFile(URL url, String filename) {
+
+	// we can only do files at this stage
+	if (filename == null) return null;
+	File file = new File(filename);
+	String local_dir = file.getParent();
+	System.err.println("local dir = "+local_dir);
+	
+	//String command = "importone.pl -saveas METS -collectdir /research/kjdon/home/gsdl3/web/sites/localsite/collect -importdir "+local_dir+" -archivedir "+local_dir+" -- kath TEXTPlug "+ file.getName();
+	String command = "importone.pl -saveas METS -collectdir "+GSFile.collectDir(this.coll_manager.siteHome)+" -importdir "+local_dir+" -archivedir "+local_dir+" -- "+this.coll_manager.collectionName +" " +this.plugin_name+" "+ file.getName();
+	System.err.println("Command = "+command);
+	
+	int exit_value = Processing.runProcess(command);
+	if (exit_value == 0) {
+	    System.out.println("converted success");
+	} else {
+	    System.out.println("Unable to convert to mets");
+	    
+	    return null ;
+	}
+	
+	//read in the archives.inf file to see the directory for the new mets
+	File arcinfo = new File(local_dir, "archives.inf");
+	if (!arcinfo.isFile()) {
+	    System.err.println("cant find archives.inf file ("+arcinfo.getPath()+", aborting");
+	    return null;
+	}
+	try {
+	String content = new String(GSFile.readFile(arcinfo.getPath()));
+	content = content.trim();
+	System.err.println("content =  "+content);
+	String items[] = content.split("\\s+");
+	String filepath = items[1];
+	// delete the archive file
+	arcinfo.delete();
+	File mets_file = new File(local_dir, filepath);
+	System.err.println("mets file = "+mets_file.getPath());
+	return new METSDocument(mets_file.toURL());
+	} catch (Exception e) {
+	    System.err.println("processfile: "+e);
+	    return null;
+	}
+    }
+}
+
+    
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 14150)
@@ -0,0 +1,354 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.net.MalformedURLException;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.*;
+
+public class HTMLDocument extends AbstractDocument
+{
+  public static final String HTML_DOCUMENT_TYPE = "HTML";
+  Document domDocument;
+
+  public HTMLDocument(DocumentID id)
+  { super(id);
+    this.domDocument = null;
+  }
+
+  /**
+   *  Create the HTMLDocument from a given URL - the URL may in fact be a reference
+   *  to a local file.
+   *
+   *  @param <code>URL</code> The location from which to load the file
+   */
+  public HTMLDocument(URL url)
+  { super(url);
+
+    this.loadDocument(url);
+
+    this._extractDocumentFiles();
+    this._extractDocumentMetadata();
+    // extract the files before resolving the images
+    resolveImages();
+    HTMLDocumentTools docTools = new HTMLDocumentTools(this.domDocument);
+    docTools.setMetsDocument(this);
+    docTools.setUrl(this.fileSet.getFile(0).getLocation());
+    docTools.findSections();
+    METSStructure sectionStruct = docTools.getStructure();
+
+    if (sectionStruct.size() > 0) {
+      METSStructureSet structureSet = this.getDocumentStructure();
+      structureSet.addStructure(sectionStruct);
+    }
+  }
+
+  private void loadDocument(URL url)
+  {
+    //    HTMLDoc htmlDoc;
+
+    HTMLTidy tidyDoc;
+    if (url.getProtocol().equals("file"))
+    { tidyDoc = new HTMLTidy(new File(url.getPath()));
+    }
+    else
+    { //htmlDoc = new HTMLDoc(url);
+      tidyDoc = new HTMLTidy(url);
+    }
+    
+    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
+
+    this.domDocument = tidyDoc.getDocument(); 
+    Runtime.getRuntime().gc();
+    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
+ }
+
+  private void _extractDocumentMetadata()
+  { 
+    NodeList metadata = this.domDocument.getElementsByTagName("META");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String name = element.getAttribute("name");
+      if (name == null || name.length() == 0) {
+	continue;
+      }
+
+      String value = element.getAttribute("content");
+      if (value == null || value.length() == 0) {
+	value = name;
+      }
+
+      this.addDocumentMetadata(new MetadataLabel(name), value);
+    }
+    
+    NodeList titles = this.domDocument.getElementsByTagName("TITLE");
+    StringBuffer title = new StringBuffer();
+    for (int n = 0; n < titles.getLength(); n ++) {
+      Node node = titles.item(n);
+      Element element = (Element) node;
+
+      NodeList childNodes = node.getChildNodes();
+      for (int c = 0; c < childNodes.getLength(); c ++) {
+	Node child = childNodes.item(c);
+	if (child.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	  title.append(child.getNodeValue());
+	}
+      }
+    }
+
+    if (title.length() > 0)
+    { this.addDocumentMetadata(new MetadataLabel("Title"), title.toString());
+    }
+
+    /* Old HTMLDoc based parsing...
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
+    boolean inTitle = false;
+    title = new StringBuffer();
+
+    for (int e = 0; e < codedContent.size(); e ++)
+    { if (codedContent.elementAt(e) instanceof HTMLTag)
+      {	HTMLTag tag = (HTMLTag) codedContent.elementAt(e);
+
+        if (tag.tagName().equals("meta"))
+	{ // check that the name of the metadata item exists
+	  String name = tag.idValue("name");
+	  if (name == null || name.length() == 0) {
+	    continue;
+	  }
+
+	  // get the value, if it exists
+	  String value = tag.idValue("content");
+
+	  // if value does not exist, default it to being the same
+	  // as the name.
+	  if (value == null || value.length() > 0) {
+	    value = name;
+	  }
+	}
+	else 
+	if (tag.tagName().equals("title"))
+	{ inTitle = true;
+	}
+	else if (tag.tagName().equals("/title"))
+	{ inTitle = false;
+	}
+	// cut off when real body content appears - not a perfect
+        // implementation, just cheap & cheerful
+	else if (tag.tagName().equals("/head"))
+	{ break;
+	}
+	else if (tag.tagName().equals("body"))
+	{ break;
+	}
+      }
+      else if (inTitle == true)
+      { title.append(codedContent.elementAt(e).toString());
+      }
+    }
+      */
+  }
+
+  private void _extractDocumentFiles()
+  { URL homeUrl = this.fileSet.getFile(0).getLocation();
+
+    NodeList metadata = this.domDocument.getElementsByTagName("img");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String location = element.getAttribute("src");
+      if (location == null || location.length() == 0) {
+	System.out.println("No name");
+	continue;
+      }
+      try 
+      { // make the url for the image, and then add it to the document list of
+	// 
+	URL imgUrl = new URL(homeUrl, location);
+	METSFile file = this.fileSet.addFile(imgUrl);
+	this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+      }
+      catch (MalformedURLException ex)
+      { // TODO: report exception/failure to resolve...
+      }
+    }
+
+  
+    /**    
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
+    for (int e = 0; e < codedContent.size(); e ++)
+    { if (codedContent.elementAt(e) instanceof HTMLTag)
+      { HTMLTag tag = (HTMLTag) codedContent.elementAt(e);
+
+        if (tag.tagName().equals("img"))
+	{ String location = tag.idValue("src");
+
+	  try 
+	  { // make the url for the image, and then add it to the document list of
+	    // 
+	    URL imgUrl = new URL(homeUrl, location);
+	    METSFile file = this.fileSet.addFile(imgUrl);
+	    this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+	  }
+	  catch (MalformedURLException ex)
+	  { // TODO: report exception/failure to resolve...
+	  }
+	}
+      }
+    }
+    */
+  }
+
+  public String getDocumentType()
+  { return HTML_DOCUMENT_TYPE;
+  }
+
+    // I think this is used for single section documents, while getSectionText
+    // is used for sectioned documents
+    // we will use the domDocument rather than reading it in again to another HTMLDoc.
+  public String getDocumentText()
+  { 
+      XMLConverter converter = new XMLConverter();
+      return converter.getPrettyString(this.domDocument.getDocumentElement());
+      /*  HTMLDoc htmlDoc;
+    URL     url =(URL) this.fileSet.getFile(0).getLocation();
+
+    this.getSectionText("1");
+
+    if (url.getProtocol().equals("file"))
+    { htmlDoc = new HTMLDoc(url, url.getPath());
+    }
+    else
+    { htmlDoc = new HTMLDoc(url);
+    }
+    return htmlDoc.getContent();
+      */ 
+  }
+
+    private void resolveImages() {
+
+	// find the path of the url relative to the collection
+	URL full_path = this.fileSet.getFile(0).getLocation();
+       
+	String base_url;
+	if (full_path.getProtocol().equals("file")) {
+	    base_url = full_path.getPath();
+	    int import_pos = base_url.indexOf("import");
+	    base_url = base_url.substring(import_pos);
+	    base_url = "_httpcollection_/"+base_url;
+	} else {
+	    base_url = full_path.toString();
+	}
+
+	// need to take off the last part
+	base_url = base_url.substring(0, base_url.lastIndexOf("/")+1);
+
+	NodeList metadata = this.domDocument.getElementsByTagName("img");
+	for (int n = 0; n < metadata.getLength(); n ++) {
+	    Node node = metadata.item(n);
+	    Element element = (Element) node;
+	    
+	    String location = element.getAttribute("src");
+	    if (location != null && location.length() > 0 && isRelative(location)) {
+		// modify the source url
+		element.setAttribute("src", base_url+location);
+	    }
+	}
+    }
+
+    private boolean isRelative(String location) {
+
+	if ( location.startsWith("http:") || location.startsWith("file:")) {
+	    return false;
+	}
+	return true;
+    }
+    
+    public Document getDOMDocument()
+    {    
+      if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+      resolveImages();
+    }
+    return this.domDocument;
+  }
+
+  private XPointer getXPointer(METSDivision division)
+  { String fileId = division.getDefaultFileReference();
+
+    if (fileId == null) {
+      System.err.println("Unable to obtain file reference for section " + division.getLabel());
+      return null;
+    }
+
+    METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+    if (fileGroup == null) {
+      System.err.println("Unable to obtain file reference for filegroup " + fileId);
+      return null;
+    }
+
+    METSFile file = fileGroup.getFile(0);
+    if (file == null) {
+      System.err.println("Unable to obtain any files within filegroup " + fileId);
+      return null;
+    }
+
+    URL url = file.getLocation();
+    XPointer xpointer = XPointer.processXPointer(this.domDocument, url);
+
+    return xpointer;
+  }
+
+  private XPointer getXPointer(String sectionId)
+  { if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+      resolveImages();
+    }
+
+    METSDivision division = 
+      this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					      sectionId);
+    if (division == null) {
+      //      System.err.println("Unable to locate section " + sectionId);
+      return null; 
+    }
+
+    return this.getXPointer(division);
+  }
+  
+  public Node getSectionStartNode(METSDivision division)
+  { XPointer xpointer = this.getXPointer(division);
+    return xpointer.getStartNode();
+  }
+
+  public Node getSectionStartNode(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    return xpointer.getStartNode();
+  }
+
+  public String getSectionText(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    if (xpointer == null) {
+      return "";
+    }
+
+    // get the XML content of the xpointer...
+    return xpointer.toString();
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 14150)
@@ -0,0 +1,328 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.w3c.dom.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.AbstractStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFilePos;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileID;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
+import org.greenstone.gsdl3.gs3build.util.*;
+
+public class HTMLDocumentTools
+{
+  Document          document;
+  URL               documentUrl;
+  int               sectionLevel;
+  StringBuffer      sectionLabel;
+  List              sectionSeq;
+  HTMLSection       rootSection;
+  HTMLSection       currentSection;
+  int               sectionSeqNo;
+  METSStructure     metsStructure;
+  AbstractStructure currentMets;
+  AbstractDocument  metsDocument;
+
+  /**
+   *
+   */
+  class HTMLSection
+  { String startLocation;
+    String endLocation;
+    String label;
+    List   children;
+    HTMLSection parent;
+    int         seqNo;
+
+    public HTMLSection(String label, int seqNo)
+    { this.label = label;
+      this.seqNo = seqNo;
+      this.parent = null;
+      this.children = new ArrayList();
+    }
+    
+    public void setStartLocation(String location)
+    { this.startLocation = location;
+    }
+    
+    public void setEndLocation(String location)
+    { this.endLocation = location;
+    }
+
+    public void addChild(HTMLSection child)
+    { this.children.add(child);
+      child.parent = this;
+    }
+    
+    public HTMLSection getParent()
+    { return this.parent;
+    }
+
+    public String getLabel()
+    { return this.label;
+    }
+
+    public String getXPointer()
+    { StringBuffer xPointerBuffer = new StringBuffer();
+    
+      xPointerBuffer.append("xpointer(");
+      xPointerBuffer.append(this.startLocation);
+      xPointerBuffer.append("/range-to(");
+      xPointerBuffer.append(this.endLocation);
+      xPointerBuffer.append("))");
+      
+      return xPointerBuffer.toString();
+    }
+  }
+
+  public HTMLDocumentTools(Document document)
+  { this.document = document;
+    this.sectionLevel = 0;
+    this.sectionSeq   = new ArrayList();
+    this.sectionSeq.add(new Integer(0));
+    this.sectionLabel = new StringBuffer();
+    this.rootSection = new HTMLSection("", 0);
+    this.currentSection = this.rootSection;
+    this.metsStructure  = new METSStructure(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					    "Section", "Section");
+    this.currentMets    = this.metsStructure;
+    this.metsDocument   = null;
+  }
+
+  public void setMetsDocument(AbstractDocument document)
+  { this.metsDocument = document;
+  }
+
+  public void setUrl(URL url)
+  { this.documentUrl = url;
+  }
+
+  public Object findSections()
+  { this.findSections(document.getDocumentElement(), "/1");
+    return this.rootSection;
+  }
+
+  public void findSections(Node node, String location)
+  { // if at an element, then we aren't going to find section information,
+    // but we do need to recurse into the children to find the section markers
+    // there
+    if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      NodeList children = node.getChildNodes();
+      StringBuffer locationBuffer = new StringBuffer();
+      locationBuffer.append(location);
+      locationBuffer.append("/");
+
+      int elementNo = 1;
+      int commentNo = 1;
+      for (int c = 0; c < children.getLength(); c++)
+      {
+	locationBuffer.setLength(location.length()+1);
+	if (children.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  locationBuffer.append(Integer.toString(elementNo));
+	  elementNo ++;
+	}
+	else if (children.item(c).getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+	  locationBuffer.append("comment()[");
+	  locationBuffer.append(Integer.toString(commentNo));
+	  locationBuffer.append("]");
+	  commentNo ++;
+	}
+	this.findSections(children.item(c), locationBuffer.toString());
+      }
+    }
+    // if at a comment node, then one needs to dissect it to discover if
+    // any section markings are included...
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      String commentText = node.getNodeValue();
+      
+      // parse the node now...checking first for any 'section' items in
+      // it - this is a crude check...
+      if (commentText.indexOf("<Section>") >= 0 ||
+	  commentText.indexOf("</Section>") >= 0) {
+	// parse the section roughly using the primitive HTMLDoc form...      
+	HTMLDoc innerDoc     = HTMLDoc.fromString("dummy", commentText);
+	HTMLBlock innerHtml  = innerDoc.getCodedContent();
+	boolean   inMetadata = false;
+	StringBuffer metaValue = new StringBuffer();
+	String       metaLabel = null;
+
+	// Now just go through the content in sequence...
+	for (int e = 0; e < innerHtml.size(); e ++) {
+	  Object element = innerHtml.elementAt(e);
+
+	  if (element instanceof HTMLTag) {
+	    HTMLTag tag = (HTMLTag) element;
+
+	    if (tag.tagName().equals("section")) {
+	      // get the counters sorted out for the sections...
+	      this.sectionSeq.set(this.sectionLevel, new Integer(((Integer) this.sectionSeq.get(this.sectionLevel)).intValue() + 1));
+	      this.sectionLevel ++;
+	      if (sectionSeq.size() == this.sectionLevel) {
+		this.sectionSeq.add(new Integer(0));
+	      }
+	      else {
+		this.sectionSeq.set(this.sectionLevel, new Integer(0));
+	      }
+
+	      // create the label for the new section...
+	      this.sectionLabel.setLength(0);
+	      for (int s = 0; s < sectionLevel; s ++) {
+		if (s > 0) {
+		  this.sectionLabel.append(".");
+		}
+		this.sectionLabel.append(this.sectionSeq.get(s).toString());
+	      }
+
+	      // note the new section in a new data structure...
+	      HTMLSection newSection = new HTMLSection(this.sectionLabel.toString(), this.sectionSeqNo);
+
+	      newSection.setStartLocation(location);
+	      this.currentSection.addChild(newSection);
+	      this.currentSection = newSection;
+
+	      // do the same for mets...
+	      METSDivision newDivision = new METSDivision("DS" + Integer.toString(this.sectionSeqNo), Integer.toString(this.sectionSeqNo),
+							  this.sectionLabel.toString(), this.sectionLabel.toString(), "Section");
+	      this.currentMets.addDivision(newDivision);
+	      this.currentMets = newDivision;
+
+	      this.sectionSeqNo ++;
+	    }
+	    else if (tag.tagName().equals("/section")) {
+	      // cope with poorly formed metadata...
+	      if (inMetadata) {
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+
+	      // note the end position of the section
+	      this.currentSection.setEndLocation(location);
+
+	      // now post that section to the file list...
+	      this.postFilePtr(this.currentSection);
+
+	      // backtrack up the section tree...
+	      this.currentSection = this.currentSection.getParent();
+	      this.currentMets = this.currentMets.getParent();
+	      this.sectionLevel --;
+
+	      // trim the section label
+	      int cutTo = this.sectionLabel.toString().lastIndexOf('.');
+	      if (cutTo >= 0) {
+		this.sectionLabel.setLength(cutTo);
+	      }
+	      else {
+		this.sectionLabel.setLength(0);
+	      }
+
+	      // we can't be in metadata either...
+	      inMetadata = false;
+	    }
+	    else if (tag.tagName().equals("metadata")) {
+	      if (inMetadata) {
+		// post the metadata
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+	      metaValue.setLength(0);
+	      metaLabel = tag.idValue("name");
+	      inMetadata = (metaLabel != null);
+	    }
+	    else if (tag.tagName().equals("/metadata")) {
+	      // post the metadata
+	      this.postMetadata(metaLabel, metaValue.toString());
+	      inMetadata = false;
+	    }
+	  }
+	  else {
+	    if (inMetadata) {
+	      metaValue.append(element.toString());
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  private String makeXPointer(HTMLSection section)
+  { StringBuffer xPointerBuffer = new StringBuffer();
+
+    if (this.documentUrl != null)
+    { xPointerBuffer.append(this.documentUrl.toString());
+      xPointerBuffer.append("#");
+    }
+    xPointerBuffer.append(section.getXPointer());
+
+    return xPointerBuffer.toString();
+  }
+
+  private void postFilePtr(HTMLSection section)
+  { String xPointer = this.makeXPointer(section);
+
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    try {
+      URL url = new URL(xPointer);
+
+      METSFileSet fileSet = this.metsDocument.getDocumentFiles();
+
+      METSFilePos filePos = new METSFilePos(url);
+      METSFile    file = fileSet.createFile(filePos, "text/html");
+      
+      METSFileGroup fileGroup;
+      String fileRef = ((METSDivision) this.currentMets).getDefaultFileReference();
+
+      if (fileRef == null) {
+	fileGroup = fileSet.createGroup();
+	fileSet.addGroup(fileGroup);
+	((METSDivision) this.currentMets).addFileReference(fileGroup.getId());
+      }
+      else {
+	// simple call - the group should be at the 'root' level, immediately below
+	// the FileSet itself...
+	fileGroup = fileSet.getGroup(fileRef);
+      }
+      fileGroup.addFile(file);
+
+      
+    }
+    catch (java.net.MalformedURLException ex) {
+      System.err.println(ex + " " + xPointer);
+    }
+  }
+
+  private void postMetadata(String label, String value)
+  {
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    String metadataRef = ((METSDivision) this.currentMets).getDefaultMetadataReference();
+    METSDescriptive descriptive;
+    if (metadataRef == null) {
+      descriptive = this.metsDocument.getDocumentMetadata().createDescriptive(this.currentSection.getLabel());
+      ((METSDivision) this.currentMets).addMetadataReference(descriptive.getID());
+    }
+    else {
+      descriptive = this.metsDocument.getDocumentMetadata().getDescriptiveById(metadataRef);
+    }
+    descriptive.addMetadata(MetadataLabel.getDefaultNamespace(), label, value);
+  }
+
+  public METSStructure getStructure()
+  { return this.metsStructure;
+  }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java	(revision 14150)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class HTMLRecogniser extends AbstractRecogniser
+{
+    
+    public HTMLRecogniser()
+    { 
+	this.preferred_mime_type = "text/html";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".htm");
+	this.filename_extensions.add(".html");
+	this.document_type = HTMLDocument.HTML_DOCUMENT_TYPE;
+	
+    }
+    
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java	(revision 14150)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class IndexDocument extends AbstractDocument
+{
+  public static final String INDEX_DOCUMENT_TYPE = "GSINDEX";
+
+  public IndexDocument(DocumentID id)
+  { super(id);
+  }
+
+  public IndexDocument(URL url)
+  { super(url);
+  }
+  
+  public String getDocumentType()
+  { return INDEX_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  {
+    return true;
+  }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java	(revision 14150)
@@ -0,0 +1,52 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+public class IndexRecogniser extends AbstractRecogniser
+{
+    
+    public IndexRecogniser(DocumentList listRepository)
+    { 
+	this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("index.txt");
+	this.document_type = IndexDocument.INDEX_DOCUMENT_TYPE;
+    }
+    
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    String leafName;
+	    int leafAt = filename.lastIndexOf(File.separator);
+	    if (leafAt >= 0) {
+		leafName = filename.substring(leafAt+1);
+	    } else {
+		leafName = filename;
+	    }
+
+	    if (leafName.equals("index.txt")) {
+		
+		System.out.println("Posting Index Document " + filename);
+		IndexDocument doc = new IndexDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		//				System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java	(revision 14150)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+
+/**
+ *  Another fairly basic class for representing JPEG documents.
+ */
+
+public class JPEGDocument extends AbstractDocument
+{
+  public static final String JPEG_DOCUMENT_TYPE = "JPEG";
+
+  public JPEGDocument(DocumentID id)
+  { super(id);
+  }
+  
+  public JPEGDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return JPEG_DOCUMENT_TYPE;
+  }
+
+  public String getDocumentText()
+  { // Just a dummy function for JPEGs
+    return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { // Just a dummy function for JPEGs
+    return "";
+  }
+
+  public boolean isMETSCompatible()
+  { return true;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java	(revision 14150)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class JPEGRecogniser extends AbstractRecogniser
+{
+    
+    public JPEGRecogniser()
+    { 
+	this.preferred_mime_type = "image/jpeg";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".jpg");
+	this.filename_extensions.add(".jpeg");
+	this.document_type = JPEGDocument.JPEG_DOCUMENT_TYPE;
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 14150)
@@ -0,0 +1,260 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+
+import java.net.URL;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTMLDoc;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+
+public class METSDocument extends AbstractDocument {
+
+    public static final String METS_DOCUMENT_TYPE = "METS";
+    Document domDocument;
+    String original_location;
+    
+    public METSDocument(DocumentID id) {
+    	super(id);
+	//domDocument = null;
+    }
+
+    public METSDocument(URL url) {
+	super(url);
+	
+	if (url.getProtocol().equals("file")) {
+	    this._parseFile(new File(url.getPath()));
+	} else {
+	    System.err.println("Error: METSDocument must currently be a file");
+	    System.err.println("       URL = " + url.getPath());
+	}
+    }
+    
+    private void _parseFile(File file) {
+	
+	String parseFilePath;
+	try {
+	    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	    DocumentBuilder builder = factory.newDocumentBuilder();
+	    domDocument = builder.parse(file);
+
+	    //int filePosition = file.getPath().indexOf("import/")+7;
+	    //parseFilePath = file.getPath().substring(0, filePosition);
+	    parseFilePath = file.getParent(); // all refs should be relative to the current doc, or absolute
+	    parseFilePath += File.separator;
+	    this.original_location = parseFilePath;
+	    // TODO: get all the types in the tree
+      
+	    // TODO: do a traverse, and thus cope with elements-within-elements if needsbe, but
+	    // this shouldn't happen except in a directly defined situation - actually doing the
+	    // parsing in part inside each node would work well provided one checked for a node
+	    // having already been done...
+	    System.out.println("Read");
+      
+	    // Get METSDmdSec metadata sections
+	    NodeList dmdSecs = domDocument.getElementsByTagName("mets:dmdSec");
+	    
+	    this.metadata = METSDescriptiveSet.parseXML(dmdSecs);
+
+	    // Get METSFileSec sections
+	    NodeList fileSecs = domDocument.getElementsByTagName("mets:fileSec");
+	    this.fileSet = METSFileSet.parseXML(fileSecs, fileSet, parseFilePath);
+	    // Get METSStructureMap Sections
+	    NodeList structMapSecs = domDocument.getElementsByTagName("mets:structMap");
+	    this.structureSet = METSStructureSet.parseXML(structMapSecs);
+
+	    //for (int g = 0; g < fileSecs.getLength(); g ++) {
+		//Schema schema = new Schema(schemas.item(s));
+		//this._parseFileSec(fileSecs.item(g));
+	    //}
+      
+	    // Get document metadata sections
+	    //NodeList dmdSecs = document.getElementsByTagName("mets:dmdSec");
+      
+	    //Schema schema = new Schema(schemas.item(s));
+	    //this.metadata = METSDescriptiveSet.parseXML(fileSecs);
+	} catch (FactoryConfigurationError e) {
+	    System.out.println(e);
+	} catch (ParserConfigurationException ex) {
+	    System.out.println(ex);
+	} catch (SAXException ex) {
+	    System.out.println(ex);
+	} catch (IOException ex) {
+	    System.out.println(ex);
+	}
+    }
+    /*
+    private void _parseFileSec(Node fileSec) {
+	// this is in effect a group without a sense of 'self'...
+	this._parseFileGroup((Element) fileSec, null);
+    }
+
+    private void _parseFileGroup(Element groupTag, METSFileGroup group) {
+	NodeList children = groupTag.getChildNodes();
+
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+
+	    System.out.println(children.item(c));
+    
+	    Element element = (Element) children.item(c);
+
+	    if (element.getNodeName().equals("mets:File")) {
+		if (group != null) {
+		    METSFile file = METSFile.parseXML(element, group);
+		} else {
+		    // TODO: error
+		}
+	    } else if (element.getNodeName().equals("mets:fileGrp")) {
+		// recurse
+		METSFileGroup childGroup = new METSFileGroup(element.getAttribute("ID"));
+
+		this._parseFileGroup(element, childGroup);
+		if (group != null) {
+		    group.addGroup(childGroup);
+		} else { 
+		    this.fileSet.addGroup(childGroup);
+		}
+	    }
+	}
+	}*/
+	
+    public String getDocumentType() {
+	return METS_DOCUMENT_TYPE;
+    }
+	
+    public String getDocumentText() {
+	// TODO: make this more than a dummy function!
+	HTMLDoc metsDoc;
+	URL url = (URL) this.fileSet.getFile(0).getLocation();
+
+	//this.getSectionText("1");
+	
+	if (url.getProtocol().equals("file")) {
+	    String path = url.getPath();
+	    File f = new File(path);
+	    if (!f.isAbsolute() && this.original_location != null) {
+		path = this.original_location + path;
+		System.err.println("new path ="+path);
+	    }
+	    metsDoc = new HTMLDoc(url,path);
+	} else {
+	    metsDoc = new HTMLDoc(url);
+	}
+	return metsDoc.getContent();
+    }
+    public Document getDOMDocument() 
+    {
+	if (domDocument == null) {
+	    URL url = (URL) this.fileSet.getFile(0).getLocation();
+	    try {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		domDocument = builder.parse(new File(url.getPath()));
+	    } catch (FactoryConfigurationError e) {
+		System.out.println(e);
+	    } catch (ParserConfigurationException ex) {
+		System.out.println(ex);
+	    } catch (SAXException ex) {
+		System.out.println(ex);
+	    } catch (IOException ex) {
+		System.out.println(ex);
+	    }
+	}
+	return domDocument;
+    }
+
+    private XPointer getXPointer(METSDivision division)
+    {
+	String fileId = division.getDefaultFileReference();
+
+	if (fileId == null) {
+	    System.err.println("Unable to obtain file reference for section " + division.getLabel());
+	    return null;
+	}
+	
+	METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+	if (fileGroup == null) {
+	    System.err.println("Unable to obtain file reference for filegroup " + fileId);
+	    return null;
+	}
+	METSFile file = fileGroup.getFile(0);
+	if (file == null) {
+	    System.err.println("Unable to obtain any files within filegroup " + fileId);
+	    return null;
+	}
+	URL url = file.getLocation();
+	XPointer xpointer = XPointer.processXPointer(domDocument, url);
+	
+	return xpointer;
+    }
+    
+    private XPointer getXPointer(String sectionId)
+    {
+	if (domDocument == null) {
+	    URL url = (URL) this.fileSet.getFile(0).getLocation();
+	    try {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		domDocument = builder.parse(new File(url.getPath()));
+	    } catch (FactoryConfigurationError e) {
+		System.out.println(e);
+	    } catch (ParserConfigurationException ex) {
+		System.out.println(ex);
+	    } catch (SAXException ex) {
+		System.out.println(ex);
+	    } catch (IOException ex) {
+		System.out.println(ex);       
+	    }
+	}
+	
+	METSDivision division = 
+	    this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, sectionId);
+	
+	if(division == null) {
+	    return null;
+	}
+      	return this.getXPointer(division);
+    }
+
+    public Node getSectionStartNode(METSDivision division)
+    {
+	XPointer xpointer = this.getXPointer(division);
+	return xpointer.getStartNode();
+    }
+
+    public Node getSectionStartNode(String sectionId)
+    {
+	XPointer xpointer = this.getXPointer(sectionId);
+	return xpointer.getStartNode();
+    }
+    
+    public String getSectionText(String sectionId) 
+    {
+	XPointer xpointer = this.getXPointer(sectionId);
+	
+	if (xpointer == null) {
+	    return "";
+	}
+
+	//get the XML content of the xpointer
+	return xpointer.toString();
+    }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java	(revision 14150)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class MetadataDocument extends AbstractDocument
+{
+  public static final String METADATA_DOCUMENT_TYPE = "GSMETADATA";
+
+  public MetadataDocument(DocumentID id)
+  { super(id);
+  }
+
+  public MetadataDocument(URL url)
+  { super(url);
+  }
+  
+  public String getDocumentType()
+  { return METADATA_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  {
+    return false;
+  }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java	(revision 14150)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+//import java.io.*;
+import java.net.URL;
+import java.util.ArrayList;
+
+
+public class MetadataRecogniser extends AbstractRecogniser
+{
+
+    public MetadataRecogniser()
+    { 
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("metadata.xml");
+	this.document_type = MetadataDocument.METADATA_DOCUMENT_TYPE;
+    }
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if (isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting Metadata Document " + filename);
+		MetadataDocument doc = new MetadataDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		// System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 14150)
@@ -0,0 +1,21 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+/** Interface for Recogniser Objects
+ *
+ * A skeleton class is provided for simple Recognisers (AbstractRecogniser)
+ * 
+*/
+public interface RecogniserInterface
+{
+    public void setCollectionManager(CollectionManager coll_man); 
+    public boolean configure(Element config_elem);
+    public void setListRepository(DocumentList docList);
+    public boolean parseDocument(METSFile fileReference);
+    public boolean parseDocument(URL fileReference);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java	(revision 14150)
@@ -0,0 +1,89 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.greenstone.gsdl3.gs3build.FileCrawlObserver;
+
+import java.io.File;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+public class RecogniserManager implements FileCrawlObserver
+{
+  RecogniserInterface list[];
+  int                 used;
+  int                 size;
+    DocumentList docList;
+  public RecogniserManager(DocumentList documentList)
+  { this.list = new RecogniserInterface[10];
+    this.used = 0;
+    this.size = 10;
+    this.docList = documentList;
+  }
+
+  /**
+   *  Add a recogniser to this manager
+   */
+    public RecogniserInterface addRecogniser(String documentType)
+    { 
+	ClassLoader loader = ClassLoader.getSystemClassLoader();
+	java.lang.Class thisClass = null;
+	try {
+	    thisClass = loader.loadClass("org.greenstone.gsdl3.gs3build.doctypes." + documentType + "Recogniser");
+	    Object instance = thisClass.newInstance();
+	    this.addRecogniser((RecogniserInterface) instance);
+	    return (RecogniserInterface)instance;
+	}
+	catch (ClassNotFoundException ex) {
+	    System.err.println(ex);
+	    return null;
+	}
+	catch (InstantiationException instEx) {
+	    System.err.println(instEx);
+	}
+	catch (IllegalAccessException accEx) {
+	    System.err.println(accEx);
+	}
+	return null;
+    }
+    
+  /**
+   *
+   */
+    public void addRecogniser(RecogniserInterface recogniser)
+    {  
+	recogniser.setListRepository(this.docList);
+	this.ensureCapacity(this.used + 1);
+  
+	this.list[this.used] = recogniser;
+	this.used ++;
+    }
+
+  public void processFile(URL url)
+  { boolean result;
+
+    for (int r = 0; r < this.used; r ++)
+    { if (list[r].parseDocument(url)) {
+        break;
+      }
+    }
+  }
+ 
+  public void processFile(File file)
+  { 
+      System.err.println("processing file "+file.getPath());
+      URL url = URLTools.getFileURL(file);
+    
+    if (url != null) 
+    { this.processFile(url);
+    }
+  }
+
+  private void ensureCapacity(int size)
+  { while (size >= this.size)
+    { RecogniserInterface newList [] = new RecogniserInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java	(revision 14150)
@@ -0,0 +1,45 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *  A pretty basic DocumentInterface class - just able to
+ *  provide unsophisticated access to the document as plain
+ *  text, and no attempt at metadata extraction
+ */
+
+public class TextDocument extends AbstractDocument
+{
+  public static final String TEXT_DOCUMENT_TYPE = "Text";
+
+  public TextDocument(DocumentID id)
+  { super(id);
+  }
+
+  public TextDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return TEXT_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return DocumentLoader.getAsString((URL) this.fileSet.getFile(0).getLocation());
+  }
+
+  /**
+   *  Text documents don't actually have more than one section (at present), so
+   *  this is a dummy function...
+   */
+  public String getSectionText(String sectionId)
+  { return null;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java	(revision 14150)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+
+public class TextRecogniser extends AbstractRecogniser
+{
+    
+    public TextRecogniser()
+    { 
+	this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".txt");
+	this.filename_extensions.add(".text");
+	this.document_type = TextDocument.TEXT_DOCUMENT_TYPE;
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java	(revision 14150)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.expander;
+
+import java.io.File;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public interface ExpanderInterface
+{
+	/**
+	 *  Record the build manager to use for this expander - this is used
+	 *  to pass back any new directories into which expanded files are placed.
+	 */
+	public void configure(BuildManager manager);
+
+	/**
+	 *  Attempt to expand a file.  The returned result is <code>true</code> if the
+	 *  file was actually expanded.
+	 *
+	 *  @param <code>URL</code> the name of the file, or its URL
+	 *  @param <code>File</code> the directory to expand files into - the expander
+	 *         should create its own sub-directory to ensure that no later conflicts
+	 *         emerge.
+	 *
+	 *  @return <code>boolean</code> whether the file was expanded.
+	 */
+	public boolean expandFile(URL fileReference, File expansionDirectory);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java	(revision 14150)
@@ -0,0 +1,81 @@
+package org.greenstone.gsdl3.gs3build.expander;
+
+import java.io.*;
+import java.net.URL;
+import java.util.*;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public class ZipExpander implements ExpanderInterface
+{ BuildManager buildManager;
+
+	public ZipExpander()
+	{
+	}
+
+	public void configure(BuildManager manager)
+	{	this.buildManager = manager;
+	}
+
+	public boolean expandFile(URL url, java.io.File expansionDirectory)
+	{ if (url.toString().startsWith("file://")) 
+		{	String fileName = url.toString().substring(7);
+
+			if (fileName.endsWith(".zip"))
+		  {	System.out.println("Expanding Zip file " + fileName);
+
+				// determine which directory to use
+
+				// get the leaf name
+				String leafName;
+				int leafAt = fileName.lastIndexOf(System.getProperty("file.separator"));
+				
+				if (leafAt >= 0)
+				{ leafName = fileName.substring(leafAt+1);
+				}
+				else
+				{	leafName = fileName;
+				}
+
+			  int extensionAt = leafName.lastIndexOf('.');
+				if (extensionAt >= 0)
+				{	leafName = leafName.substring(0, extensionAt);
+				}
+
+				File targetDir = new File(expansionDirectory, leafName);
+				if (targetDir.exists())
+				{	// find alternative name
+					int  extension = 1;
+					File expandedTarget;
+
+					do
+					{ expandedTarget = new File(targetDir.toString() + Integer.toString(extension));
+					} while (expandedTarget.exists());
+
+					targetDir = expandedTarget;
+				}
+				if (!targetDir.mkdir())
+				{	// TODO: pass/log error - unable to make directory to
+					//       expand file.
+					return false;
+				}
+
+				// Call zip expansion
+				try
+				{
+					Runtime.getRuntime().exec("unzip -d "+targetDir.toString());
+				}
+				catch (IOException io)
+				{ // TODO: handle errors - report to log, etc.
+					return false;
+				}
+
+				// ensure that the build manager knows about the new directory,
+				// and will expand any content in it, etc. in turn
+				this.buildManager.addInputDirectory(targetDir.toString());
+				return true;
+			}
+		}
+		return false;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java	(revision 14150)
@@ -0,0 +1,15 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public interface ExtractorInterface
+{
+  public void configure(String outputDir);
+  public void configure(DocumentList list);
+  public void startPass(int passNumber);
+  public void extractDocument(DocumentID documentID, DocumentInterface document);
+  public void endPass(int passNumber);
+  public int getNumberOfPasses();
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java	(revision 14150)
@@ -0,0 +1,62 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.util.Iterator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public class ExtractorManager
+{
+  public static final String ACCUMULATE_MODE = "accumulate";
+
+  DocumentList          documents;
+  ExtractorInterface [] list;
+  int                   size;
+  int                   used;
+
+  public ExtractorManager(DocumentList documentList)
+  { this.list = new ExtractorInterface[10];
+    this.size = 10;
+    this.used = 0;
+    this.documents = documentList;
+  }
+
+  public void addExtractor(ExtractorInterface extractor)
+  { this.ensureSize(this.used + 1);
+    this.list[this.used] = extractor;
+    this.used ++;
+
+    extractor.configure(this.documents);
+  }
+
+  public void extractDocument(DocumentID docId, DocumentInterface document)
+  { for (int i = 0; i < this.used; i ++)
+    { this.list[i].extractDocument(docId, document);
+    }
+  }
+
+  public void extractDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.list[i].getNumberOfPasses(); p ++)
+      {	this.list[i].startPass(p);
+
+        Iterator iterator = documents.iterator();
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+	  this.list[i].extractDocument(document.getID(), document);
+	}
+	this.list[i].endPass(p);
+      }
+    }
+  }
+
+  public void ensureSize(int size)
+  { while (size >= this.size)
+    { ExtractorInterface newList [] = new ExtractorInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java	(revision 14150)
@@ -0,0 +1,167 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.doctypes.GMLDocument;
+
+public class GMLExtractor implements ExtractorInterface
+{
+  public static final String ACCUMULATE_MODE = "accumulate";
+
+  /**
+   *  An inner class to handle GML files
+   */
+  class GMLHandler extends DefaultHandler
+  { List   files;
+    String label;
+    StringBuffer value;
+    boolean inElement;
+    boolean accumulate;
+
+    GMLHandler()
+    { super();
+    
+      this.label = null;
+      this.value = null;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("FileName"))
+      {	this.value = new StringBuffer();
+      }
+      else if (localName.equals("FileSet"))
+      { this.files = new ArrayList();
+      }
+      else if (localName.equals("Description"))
+      {
+      }
+      else if (localName.equals("Metadata"))
+      { this.label = attributes.getValue("name");
+        this.value = new StringBuffer();
+
+	String mode = attributes.getValue("mode");
+	this.accumulate = mode.equals(ExtractorManager.ACCUMULATE_MODE);	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("FileName"))
+      {	String file = this.value.toString();
+        this.value = null;
+	this.files.add(file);
+      }
+      else if (localName.equals("FileSet"))
+      { // post the existing files item...
+      }
+      else if (localName.equals("Description"))
+      {
+      }
+      else if (localName.equals("Metadata"))
+      { GMLExtractor.postMetadata(this.files, this.label, this.value.toString());
+        this.value = null;
+	this.label = null;
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { if (this.value != null)
+      { String string = new String(c, start, length);
+        this.value.append(string);
+      }
+    }
+  }
+
+  /**
+   *  Construct of extractor
+   */
+  public GMLExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList documentList)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a GML document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  {	
+    if (document.getDocumentType().equals(GMLDocument.GML_DOCUMENT_TYPE))
+    { // Extract the content from the GML file
+      try {
+	XMLReader reader = XMLReaderFactory.createXMLReader();
+	GMLHandler handler = new GMLHandler();
+	reader.setContentHandler(handler);
+	reader.setErrorHandler(handler);
+	  
+	// A GML document consists of one file only - get it from the 'default'
+	// file group
+	FileReader fileReader = new FileReader(document.getDocumentFiles().getFile(0).toString());
+	reader.parse(new InputSource(fileReader));
+      }
+      catch (SAXException saxException)
+      { // TODO: log error
+      }
+      catch (java.io.FileNotFoundException fileException)
+      {
+      }
+      catch (java.io.IOException ioException)
+      {
+      }
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(List files, String value, String label)
+  {
+    
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java	(revision 14150)
@@ -0,0 +1,256 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.IndexDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+public class IndexExtractor implements ExtractorInterface
+{
+  class IndexHandlerException extends Exception
+  { public IndexHandlerException(String value)
+    { super(value);
+    }
+  }
+
+  /**
+   *  An inner class to handle GML files
+   */
+  class IndexHandler extends GS2TextFileHandler
+  { List    labels;
+    URL     base;
+
+    IndexHandler(String content, URL url, DocumentList documentList) throws IndexHandlerException
+    { 
+      super(content);
+
+      this.labels  = new ArrayList();
+      this.base    = url;
+
+      String parentDir;
+      int leaf = this.base.toString().lastIndexOf('/');
+      if (leaf >= 0) {
+	parentDir = this.base.toString().substring(0, leaf+1);
+      }
+      else {
+	parentDir = this.base.toString();
+      }
+
+      // get the first line
+      this.getLine();
+
+      if (!this.hasMore())
+      { throw new IndexHandlerException("No title line");
+      }
+      
+      // get the first totem - it should be "key:"
+      String entry = this.getEntry(true);
+      
+      // now get all the labels
+      while (this.hasMore())
+      { String label = this.getEntry(true);
+        if (label == null || label.length() == 0) {
+	  continue;
+	}
+	
+	this.labels.add(label);
+	System.out.println("Adding label: " + label);
+      }
+
+      while (this.hasMoreLines()) {
+	this.getLine();
+
+	// Get the file pattern itself
+	String filePattern = this.getEntry(true);
+	if (filePattern == null || filePattern.length() == 0) {
+	  continue;
+	}
+
+	// get a list of documents that match the file pattern
+	List documentIds = documentList.findDocumentIdsUsingFile(filePattern);
+	if (documentIds != null) {
+	  Iterator iterator = documentIds.iterator();
+	  while (iterator.hasNext()) {
+	    System.out.println("Matches file " + iterator.next().toString());
+	  }
+	}
+
+	// if no files match this data, then skip this row
+	// TODO: raise a quality error message
+	if (documentIds == null || documentIds.size() == 0) {
+	  continue;
+	}
+
+	// cache up the documents that match for speed improvements...
+	List documents = new ArrayList();
+	Iterator idIterator = documentIds.iterator();
+	while (idIterator.hasNext()) {
+	  String docIdString = idIterator.next().toString();
+	  System.out.println(docIdString);
+	  DocumentID docId   = new DocumentID(docIdString);
+	  DocumentInterface document = documentList.getDocument(docId);
+	  if (document != null) {
+	    documents.add(document);
+	  }
+	}
+
+	// Next, split the row into the separate metadata items
+	int entryNo = 0;
+	while (this.hasMore()) {
+	  String item = this.getEntry(true);
+	  if (item == null || item.length() == 0) {
+	    entryNo ++;
+	    continue;
+	  }
+
+	  String label = null;
+	  if (item.startsWith("<")) {
+	    int labelEnd = item.indexOf('>');
+	    if (labelEnd >= 0) {
+	      label = item.substring(1, labelEnd);
+
+	      item = item.substring(labelEnd+1, item.length());
+
+	      // eliminate any weird whitespace
+	      item.trim();
+
+	      // cope with a solo 'item' label with no following string
+	      if (item.length() == 0) {
+		entryNo ++;
+		continue;
+	      }
+	    }
+	    // starts with a bracketed label
+	  }
+	  else if (entryNo < this.labels.size()) {
+	    label = (String) this.labels.get(entryNo);
+	  }
+
+	  // Actually post the metadata -
+	  // it may be good to have cached all the documents that we're going to change
+	  // in order to minimise rewrites...
+	  if (label != null) {
+	    Iterator docIterator = documents.iterator();
+	    while (docIterator.hasNext()) {
+	      DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	      // Post to document
+	      // TODO: tailor this to posting documents to *sections* as required...
+	      document.addDocumentMetadata(new MetadataLabel(label), item);
+	      System.out.println("Posting " + label + "=" + item + " to " + parentDir + filePattern);
+	    }
+	  }
+	  entryNo ++;
+	}
+	
+	// write out the modified documents
+	// TODO: nicer/more generalised interface for this and related activity in
+	//       extractor manager (actually, enricher manager);
+	Iterator docIterator = documents.iterator();
+	while (docIterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	  System.out.println("Writing modified document " + document.getID());
+	  documentList.storeChangedDocument(document);
+	}
+      }
+    }
+
+  }
+
+  private DocumentList documentList;
+
+  /**
+   *  Construct of extractor
+   */
+  public IndexExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList list)
+  { this.documentList = list;
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a GML document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  { if (document.getDocumentType().equals(IndexDocument.INDEX_DOCUMENT_TYPE))
+    { // Extract the content from the index file
+      
+      // get the file
+      String documentText = 
+	DocumentLoader.getAsString(document.getDocumentFiles().getFile(0).getURL());
+	
+      if (documentText == null) {
+	System.err.println("IndexExtractor: Unable to load any content for " + document.getDocumentFiles().getFile(0).toString());
+	return;
+      }
+
+      try {
+	IndexHandler handler = new IndexHandler(documentText, document.getDocumentFiles().getFile(0).getURL(), this.documentList);
+      }
+      catch (IndexHandlerException ex) {
+      }
+      
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(String file, String value, String label)
+  {
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java	(revision 14150)
@@ -0,0 +1,268 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.apache.xerces.parsers.SAXParser;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.MetadataDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+public class MetaXMLExtractor implements ExtractorInterface
+{
+  /**
+   *  An inner class to handle Metadata files
+   */
+  class MetadataHandler extends DefaultHandler
+  { List         files;
+    String       label;
+    StringBuffer value;
+    URL          url;
+    boolean      inElement;
+    boolean      accumulate;
+    DocumentList documentList;
+    List         documentIds;
+    List         documents;
+
+    MetadataHandler(DocumentList documentList)
+    { super();
+    
+      this.label = null;
+      this.value = null;
+      this.documentList = documentList;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("FileName"))
+      {	this.value = new StringBuffer();
+      }
+      else if (localName.equals("FileSet"))
+      { this.files = new ArrayList();
+      }
+      else if (localName.equals("Description"))
+      { this.documentIds = this.documentList.findDocumentIdsUsingFiles(this.files, this.url.toString());	
+
+	if (documentIds != null && documentIds.size() > 0) {
+	  this.documents = new ArrayList();
+	  
+	  Iterator idIterator = documentIds.iterator();
+	  while (idIterator.hasNext()) {
+	    String docIdString = idIterator.next().toString();
+	    DocumentID docId   = new DocumentID(docIdString);
+	    DocumentInterface document = documentList.getDocument(docId);
+	    if (document != null) {
+	      documents.add(document);
+	    }
+	  }
+	}
+      }
+      else if (localName.equals("Metadata"))
+      { this.label = attributes.getValue("name");
+        this.value = new StringBuffer();
+
+	String mode = attributes.getValue("mode");
+	this.accumulate = mode.equals(ExtractorManager.ACCUMULATE_MODE);	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("FileName"))
+      {	String file = this.value.toString();
+        this.value = null;
+	this.files.add(file);
+      }
+      else if (localName.equals("FileSet"))
+      { // post the existing files item...
+      }
+      else if (localName.equals("Description"))
+      {
+	if (this.documents != null && documents.size() > 0) {
+	  // write out the modified documents
+	  // TODO: nicer/more generalised interface for this and related activity in
+	  //       extractor manager (actually, enricher manager);
+	  Iterator docIterator = documents.iterator();
+	  while (docIterator.hasNext()) {
+	    DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	    //  System.out.println("Writing modified document " + document.getID());
+	    documentList.storeChangedDocument(document);
+	  }
+	}
+      }
+      else if (localName.equals("Metadata"))
+      { MetaXMLExtractor.postMetadata(this.url, this.files, 
+				      this.label, this.value.toString(), 
+				      this.accumulate);
+	/*
+	if (documentIds != null) {
+	  Iterator iterator = documentIds.iterator();
+	  while (iterator.hasNext()) {
+	    System.out.println("Matches file " + iterator.next().toString());
+	  }
+	}
+	*/
+	
+	if (documentIds != null && documentIds.size() > 0) {
+	  Iterator docIterator = this.documents.iterator();
+	  while (docIterator.hasNext()) {
+	    DocumentInterface document = (DocumentInterface) docIterator.next();
+	    
+	    // Post to document
+	    // TODO: tailor this to posting documents to *sections* as required...
+	    if (accumulate) {
+ 	      document.addDocumentMetadata(new MetadataLabel(this.label), this.value.toString());
+	    }
+	    else {
+ 	      document.setDocumentMetadata(new MetadataLabel(this.label), this.value.toString());
+	    }
+	  }	
+	}
+
+	// flatten the metadata items again...
+        this.value = null;
+	this.label = null;
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { if (this.value != null)
+      { String string = new String(c, start, length);
+        this.value.append(string);
+      }
+    }
+
+    public void setUrl(URL url)
+    { this.url = url;
+    }
+  }
+
+  private DocumentList documentList;
+
+  /**
+   *  Construct of extractor
+   */
+  public MetaXMLExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList list)
+  { this.documentList = list;
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a metadata document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  { if (document.getDocumentType().equals(MetadataDocument.METADATA_DOCUMENT_TYPE))
+    { // Extract the content from the metadata file
+      URL url;
+
+      try {
+	SAXParser parser = new SAXParser();
+	MetadataHandler handler = new MetadataHandler(this.documentList);
+        /*
+	XMLReader reader = XMLReaderFactory.createXMLReader();
+	reader.setContentHandler(handler);
+	reader.setErrorHandler(handler);*/
+	parser.setContentHandler(handler);
+	  
+	// Get path of file; we cheat here by assuming that the url is a file - this
+	// really ought to be done better [TODO: fix to handle full paths & URLs]
+	url = document.getDocumentFiles().getFile(0).getURL();
+	String filePath = url.getPath();
+	handler.setUrl(new URL(url, "."));
+
+	// A metadata document consists of one file only - get it from the 'default'
+	// file group
+	/*
+	FileReader fileReader = new FileReader(filePath);
+	reader.parse(new InputSource(fileReader));
+	*/
+	parser.parse(filePath);
+      }
+      catch (SAXException saxException)
+      { // TODO: log error
+	System.err.println(saxException);
+      }
+      catch (java.io.FileNotFoundException fileException)
+      { System.err.println(fileException);
+      }
+      catch (java.io.IOException ioException)
+      { System.err.println(ioException);
+      }
+      /*      catch (java.net.MalformedURLException malEx) {
+	System.err.println("Unable to get parent of URL "+url.toString()+" in metadata extraction.");
+	return;
+      }
+      */
+
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(URL url, List files, String label, String value, boolean accumulate)
+  { String file;
+
+    Iterator fileIter = files.iterator();
+    while (fileIter.hasNext()) {
+      file = fileIter.next().toString();
+
+      System.out.println(url.toString() + " " + file + ": " + label + "=" + value);
+    }
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java	(revision 14150)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.util.GSXML;
+
+public abstract class AbstractIndexer implements IndexerInterface
+{ 
+  class CollectionIndex
+  { String level;
+    String name;
+    List fields;
+
+    public CollectionIndex(String indexName, Node node)
+    { this.fields = new ArrayList();
+      this.name = indexName;
+      this.level = IndexerManager.DEFAULT_LEVEL;
+    
+      NodeList children = node.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++) {
+	Node child = children.item(c);
+	
+	if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  String name = child.getNodeName();
+	  
+	  if (name.equals(GSXML.LEVEL_ELEM)) {
+	    String levelName = DOMUtils.getNodeChildText(children.item(c));
+	    this.level = levelName;
+	  }
+	  else if (name.equals(GSXML.FIELD_ELEM)) {
+	    String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	    this.fields.add(fieldName.toString());
+	  }
+	}
+      }
+      if (this.fields.size() == 0) {
+	this.fields.add(IndexerManager.DEFAULT_FIELD);
+      }
+    }
+
+    public void prepareIndexer(IndexerInterface indexer)
+      { //indexer.addIndex(this.name, this.level, this.fields.get(0).toString());
+    }
+  }
+
+
+  public boolean configure(org.w3c.dom.Node searchNode)
+  {
+    NodeList indexChildren = searchNode.getChildNodes();
+
+    // check for <index> tag children
+    for (int i = 0; i < indexChildren.getLength(); i ++) {
+      String childName = indexChildren.item(i).getNodeName();
+
+      if (childName.equals(GSXML.INDEX_ELEM))
+      { NamedNodeMap atts = indexChildren.item(i).getAttributes();
+	
+	Node attribute = atts.getNamedItem(GSXML.NAME_ATT);
+	String indexName = attribute.getNodeValue();
+	      
+	// create the local indexer representation
+	CollectionIndex index = new CollectionIndex(indexName, indexChildren.item(i));
+	     
+	// prepare it...
+	index.prepareIndexer(this);
+      }
+    }
+    return true;
+  }
+
+  public void tidyup()
+  {
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java	(revision 14150)
@@ -0,0 +1,13 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+public class IndexerFactory
+{ public static final IndexerInterface makeIndexer(String type, String name)
+  { if (type.equals(MGIndexer.MG_INDEX_TYPE)) 
+    { return new MGIndexer(name);
+    }
+    else if (type.equals(MGPPIndexer.MGPP_INDEX_TYPE)) {
+      return new MGPPIndexer(name);
+    }
+    return null;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java	(revision 14150)
@@ -0,0 +1,29 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public interface IndexerInterface
+{
+  public static final String DOCUMENT_LEVEL = "document";
+  public static final String SECTION_LEVEL = "section";
+  public static final String PARAGRAPH_LEVEL = "paragraph";
+
+  public static final String GS2_INDEX_LABEL = "Index";  
+
+  public String  getIndexType();
+  public String  getName();
+  public int     getNumberOfPasses();
+  // TODO: add a function call to provide appropriate XML for the service description
+  public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list);
+  public void    tidyup();
+
+  public boolean configure(String label, String value);
+  public boolean configure(org.w3c.dom.Node node);
+    //  public boolean addIndex(String name, String level, String field);
+
+  public boolean startPass(int passNumber);
+  public boolean indexDocument(DocumentID id, DocumentInterface document);
+  public boolean endPass(int passNumber);
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java	(revision 14150)
@@ -0,0 +1,100 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public class IndexerManager
+{
+  IndexerInterface indexers[];
+  int size;
+  int used;
+  DocumentList documentList;
+
+  public static final String outputDir = "outputDir";
+  public static final String DEFAULT_LEVEL = "document";
+  public static final String DEFAULT_FIELD = "text";
+
+  public IndexerManager(DocumentList documentList)
+  { this.indexers  = new IndexerInterface[10];
+    this.size      = 10;
+    this.used      = 0;
+    this.documentList = documentList;
+  }
+
+  public void addIndexer(IndexerInterface indexer)
+  { this.ensureCapacity(this.used + 1);
+
+    this.indexers[this.used] = indexer;
+    this.used ++;
+  }
+
+  public void indexDocument(DocumentID docId, DocumentInterface document)
+  { if (!document.isIndexed())
+      return;
+	  
+    for (int i = 0; i < this.used; i ++)
+    { this.indexers[i].indexDocument(docId, document);
+    }
+  }
+
+  public void indexDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.indexers[i].getNumberOfPasses(); p ++)
+      {	
+	if (!this.indexers[i].startPass(p)) {
+	  continue;
+	}
+
+	Iterator iterator = this.documentList.iterator();
+
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+
+	  if (document.isIndexed()) {
+	    if (!this.indexers[i].indexDocument(document.getID(), document)) {
+	      System.out.println("Ending document " + document.getID());
+	    }
+	    
+	    // note any changes made to this document...
+	    if (document.isChanged()) {
+	      //System.out.println("Writing document "+document.getID());
+	      this.documentList.storeChangedDocument(document);
+	    }
+	    /**
+	    if (p == 0) {
+	      System.out.println("Writing document "+document.getID());
+	    }
+	    */
+	  }
+	}
+	this.indexers[i].endPass(p);
+      }
+    }
+  }
+
+  public void ensureCapacity(int needSize)
+  { while (needSize >= this.size)
+    { int newSize = this.size * 2;
+      IndexerInterface newIndexers [] = new IndexerInterface[newSize];
+      System.arraycopy(indexers, 0, newIndexers, 0, this.used);
+      this.indexers = newIndexers;
+      this.size = newSize;
+    }
+  }
+
+    public boolean addServiceDescriptions(Element service_class_list) {
+	boolean success = true;
+	for (int i = 0; i < this.used; i ++) {
+	    if (!this.indexers[i].addServiceDescriptions(service_class_list)) {
+		success = false;
+	    }
+	}
+	return success;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 14150)
@@ -0,0 +1,757 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.w3c.dom.*;
+
+import org.greenstone.mg.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.Processing;
+
+// for debug
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGIndexer //extends AbstractIndexer
+    implements IndexerInterface
+{ 
+    int          pass;
+    int          documentSeqNo;
+    int          sectionSeqNo;
+    boolean      firstDocument;
+    String       outputDirectory;
+    StringBuffer indexBuffer;
+    File         textDirectory;
+    File         indexDirectory;
+    String       indexStem;
+    String       textStem;
+    List         indexes;
+    String       overallName; 
+    
+    MGIndex current_index = null;
+    MGPassesWrapper mgPasses;
+    
+    static final char END_OF_DOCUMENT = (char) 2;
+    //static final char END_OF_SECTION  = (char) 3; // actually this is end of para for mg - don't use this yet
+    static final char END_OF_STREAM   = (char) 4;
+
+    public static final String MG_INDEX_TYPE = "mg";
+    public static final String INDEX_FILE_STEM = "index";
+    
+    class MGIndex
+    { 
+	String name=null;
+	String level=null;
+	List fields=null;
+	boolean error = false;// assume built until we get an error
+	
+	public MGIndex(Element index_element) {
+	    
+	    this.fields = new ArrayList();
+	    this.name = index_element.getAttribute(GSXML.NAME_ATT);
+	    if (this.name.equals("")) {
+		// TODO make this dynamic
+		this.name = "xx";
+	    }
+	    NodeList children = index_element.getChildNodes();
+	    for (int c = 0; c < children.getLength(); c ++) {
+		Node child = children.item(c);
+		
+		if (child.getNodeType() == Node.ELEMENT_NODE) {
+		    String name = child.getNodeName();
+	  
+		    if (name.equals(GSXML.LEVEL_ELEM)) {
+			this.level = DOMUtils.getNodeChildText(children.item(c));
+		    }
+		    else if (name.equals(GSXML.FIELD_ELEM)) {
+			String fieldName = DOMUtils.getNodeChildText(children.item(c));
+			this.fields.add(fieldName);
+		    }
+		}
+	    }
+	}
+
+	public MGIndex(String name, String level, List fields)
+	{ 
+	    this.name  = name;
+	    this.level = level;
+	    this.fields = fields;
+	}
+	
+	// old gs2 style config - can we get rid of this??
+	public MGIndex(String indexLabel)
+	{ 
+	    int colonAt = indexLabel.indexOf(':');
+	    
+	    if (colonAt < 0) {
+		System.err.println("MGIndex(): invalid index specification: "+indexLabel);
+		return;
+	    }
+	    String field_string = indexLabel.substring(colonAt+1);
+	    String [] field_list = field_string.split(",");
+	    this.fields = new ArrayList();
+	    for (int i=0; i<field_list.length; i++) {
+		this.fields.add(field_list[i]);
+	    }
+	    this.level = indexLabel.substring(0, colonAt);
+	    //createIndexName
+	    StringBuffer new_name = new StringBuffer();
+	    new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+	    int c, w;
+	    w = 0;
+	    c = 0;
+	    while (c < field_string.length() && w < 2) {
+		char ch = field_string.charAt(c);
+		
+		ch = Character.toLowerCase(ch);
+		if (Character.isLetter(ch)) {
+		    if (ch != 'a' && ch != 'e' && ch != 'i' &&
+			ch != 'o' && ch != 'u') {
+			new_name.append(ch);
+			w++;
+		    }
+		}
+		c ++;
+	    }
+	    this.name = new_name.toString();
+	}
+
+	
+	public String getLevel()
+	{ 
+	    return this.level;
+	}
+
+	public List getFields()
+	{ 
+	    return this.fields;
+	}
+
+	public String getName()
+	{ 
+// 	    if (this.name==null || this.name.equals("")) {
+// 		createIndexName();
+// 	    }
+	    return this.name;
+	}
+
+	public boolean hasError() {
+	    return this.error;
+	}
+	public void setError(boolean b) {
+	    this.error = b;
+	}
+
+// 	private void createIndexName() {
+// 	    StringBuffer new_name = new StringBuffer();
+// 	    new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+
+// 	    int c, w;
+// 	    w = 0;
+// 	    c = 0;
+// 	    String [] fields_concat = this.fields.toArray
+// 	    while (c < this.field.length() && w < 2) {
+// 		char ch = this.field.charAt(c);
+
+// 		ch = Character.toLowerCase(ch);
+// 		if (Character.isLetter(ch)) {
+// 		    if (ch != 'a' && ch != 'e' && ch != 'i' &&
+// 			ch != 'o' && ch != 'u') {
+// 			new_name.append(ch);
+// 			w++;
+// 		    }
+// 		}
+// 		c ++;
+// 	    }
+// 	    this.name = new_name.toString();
+// 	}
+ 
+
+    } // MGIndex
+
+    public MGIndexer(String name)
+    { 
+	this.indexes = new ArrayList();
+	this.overallName = name; 
+    }
+
+    public String getIndexType()
+    { 
+	return MG_INDEX_TYPE;
+    } 
+
+    public String getName()
+    { 
+	return this.overallName; 
+    }
+    
+    public boolean configure(Node search_node)
+    {
+	NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+	
+	// add a text 'index' - we should be able to turn this off in the config file? actually mg needs a text index
+	ArrayList list = new ArrayList();
+	list.add("text");
+	MGIndex index = new MGIndex("text", "section", list);
+	indexes.add(index);
+	for (int i = 0; i < index_children.getLength(); i ++) {
+	    Element index_elem = (Element)index_children.item(i);
+	    index = new MGIndex(index_elem);
+	    if (index.getName() != null && index.getLevel() != null && index.getFields()!= null) {
+		
+		indexes.add(index);
+	    } else {
+		System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+	    }
+	}
+	// TODO make sure all index names are unique
+	return true;
+    }
+
+    /**
+     *  The output directory should be (collection)/building/text/ for
+     *  normal Greenstone builds.
+     *
+     *  @param <code>String</code> the label to configure
+     *  @param <code>String</code> the value...
+     */ 
+    public boolean configure(String label, String value)
+    { 
+	if (label.equals(IndexerManager.outputDir)) {
+	    this.outputDirectory = value;
+	    this.pass = 0;
+    
+	    // attempt to ensure that the text subdirectory exists
+	    this.textDirectory = new File(outputDirectory, "text");
+	    if (!textDirectory.exists()) {
+		if (!textDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!textDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+
+	    // Sign to the user which mg directory is being used...
+	    System.out.println("Output MG directory is " + this.textStem);
+	}
+	else if (label.equals(IndexerInterface.GS2_INDEX_LABEL)) {
+	    this.indexes.add(new MGIndex(value));
+	}
+
+	return true;
+    }
+
+
+    private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	    // try doing this for all index types
+	    // actually we should only need to do this once ????
+	    if (this.pass == 0) {
+	    //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+		METSDivision division = (METSDivision) structure;
+
+		// get the division metadata block
+		METSDescriptive descriptive;
+		String metadataId = division.getDefaultMetadataReference();
+		if (metadataId == null) {
+		    descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		    division.addMetadataReference(descriptive.getID());
+		}
+		else {
+		    // Get the descriptive item...
+		    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		}
+	    
+		descriptive.addMetadata("gsdl3", "mgseqno", this.overallName + "." + Integer.toString(this.sectionSeqNo));
+	    
+		metsDoc.setChanged(true);
+		//metsDoc.setModified(true);
+		//	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	    } // first pass
+	
+	    // append an 'end of section' marker
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+	
+	    // for document-level indexes, always append an 'end of document' tag at the
+	    // end of the document for each section.  Otherwise, each section is followed
+	    // by an end of document character.  This ensures that all indexes use the
+	    // same document numbering...
+	    if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
+		extraBuffer.append(END_OF_DOCUMENT);
+	    }
+	    else {
+		textBuffer.append(END_OF_DOCUMENT);
+		this.documentSeqNo ++;
+	    }
+	    
+	    // produce the body here for metadata output of divisions - in the case of
+	    // text output, that will happen below...
+	    
+	    if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+		// if there is only text, don't do this
+		METSDescriptive descriptive;
+	
+		METSDivision division = (METSDivision) structure;
+
+		String metadataId = division.getDefaultMetadataReference();
+		// are there other metadata refs to get??
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		if (descriptive != null) {
+		    for (int i=0; i<fields.size(); i++) {
+			String field = (String)fields.get(i);
+			if (field.equals("text")) {
+			    continue;
+			}
+			List values = descriptive.getMetadata(namespace, field);
+			if (values != null) {	
+			    Iterator valueIter = values.iterator();
+			    while (valueIter.hasNext()) {
+				String value = valueIter.next().toString();
+				textBuffer.append(value);
+				textBuffer.append(" ");
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	
+	// go through our children as required...
+	Iterator children = structure.getChildIterator();
+	Node startNode;
+	boolean index_text = fields.contains("text");
+	while (children.hasNext()) {
+	    AbstractStructure child = (AbstractStructure) children.next();
+      
+	    // get xpointer for child 
+	    // get start position node
+	    if (metsDoc.getDocumentType() == "METS"){
+		startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+	    } else {
+		startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+	    }
+	    
+	    // while this node isn't the child's start node, produce the 
+	    // HTML node text, if in text field mode...
+	    if (index_text) {
+		while (node != startNode) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+	    
+	    // recurse to child
+	    node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+	} // while next child
+
+	// close a document - the actual closing \B will be done by the main
+	// loop, so only a required \C is printed here...
+	// why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
+	if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+	    if (index_text) {
+		while (node != null) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+      
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+      
+	}
+	return node;
+    }
+
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+    {
+	//  String name, String namespace, String field)
+	StringBuffer extraBuffer = new StringBuffer();
+	Node node = document.getDocumentElement();
+	StringBuffer textBuffer = new StringBuffer();
+	
+	this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
+	textBuffer.append(extraBuffer.toString());
+	return textBuffer.toString();
+    }
+
+    /**
+     *  Index a single document; the document interface can be used to extract individual
+     *  metadata items etc. as required or desired and index those instead or as well as
+     *  the body text of the document.
+     */
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+	int count = ((AbstractDocument)document).getNumSections();
+	String docText = null;
+	// set the mgseqno if first pass
+	if (this.pass == 0) {
+	    document.removeAllMetadata("gsdl3", "mgseqno");
+	    document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+	}
+
+	this.sectionSeqNo ++;
+	
+	//long start = System.currentTimeMillis();
+	if (this.current_index.getLevel().equals("section")) {
+
+	    Document domDocument = document.getDOMDocument();
+	    if (domDocument != null) {
+		System.err.println("dom doc is not null");
+		METSStructure sections = document.getDocumentStructure().getStructure("Section");
+		if (sections != null) {
+		    System.err.println("sections are not null");
+		    docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+		    //	System.out.println(docText);
+		}
+	    }
+	} 
+	    
+	
+	//long finish = System.currentTimeMillis();
+	//System.err.println("dom doc = "+ Long.toString(finish-start));
+	//start = System.currentTimeMillis();
+	if (docText == null) {
+	    System.err.println("dom doc or sections was null - asking for doc text");
+	    StringBuffer doc_text_buffer = new StringBuffer();
+	    List fields = this.current_index.getFields();
+	    for (int i=0; i<fields.size(); i++) {
+		String field = (String)fields.get(i);
+		if (field.equals("text")) {
+		    doc_text_buffer.append(document.getDocumentText());
+		    doc_text_buffer.append(" ");
+		}  else {
+		    // its a metadata - do namespace properly!!
+		    List values = document.getDocumentMetadataItem("gsdl3", field);
+		    if (values != null) {
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    doc_text_buffer.append(value);
+			    doc_text_buffer.append(" ");
+			}
+		    }
+		}
+	    } // for each field
+	    docText = doc_text_buffer.toString();
+	    sectionSeqNo ++;
+	    int num_secs = 0;
+	}
+	//finish = System.currentTimeMillis();
+	//System.err.println("whole doc = "+ Long.toString(finish-start));
+	
+	this.indexBuffer.append(docText);
+	// remember that we're not on the first document, 
+	//this.firstDocument = false;
+	this.documentSeqNo ++;
+	//if (!this.firstDocument) {
+	this.indexBuffer.append(END_OF_DOCUMENT);
+	mgPasses.processDocument(indexBuffer.toString());
+	String filename="";
+	try {
+	    filename = "pass"+this.pass+"doc"+this.documentSeqNo+".txt";
+	    System.err.println("trying to write to "+filename);
+	    GSFile.writeFile(indexBuffer.toString().getBytes(), filename);
+	} catch (Exception e) {
+	    System.err.println("COUldn't write to file, "+filename);
+	}
+	this.indexBuffer.delete(0, this.indexBuffer.length());
+	    
+	
+
+	return true;
+    }
+
+    /**
+     *  Initialise the pass: open required files, check status
+     */
+    public boolean startPass(int passNumber) {
+    	
+	
+	this.pass = passNumber;
+	this.firstDocument = true;
+	this.documentSeqNo = 1;
+	this.sectionSeqNo  = 1;
+	
+	this.mgPasses = new MGPassesWrapper();
+	this.indexBuffer = new StringBuffer();
+	int indexNo = this.pass/2;
+	
+	this.current_index = (MGIndex) this.indexes.get(indexNo);
+	if (this.current_index.hasError()) {
+	    // an error has already occurred for this index, don't continue
+	    System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+	    return false;
+	}
+	// attempt to ensure that the text/index subdirectory exists
+	this.indexDirectory = new File(outputDirectory, current_index.getName());
+	if (!indexDirectory.exists()) {
+	    if (!indexDirectory.mkdir()) {
+		return false;
+	    }
+	}
+	else if (!indexDirectory.isDirectory()) {
+	    return false;
+	}
+
+	this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+	if (this.pass == 0) {
+	    // first pass, also set up the textStem
+	    this.textDirectory = this.indexDirectory;
+	    this.textStem = this.indexStem;
+	}
+	
+	mgPasses.setFileName(this.indexStem);
+	if (!Misc.isWindows()) {
+	    mgPasses.setBasePath("/");
+	}
+	int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	
+	mgPasses.setBufferSize(100000);
+ 	
+	switch (mgPass) {
+	case 0:
+	    // -b 100000 -T1
+	    mgPasses.addPass(MGPassesWrapper.TEXT_PASS_1);
+	    
+	    
+	    break;
+	    
+	case 1:
+	    // -b 100000 -T2
+	    mgPasses.addPass(MGPassesWrapper.TEXT_PASS_2);
+	    break;
+	    
+	case 2:
+	    // -b 100000 -2 -m 32 -s 0 -G -t 10 -N1
+	    mgPasses.addPass(MGPassesWrapper.INDEX_PASS_1);
+	    mgPasses.setInvfLevel(MGPassesWrapper.INVF_LEVEL_2);
+	    mgPasses.setStemOptions(MGPassesWrapper.STEMMER_ENGLISH, MGPassesWrapper.NO_STEM_OR_CASE);
+	    mgPasses.setInversionMemLimit(32);
+	    mgPasses.ignoreSGMLTags(true);
+	    break;
+	    
+	case 3:
+	    // -b 100000 -2 -c 3 -G -t 10 -N2
+	    mgPasses.addPass(MGPassesWrapper.INDEX_PASS_2);
+	    mgPasses.setInvfLevel(MGPassesWrapper.INVF_LEVEL_2);
+	    mgPasses.ignoreSGMLTags(true);
+	    break;
+	}
+	
+	mgPasses.init();
+	System.out.println("Starting Pass " + this.pass);
+	return true;
+    }
+  
+    /**
+     *  Complete a pass - reset file counters, close files, etc.
+     */
+    public boolean endPass(int passNumber) {
+	try {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    Thread.sleep(1000); // what for??
+	}
+	catch (InterruptedException ex) {
+	    System.out.println(ex);
+	}
+	mgPasses.finish();
+	try {
+	    Thread.sleep(1000);
+	} catch (Exception e) {}
+	
+	int exit_value = mgPasses.exitValue();
+	System.out.println("Pass " + this.pass + " completed with " + exit_value);
+	if (exit_value !=0) {
+	    //assume something has gone wrong, don't continue
+	    current_index.setError(true);
+	    return false;
+	}
+	int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	String osextra = "";
+	if (!Misc.isWindows()) {
+	    osextra = " -d / ";
+	}
+	
+	switch (mgPass) {
+	    
+	case 0:
+	    System.out.println("Compressing dictionary");
+	    exit_value = Processing.runProcess("mg_compression_dict -f " + this.textStem + osextra + " -S -H -2 -k 5120");
+	    if (exit_value == 0) {
+		System.out.println("Compressed dictionary successfully written");
+	    } else {
+		System.err.println("Error from mg_compression_dict: " + exit_value);
+		current_index.setError(true);
+		
+		return false;
+	    }
+	    break;
+	    
+	case 2:
+	    System.out.println("Creating perfect hash");
+	    exit_value = Processing.runProcess("mg_perf_hash_build -f " + this.indexStem+osextra);
+	    if (exit_value ==0) {
+		System.out.println("Perfect hashes completed");
+	    } else {
+		System.err.println("Unable to build the perfect hash");
+		current_index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 3:
+	    System.out.println("Writing weights file");
+	    exit_value = Processing.runProcess("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Weights file successfully written");
+	    } else {
+		System.err.println("Unable to create weights file");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating inverted dictionary");
+	    exit_value = Processing.runProcess("mg_invf_dict -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Inverted dictionary file successfully written");
+	    } else {
+		System.out.println("Unable to create inverted dictionary file");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating Stem indexes");
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s1 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 1 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 1");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s2 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 2 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 2");
+		current_index.setError(true);
+		return false;
+	    }
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s3 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 3 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 3");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    break;
+	} // switch
+
+	mgPasses = null;
+	return true;
+    }
+    
+    /**
+     *  Do any tidying up
+     */
+    public void tidyup()
+    {
+    }
+    
+    /**
+     *  Return the number of passes required for this index.
+     */
+    public int getNumberOfPasses()
+    { 
+	//return 2 + this.indexes.size() * 2;
+	return this.indexes.size()*2;
+    }
+
+    public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list)
+    {
+	Document doc = service_rack_list.getOwnerDocument();
+
+	// generate the list of indexes
+	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	boolean found_index = false;
+	String def_index = ""; // the default index will just be the first one created for now.
+	for (int i=1; i<this.indexes.size(); i++) {
+	    // start at 1: 0 will be the text index
+	    MGIndex index = (MGIndex)this.indexes.get(i);
+	    if (!index.hasError()) {
+		Element e = doc.createElement(GSXML.INDEX_ELEM);
+		e.setAttribute(GSXML.NAME_ATT, index.getName());
+		index_list.appendChild(e);
+		if (found_index == false) {
+		    // this is the first index
+		    found_index = true;
+		    def_index = index.getName();
+		}
+	    }
+	}
+	
+	if (!found_index) {
+	    // no indexes were able to be created, so we can't use them or the text
+	    return false;
+	}
+	Element default_index = doc.createElement("defaultIndex");
+	default_index.setAttribute(GSXML.NAME_ATT, def_index);
+	Element base_index_name = doc.createElement("baseIndexPrefix");
+	base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
+	Element index_stem = doc.createElement("indexStem");
+	index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
+	
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	service_rack_list.appendChild(search_service_elem);
+	service_rack_list.appendChild(retrieve_service_elem);
+
+	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGSearch");  
+	search_service_elem.appendChild(index_list);
+	search_service_elem.appendChild(default_index);
+	search_service_elem.appendChild(base_index_name);
+	search_service_elem.appendChild(index_stem);
+	
+	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGRetrieve");
+	retrieve_service_elem.appendChild(default_index.cloneNode(true));
+	retrieve_service_elem.appendChild(base_index_name.cloneNode(true));
+	retrieve_service_elem.appendChild(index_stem.cloneNode(true));
+
+	return true;
+    }
+    
+}
+    
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 14150)
@@ -0,0 +1,933 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import org.greenstone.mgpp.MGPPPassesWrapper;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
+import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.Processing;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.io.File;
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGPPIndexer //extends AbstractIndexer
+    implements IndexerInterface
+{ 
+    int          pass;
+    int          documentSeqNo;
+    int          sectionSeqNo;
+    boolean      firstDocument;
+    File indexDirectory;
+    //    File textDirectory;
+    String       indexStem;
+    //    String       textStem;
+    StringBuffer indexBuffer;
+    String       outputDirectory;
+    String       overallName; 
+    
+    List indexes;
+    MGPPIndex current_index = null;
+    MGPPPassesWrapper mgppPasses;
+    
+
+    static final String DOCUMENT = "Doc";
+    static final String SECTION = "Sec";
+    static final String START_OF_DOCUMENT = "<"+DOCUMENT+">";
+    static final String END_OF_DOCUMENT = "</"+DOCUMENT+">";
+    static final String START_OF_SECTION = "<"+SECTION+">";
+    static final String END_OF_SECTION = "</"+SECTION+">";
+    
+    
+    public static final String MGPP_INDEX_TYPE = "mgpp";
+    public static final String INDEX_FILE_STEM = "index";
+
+
+    class MGPPIndex
+    {
+	public String name = null;
+	public String doc_level = null; 
+	public List levels = null;
+	public List fields = null;
+	boolean error = false;// assume built until we get an error
+	
+	public MGPPIndex(Element index_element) {
+	    
+	    this.fields = new ArrayList();
+	    this.levels = new ArrayList();
+	    this.name = index_element.getAttribute(GSXML.NAME_ATT);
+	    if (this.name.equals("")) {
+		// TODO make this dynamic
+		this.name = "xx";
+	    }
+	    NodeList children = index_element.getChildNodes();
+	    for (int c = 0; c < children.getLength(); c ++) {
+		Node child = children.item(c);
+		
+		if (child.getNodeType() == Node.ELEMENT_NODE) {
+		    String name = child.getNodeName();
+		    
+		    if (name.equals(GSXML.LEVEL_ELEM)) {
+			String level = DOMUtils.getNodeChildText(children.item(c));
+			this.levels.add(level);
+		    }
+		    else if (name.equals(GSXML.FIELD_ELEM)) {
+			String fieldName = DOMUtils.getNodeChildText(children.item(c));
+			this.fields.add(fieldName);
+		    }
+		}
+	    }
+	}
+	
+	public MGPPIndex(String name) {
+	    this.name = name;
+	    this.doc_level = DOCUMENT;
+	    this.fields = new ArrayList();
+	    this.levels = new ArrayList();
+	}
+	
+	public void setDocLevel(String doc_level) {
+	    this.doc_level = doc_level;
+	}
+
+	public void addLevel(String level) {
+	    this.levels.add(level);
+	}
+
+	// change to allow nested fields
+	public void addField(String field) {
+	    this.fields.add(field);
+	}
+	public List getLevels() {
+	    return this.levels;
+	}
+	public List getFields() {
+	    return this.fields;
+	}
+	public String getName() {
+	    return this.name;
+	}
+	public boolean hasError() {
+	    return this.error;
+	}
+	public void setError(boolean b) {
+	    this.error = b;
+	}
+
+    }
+    
+    
+    public MGPPIndexer(String name)
+    { 
+	this.overallName = name;
+	this.indexes = new ArrayList();
+	//this.passExtra = "";
+    }
+
+    public String getName()
+    { 
+	return this.overallName;
+    }
+    
+    public String getIndexType()
+    { 
+	return MGPP_INDEX_TYPE;
+    }
+
+    // for now make all indexes use document and section levels. 
+    // then when writing the buildconfig, only display the levels 
+    // that the user has specified (likely to be both doc and sec).
+    public boolean configure(Node search_node)
+    {
+	NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+	
+	// add a text 'index' - we should be able to turn this off in the config file? 
+	MGPPIndex text_index = new MGPPIndex("text");
+	text_index.addField("text");
+	// always do eveything at doc and sec level at the moment
+	text_index.addLevel(DOCUMENT);
+	text_index.addLevel(SECTION);
+	indexes.add(text_index);
+	
+	for (int i = 0; i < index_children.getLength(); i ++) {
+	    Element index_elem = (Element)index_children.item(i);
+	    MGPPIndex index = new MGPPIndex(index_elem);
+	    if (index.getName() != null && index.getLevels() != null && index.getFields()!= null) {
+		indexes.add(index);
+	    } else {
+		System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+	    }
+	}
+	// TODO make sure all index names are unique
+	return true;
+    }
+
+    /**
+     *  The output directory should be (collection)/building/text/ for
+     *  normal Greenstone builds
+     */ 
+    public boolean configure(String label, String value)
+    { 
+	if (label.equals(IndexerManager.outputDir)) {
+	    this.outputDirectory = value;
+	    
+	    // attempt to ensure that the text subdirectory exists
+// 	    this.textDirectory = new File(outputDirectory, "text");
+// 	    if (!textDirectory.exists()) {
+// 		if (!textDirectory.mkdir()) {
+// 		    return false;
+// 		}
+// 	    }
+// 	    else if (!textDirectory.isDirectory()) {
+// 		return false;
+// 	    }
+// 	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+// 	    // attempt to ensure that the index subdir exists
+// 	    this.indexDirectory = new File(outputDirectory, "idx");
+// 	    if (!indexDirectory.exists()) {
+// 		if (!indexDirectory.mkdir()) {
+// 		    return false;
+// 		}
+// 	    }
+// 	    else if (!indexDirectory.isDirectory()) {
+// 		return false;
+// 	    }
+// 	    this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+	    // Sign to the user which mg directory is being used...
+// 	    System.out.println("Output MGPP text directory is " + this.textStem);
+// 	    System.out.println("Output MGPP index directory is " + this.indexStem); 
+	}
+	this.pass = 0;
+	return true;
+    }
+
+
+//     public boolean addIndex(String name, String level, String field)
+//     { 
+// // 	if (level == "doc_level") {
+// // 	    passExtra = " -J " + level;
+// // 	}
+// // 	else {
+// // 	    passExtra = " -K " + level;
+// // 	}
+// 	return true;
+//     }
+
+    /**
+     *  Index a single document; the document interface can be used to extract individual
+     *  metadata items etc. as required or desired and index those instead or as well as
+     *  the body text of the document.
+     */
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+	if (this.pass == 0) {
+	    document.removeAllMetadata("gsdl3", "mgseqno");
+	    document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+	}
+	
+	
+	String docText = null;
+	
+	this.sectionSeqNo ++;
+	
+	Document domDocument = document.getDOMDocument();
+	if (domDocument != null) {
+	    System.err.println("dom doc is not null");
+	    METSStructure sections = document.getDocumentStructure().getStructure("Section");
+	    if (sections != null) {
+		System.err.println("sections are not null");
+		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); 
+	    }
+	}
+	if (docText == null) {
+	    System.err.println("dom doc or sections was null - asking for doc text");
+	    StringBuffer doc_text_buffer = new StringBuffer();
+	    List fields = this.current_index.getFields();
+	    for (int i=0; i<fields.size(); i++) {
+		String field = (String)fields.get(i);
+		if (field.equals("text")) {
+		    doc_text_buffer.append(document.getDocumentText());
+		    doc_text_buffer.append(" ");
+		}  else {
+		    // its a metadata - do namespace properly!!
+		    List values = document.getDocumentMetadataItem("gsdl3", field);
+		    if (values != null) {
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    doc_text_buffer.append(value);
+			    doc_text_buffer.append(" ");
+			}
+		    }
+		}
+	    } // for each field
+	    docText = doc_text_buffer.toString();
+	    sectionSeqNo ++;
+	}
+	
+	this.indexBuffer.append(START_OF_DOCUMENT);
+	this.indexBuffer.append(START_OF_SECTION);
+	this.indexBuffer.append(docText);
+	this.indexBuffer.append(END_OF_SECTION);
+	this.indexBuffer.append(END_OF_DOCUMENT);
+	this.mgppPasses.processDocument(indexBuffer.toString());
+	this.indexBuffer.delete(0, this.indexBuffer.length());
+
+	this.firstDocument = false;
+
+	this.documentSeqNo++;
+	
+	return true;
+    }
+
+    /**
+     *  Initialise the pass: open required files, check status
+     */
+    public boolean startPass(int passNumber)
+    { 
+	this.pass = passNumber;
+	this.firstDocument = true;
+	this.documentSeqNo = 1;
+	this.sectionSeqNo = 1;
+
+	this.mgppPasses = new MGPPPassesWrapper();
+	this.indexBuffer = new StringBuffer();
+	
+	int indexNo = this.pass/2;
+	this.current_index = (MGPPIndex) this.indexes.get(indexNo);
+
+	if (this.current_index.hasError()) {
+	    // an error has already occurred for this index, don't continue
+	    System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+	    return false;
+	}
+	
+	// attempt to ensure that the text/index subdirectory exists
+	this.indexDirectory = new File(outputDirectory, current_index.getName());
+	if (!indexDirectory.exists()) {
+	    if (!indexDirectory.mkdir()) {
+		return false;
+	    }
+	}
+	else if (!indexDirectory.isDirectory()) {
+	    return false;
+	}
+
+	this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+// 	if (this.pass == 0) {
+// 	    // first pass, also set up the textStem
+// 	    this.textDirectory = this.indexDirectory;
+// 	    this.textStem = this.indexStem;
+// 	}
+
+	// get the parameters for this execution of mg_passes
+	mgppPasses.setFileName(this.indexStem);
+	if (!Misc.isWindows()) {
+	    mgppPasses.setBasePath("/");
+	}
+	
+	// always use Doc and Sec for now
+	mgppPasses.setDocumentTag(DOCUMENT);
+	mgppPasses.addLevelTag(SECTION);
+	
+	//this.currentIndexLevel = "Document";// index.getLevel();
+	//this.currentIndexField = "text";//index.getField();
+	//this.currentIndexName = "idx"; //index.getName();
+	    
+	int mgppPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	switch (mgppPass) {
+	case 0:
+	    // -T1
+	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_1);
+	    break;
+	    
+	case 1:
+	    // -T2
+	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_2);
+	    break;
+	    
+	case 2:
+	    // -I1
+	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_1);
+	    break;
+	    
+	case 3:
+	    // -I2
+	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_2);
+	    break;
+	}
+	
+	mgppPasses.init();
+	System.out.println("Pass " + this.pass);
+	return true;
+    }
+
+    /**
+     *  Complete a pass - reset file counters, close files, etc.
+     */
+    public boolean endPass(int passNumber)
+    { 
+	try {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgppPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    Thread.sleep(1000); // what for??
+	}
+	catch (InterruptedException ex) {
+	    System.out.println(ex);
+	}
+	
+	mgppPasses.finish();
+	
+	try {
+	    Thread.sleep(1000);
+	} catch (Exception e) {}
+	
+	int exit_value = mgppPasses.exitValue();
+	System.out.println("Pass " + this.pass + " completed with " + exit_value);
+	if (exit_value !=0) {
+	    this.current_index.setError(true);
+	    return false;
+	}
+	
+	int mgppPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	
+	String osextra = "";
+	if (!Misc.isWindows()) {
+	    osextra = " -d / ";
+	}
+
+	switch (mgppPass) {
+	case 0:
+	    System.out.println("Compressing dictionary");
+	    exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.indexStem + " -S -H -2 -k 5120"+ osextra);
+	    
+	    if (exit_value == 0) {
+		System.out.println("Compressed dictionary successfully written");
+	    } else {
+		System.err.println("Error from mgpp_compression_dict: " + exit_value);
+		this.current_index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 2:
+	    System.out.println("Creating perfect hash");
+	    exit_value = Processing.runProcess("mgpp_perf_hash_build -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Perfect hashes completed");
+	    } else {
+		System.err.println("Unable to build the perfect hash");
+		this.current_index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 3:
+	    System.out.println("Writing weights file");
+	    exit_value = Processing.runProcess("mgpp_weights_build -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Weights file successfully written");
+	    } else {
+		System.err.println("Unable to create weights file");
+		this.current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating inverted dictionary");
+	    exit_value = Processing.runProcess("mgpp_invf_dict -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Inverted dictionary file successfully written");
+	    } else {
+		System.out.println("Unable to create inverted dictionary file");
+		this.current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating Stem indexes");
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s1 -f " + this.indexStem +osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 1 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 1");
+		this.current_index.setError(true);
+		return false;
+	    }
+	    
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s2 -f " + this.indexStem + osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 2 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 2");
+		this.current_index.setError(true);
+		return false;
+	    }
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s3 -f " + this.indexStem + osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 3 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 3");
+		this.current_index.setError(true);
+		return false;
+	    }
+	    
+	    break;
+	} // switch
+	
+	return true;
+    }
+	
+    /**
+     *  Do any tidying up
+     */
+    public void tidyup()
+    {
+    }
+    
+    /**
+     *  Return the number of passes required for this index.
+     */
+    public int getNumberOfPasses()
+    { 
+	//return this.indexes.size()*2;
+	return 4;
+    }
+
+    public boolean addServiceDescriptions(Element service_rack_list) {
+	
+	// we only have one real index at the moment, - the first index in the list will be the text one.
+	MGPPIndex index = (MGPPIndex)this.indexes.get(1);
+	if (index.hasError()) {
+	    // we weren't able to create any indexes - don't add a search service
+	    return false;
+	}
+	
+	Document doc = service_rack_list.getOwnerDocument();
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	service_rack_list.appendChild(search_service_elem);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);		
+	service_rack_list.appendChild(retrieve_service_elem);
+	
+	// generate the list of indexes - with only one index in it
+	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	Element index_elem = doc.createElement(GSXML.INDEX_ELEM);
+	index_elem.setAttribute(GSXML.NAME_ATT, index.getName());
+	index_list.appendChild(index_elem);
+	
+	Element default_index = doc.createElement("defaultIndex");
+	default_index.setAttribute(GSXML.NAME_ATT, index.getName());
+
+	Element base_index_name = doc.createElement("baseIndexPrefix");
+	base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
+
+	Element index_stem = doc.createElement("indexStem");
+	index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
+	
+	Element default_level = doc.createElement("defaultLevel");
+	default_level.setAttribute(GSXML.NAME_ATT, SECTION);
+
+	// always have doc and sec at the moment
+	Element level_list = doc.createElement(GSXML.LEVEL_ELEM+GSXML.LIST_MODIFIER);
+	Element level = doc.createElement(GSXML.LEVEL_ELEM);
+	level.setAttribute(GSXML.NAME_ATT, DOCUMENT);
+	level_list.appendChild(level);
+
+	level = doc.createElement(GSXML.LEVEL_ELEM);
+	level.setAttribute(GSXML.NAME_ATT, SECTION);
+	level_list.appendChild(level);
+	
+	Element field_list = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	Element field;
+	List fields = index.getFields();
+	for (int i=0; i<fields.size(); i++) {
+	    String f = (String) fields.get(i);
+	    field = doc.createElement(GSXML.FIELD_ELEM);
+	    field.setAttribute(GSXML.NAME_ATT, f);
+	    field_list.appendChild(field);
+	}
+	
+	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPSearch");  
+	search_service_elem.appendChild(index_list);
+	search_service_elem.appendChild(default_index);
+	search_service_elem.appendChild(level_list);
+	search_service_elem.appendChild(default_level);
+	search_service_elem.appendChild(field_list); 
+	search_service_elem.appendChild(base_index_name);
+	search_service_elem.appendChild(index_stem);
+	
+	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPRetrieve");
+	retrieve_service_elem.appendChild(default_level.cloneNode(true));
+	retrieve_service_elem.appendChild(base_index_name.cloneNode(true));
+	retrieve_service_elem.appendChild(index_stem.cloneNode(true));
+
+	return true;
+    }
+ 
+ 
+    private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	    // try doing this for all index types
+	    // actually we should only need to do this once ????
+	    if (this.pass == 0) {
+	    System.err.println("division structure");
+	    //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+		METSDivision division = (METSDivision) structure;
+
+		// get the division metadata block
+		METSDescriptive descriptive;
+		String metadataId = division.getDefaultMetadataReference();
+		if (metadataId == null) {
+		    descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		    division.addMetadataReference(descriptive.getID());
+		}
+		else {
+		    // Get the descriptive item...
+		    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		}
+	    
+		descriptive.addMetadata("gsdl3", "mgseqno", this.overallName + "." + Integer.toString(this.sectionSeqNo));
+	    
+		metsDoc.setChanged(true);
+		//metsDoc.setModified(true);
+		//	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	    } // first pass
+	
+	    // append an 'end of section' marker
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+	    textBuffer.append(END_OF_SECTION);
+	    textBuffer.append(START_OF_SECTION);
+	    // for document-level indexes, always append an 'end of document' tag at the
+	    // end of the document for each section.  Otherwise, each section is followed
+	    // by an end of document character.  This ensures that all indexes use the
+	    // same document numbering...
+	    
+// 	    if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
+// 		extraBuffer.append(END_OF_DOCUMENT);
+// 	    }
+// 	    else {
+// 		textBuffer.append(END_OF_DOCUMENT);
+// 		this.documentSeqNo ++;
+// 	    }
+	    
+	    // produce the body here for metadata output of divisions - in the case of
+	    // text output, that will happen below...
+	    
+	    if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+		// if there is only text, don't do this
+		METSDescriptive descriptive;
+	
+		METSDivision division = (METSDivision) structure;
+
+		String metadataId = division.getDefaultMetadataReference();
+		// are there other metadata refs to get??
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		if (descriptive != null) {
+		    for (int i=0; i<fields.size(); i++) {
+			String field = (String)fields.get(i);
+			if (field.equals("text")) {
+			    continue;
+			}
+			List values = descriptive.getMetadata(namespace, field);
+			if (values != null) {	
+			    Iterator valueIter = values.iterator();
+			    while (valueIter.hasNext()) {
+				String value = valueIter.next().toString();
+				textBuffer.append("<"+field+">");
+				textBuffer.append(value);
+				textBuffer.append("</"+field+">");
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	
+	// go through our children as required...
+	Iterator children = structure.getChildIterator();
+	Node startNode;
+	boolean index_text = fields.contains("text");
+	while (children.hasNext()) {
+	    AbstractStructure child = (AbstractStructure) children.next();
+      
+	    // get xpointer for child 
+	    // get start position node
+	    if (metsDoc.getDocumentType() == "METS"){
+		startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+	    } else {
+		startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+	    }
+	    
+	    // while this node isn't the child's start node, produce the 
+	    // HTML node text, if in text field mode...
+	    if (index_text) {
+		while (node != startNode) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+	    
+	    // recurse to child
+	    node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+	} // while next child
+
+	// close a document - the actual closing \B will be done by the main
+	// loop, so only a required \C is printed here...
+	// why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
+	if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+	    if (this.pass == 0) {
+		System.err.println("structure structure");
+	    }
+	    if (index_text) {
+		while (node != null) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+      
+	    //textBuffer.append(END_OF_SECTION);
+	    //this.sectionSeqNo ++;
+      
+	}
+	return node;
+    }
+
+    
+    private void printNode(Node node, StringBuffer buffer, 
+			   boolean preserve_format, boolean close) {
+	
+	if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	    if (!close) {
+		buffer.append(node.getNodeValue());
+		buffer.append(" ");
+	    }
+	    return;
+	}
+	
+	if (node.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+	   if (!close) {
+	       buffer.append("&");
+	       buffer.append(node.getNodeValue());
+	       buffer.append(";");
+	       buffer.append(" ");
+	   }
+	   return;
+	}
+	if (!preserve_format) return; // we don't want any xml output
+	/*
+	else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      if (!close) {
+	buffer.append("<!-- ");
+	buffer.append(node.getNodeValue());
+	buffer.append(" -->");
+      }
+      }*/
+	if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	    
+	    boolean hasChildren = (node.getChildNodes().getLength() > 0);
+	    
+	    if (close) {
+		if (hasChildren) {
+		    // put the close node
+		    buffer.append("</");
+		    buffer.append(node.getNodeName());
+		    buffer.append(">");
+		}
+		// else it was closed off previously
+		return;
+	    }
+	    
+	    
+	    buffer.append("<");
+	    buffer.append(node.getNodeName());
+	    NamedNodeMap attributes = ((Element) node).getAttributes();
+	    for (int a = 0; a < attributes.getLength(); a ++) {
+		Node attributeNode = attributes.item(a);
+		buffer.append(" ");
+		buffer.append(attributeNode.getNodeName());
+		
+		String value = attributeNode.getNodeValue();
+		if (value != null && value.length() > 0) {
+		    buffer.append("=\"");
+		    buffer.append(value);
+		    buffer.append("\"");
+		}
+	    }
+	    
+
+	    if (!hasChildren) {
+		buffer.append(" /");
+	    }
+	    buffer.append(">");
+	    
+	    //if (close && node.getNodeName()=="Section") {
+	    //buffer.append((char) 3);
+	    // }
+	}
+	
+    }
+    
+    /*
+    private Node recurseDOMold(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	// try doing this for all index types
+	    if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+	    METSDivision division = (METSDivision) structure;
+
+	    // get the division metadata block
+	    METSDescriptive descriptive;
+	    String metadataId = division.getDefaultMetadataReference();
+	    if (metadataId == null) {
+		descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		division.addMetadataReference(descriptive.getID());
+	    }
+	    else {
+		// Get the descriptive item...
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+	    }
+	    
+	    descriptive.addMetadata("gsdl3", "mgseqno", this.name + "." + Integer.toString(this.sectionSeqNo));
+	    
+	    metsDoc.setChanged(true);
+	    //metsDoc.setModified(true);
+	    //	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	} // section level
+	
+	// append an 'end of section' marker
+	//textBuffer.append(END_OF_SECTION);
+	this.sectionSeqNo ++;
+	
+	// for document-level indexes, always append an 'end of document' tag at the
+	// end of the document for each section.  Otherwise, each section is followed
+	// by an end of document character.  This ensures that all indexes use the
+	// same document numbering...
+	if (this.currentIndexLevel == null ||
+	    this.currentIndexLevel.equals(IndexerInterface.DOCUMENT_LEVEL)) {
+	    extraBuffer.append(END_OF_DOCUMENT);
+	}
+	else {
+	    textBuffer.append(END_OF_DOCUMENT);
+	    this.documentSeqNo ++;
+	}
+	
+	// produce the body here for metadata output of divisions - in the case of
+      // text output, that will happen below...
+      if (!this.currentIndexField.equals("text")) 
+      { METSDescriptive descriptive;
+	
+	METSDivision division = (METSDivision) structure;
+
+	String metadataId = division.getDefaultMetadataReference();
+	    
+	descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+	if (descriptive != null) {
+	  List values = descriptive.getMetadata(namespace, this.currentIndexField);
+	  
+	  if (values != null) {	
+	    Iterator valueIter = values.iterator();
+	    while (valueIter.hasNext()) {
+	      String value = valueIter.next().toString();
+	      
+	      textBuffer.append(value);
+	      if (valueIter.hasNext()) {
+		  //textBuffer.append(END_OF_SECTION);
+	      }
+	    }
+	  }
+	}
+      }
+    }
+
+    // go through our children as required...
+    Iterator children = structure.getChildIterator();
+    Node startNode;
+    while (children.hasNext()) {
+      AbstractStructure child = (AbstractStructure) children.next();
+      
+      // get xpointer for child 
+      // get start position node
+      if (metsDoc.getDocumentType() == "METS"){
+	  startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+      } else {
+	  startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+      }
+	    //Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+      
+      // while this node isn't the child's start node, produce the HTML node text, if
+      // in text field mode...
+      if (this.currentIndexField.equals("text")) {
+	while (node != startNode) {
+	  XPointer.printNode(node, textBuffer, false);
+
+	  // print buffer to node
+	  node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+	}
+      }
+      
+      // recurse to child
+      node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+    } // while next child
+
+    // close a document - the actual closing \B will be done by the main
+    // loop, so only a required \C is printed here...
+    if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+      while (node != null) {
+	if (this.currentIndexField.equals("text")) {
+	  XPointer.printNode(node, textBuffer, false);
+	}
+	node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+      }
+      
+      //textBuffer.append(END_OF_SECTION);
+      this.sectionSeqNo ++;
+      
+    }
+    return node;
+  }
+    */
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+	//  String name, String namespace, String field)
+    { 
+	StringBuffer extraBuffer = new StringBuffer();
+	Node node = document.getDocumentElement();
+	StringBuffer textBuffer = new StringBuffer();
+	
+	this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); 
+	textBuffer.append(extraBuffer.toString());
+	return textBuffer.toString();
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java	(revision 14150)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class AbstractIdentifierFactory
+{
+  private String idPrelude;
+  private int    idSeqNo;
+
+  public AbstractIdentifierFactory(String idPrelude)
+  { this.idPrelude = idPrelude;
+    this.idSeqNo   = 1;
+  }
+
+  public String getNextIdentifier()
+  { int id = this.idSeqNo ++;
+
+    return idPrelude + Integer.toString(id);
+  }
+
+  public void noteIdentifier(String identifier)
+  { if (!identifier.startsWith(this.idPrelude)) 
+      return;
+  
+    identifier = identifier.substring(this.idPrelude.length());
+
+    int value = Integer.parseInt(identifier);
+    if (value >= this.idSeqNo) {
+      this.idSeqNo = value + 1;
+    }
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java	(revision 14150)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class AbstractMetadataGroup
+{ private String ID;
+
+  public AbstractMetadataGroup(String ID)
+  { this.ID = ID;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java	(revision 14150)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+
+/**
+ *  A helpful 'superclass' of METSStructure and METSDivision which allows for some
+ *  simple generalisation of the two classes.
+ */
+
+public abstract class AbstractStructure
+{ Map               children;
+  AbstractStructure parent;
+  String            ID;     // the unique identifier for this division (unique within the
+                            // document, not necessarily unique within the collection).
+
+  public AbstractStructure(String ID)
+  { this.children = new LinkedHashMap();
+    this.parent   = null;
+    this.ID       = ID;
+  }
+
+  public AbstractStructure getParent()
+  { return this.parent;
+  }
+
+  public void setParent(AbstractStructure parent)
+  { this.parent = parent;
+  }
+
+  /**
+   *  Get the identifer of this structure
+   *
+   *  @param <code>String</code> the identifier.
+   */
+  public String getID()
+  { return this.ID;
+  }
+
+  public abstract String getStructureType();
+
+  /**
+   *  Add a child division, or sub-division to the document.  For example,
+   *  sections within a chapter.
+   *
+   *  @param <code>METSDivision</code> the child division.
+   */
+  public void addDivision(METSDivision child)
+  { this.children.put(child.getLabel(), child);
+    child.setParent(this);
+  }
+
+  public Iterator getChildIterator()
+  { return this.children.values().iterator();
+  }
+    
+  public int size()
+  { return this.children.size();
+  }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java	(revision 14150)
@@ -0,0 +1,18 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  TODO: complete a class to create file identifiers
+ */
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class FileIdentifierFactory extends AbstractIdentifierFactory
+{
+    public static final String FILE_IDENTIFIER_PRELUDE = "FILE";
+        
+    public FileIdentifierFactory()
+    { 
+	super(FILE_IDENTIFIER_PRELUDE);
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java	(revision 14150)
@@ -0,0 +1,113 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+/**
+ *  Implement a simple, unordered, namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class GSDL3Namespace extends SimpleNamespace
+{
+    public static String GSDL3_NAMESPACE_ID = "gsdl3";
+    private static String GSDL3_METADATA_TAG = "Metadata";
+    
+    public GSDL3Namespace()
+    { 
+	super(GSDL3_NAMESPACE_ID);
+    }
+    
+    public GSDL3Namespace(METSLocation location)
+    { 
+	super(GSDL3_NAMESPACE_ID, location);
+    }
+    
+    public GSDL3Namespace(Element mdWrapTag)
+    {	
+	super(GSDL3_NAMESPACE_ID);
+	
+	NodeList childNodes = mdWrapTag.getChildNodes();
+	for (int c = 0; c < childNodes.getLength(); c ++){
+	    // a metadata node
+	    if (childNodes.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE){
+		String metadataLabel = childNodes.item(c).getNodeName();
+
+		// cull any leading "GSDL3" tag and its colon...
+		if (metadataLabel.startsWith(GSDL3_NAMESPACE_ID + ":")){
+		    metadataLabel = metadataLabel.substring(this.name.length() + 1);
+		}
+		
+		// TODO: throw error: bad metadata type
+		if (metadataLabel.equals(GSDL3_METADATA_TAG) == false){
+		    continue;
+		}
+		
+		// Get the label - if the name isn't set, abort.
+		// TODO: throw error: incomplete metadata tag
+		metadataLabel = ((Element) childNodes.item(c)).getAttribute("name");
+		if (metadataLabel == null) {
+		    continue;
+		}
+		
+		// now build the associated value.
+		String metadataValue = "";
+		
+		Element childElement = (Element) childNodes.item(c);
+		for (int i = 0; i < childElement.getChildNodes().getLength(); i ++) {
+		    metadataValue = metadataValue + childElement.getChildNodes().item(i).toString();
+		}
+		
+		this.addMetadata(metadataLabel, metadataValue);
+	    }
+	}
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, String label)
+    {	
+	if (this.location == null){
+	    // just place the metadata in a simple wrapper
+	    Iterator values = this.metadataMap.getAll(label).iterator();
+	    
+	    while (values.hasNext()){
+		String tag;
+		String value = values.next().toString();
+		
+		tag = XMLTools.getOpenTag(GSDL3_NAMESPACE_ID, GSDL3_METADATA_TAG);
+		tag = XMLTools.addAttribute(tag, "name", label);
+		writer.print(tag);
+		writer.print(value);
+		
+		writer.println(XMLTools.getCloseTag(this.name, GSDL3_METADATA_TAG));
+	    }
+	}
+	else {
+	    String tag = XMLTools.getOpenTag("mets", "mdRef");
+	    tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	    tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", this.getName());
+	    tag = XMLTools.makeSingleton(tag);
+	    writer.println(tag);
+	}
+	return true;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java	(revision 14150)
@@ -0,0 +1,11 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  METS Administrative metadata
+ *
+ *  May contain zero or more instances of administrative metadata
+ */
+
+public class METSAdmin
+{
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java	(revision 14150)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSAgent
+{
+	String name;
+	String note;
+
+	String role;
+	String otherrole;
+	String type;
+	String othertype;
+
+	public METSAgent(String name, String note, String role, String type)
+	{	this.name = name;
+		this.note = note;
+
+		this.setRole(role);
+		this.setType(type);
+	}
+
+	/**
+	 *  Check if the role matches one of the standard offerings
+	 */
+	private boolean validateRole(String role)
+	{	return true;
+	}
+
+	/**
+	 *  Check if the type matches one of the standard offerings
+	 */
+	private boolean validateType(String role)
+	{	if (role.equals("INDIVIDUAL") ||
+		    role.equals("ORGANIZATION")) {
+			return true;
+		}
+		return false;
+	}
+
+	public void setRole(String otherRole)
+	{	// automatically use 'OTHER' if either the role or type of agent don't
+		// validate against the standard list
+		if (this.validateRole(role)) {
+			this.role = role;
+			this.otherrole = null;
+		}
+		else {
+			this.role = "OTHER";
+			this.otherrole = role;
+		}
+	}
+
+	public void setType(String otherType)
+	{	// automatically use 'OTHER' if either the role or type of agent don't
+		// validate against the standard list
+		if (this.validateType(type)) {
+			this.type = type;
+			this.othertype = null;
+		}
+		else {
+			this.type = "OTHER";
+			this.othertype = type;
+		}
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDate.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDate.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDate.java	(revision 14150)
@@ -0,0 +1,153 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSDate
+{
+	String era;
+	String year;
+	String month;
+	String day;
+	String hour;
+	String minute;
+	String second;
+	String timezone;
+
+	public void METSDate(String string)
+	{	String date;
+
+		int timeAt = string.indexOf('T');
+
+		// if we've got a time signature, get it
+		if (timeAt >= 0) {
+		  // get the date for later use
+			date = string.substring(0, timeAt);
+
+			// extract the time string
+			String time = string.substring(timeAt+1);
+
+			// check for time zone usage - currently '+' and '-' only...
+			int zoneAt = time.indexOf('+');
+			if (zoneAt == -1) {	
+				zoneAt = time.indexOf('-');
+			}
+			if (zoneAt >= 0) {
+				this.timezone = time.substring(zoneAt);
+				time = time.substring(0, zoneAt);
+			}
+
+			// finally, divide into hours, minutes etc.
+			int hourAt = time.indexOf(':');
+			if (hourAt >= 0) {
+				this.hour = time.substring(0, hourAt);
+
+				time = time.substring(hourAt+1);
+				int minuteAt = time.indexOf(':');
+				if (minuteAt >= 0)
+				{	this.minute = time.substring(0, minuteAt);
+					this.second = time.substring(minuteAt+1);
+				}
+				else
+				{ this.minute = time;
+					this.second = null;
+				}
+			}
+		}
+		else 
+		{ this.timezone = null;
+			this.hour     = null;
+			this.minute   = null;
+			this.second   = null;
+
+			date = string;
+		}
+
+		// get the era
+		int prefixEnds = 0;
+		while (!Character.isDigit(date.charAt(prefixEnds)))
+		{ prefixEnds ++;
+		}
+
+		if (prefixEnds != 0) {
+			this.era = date.substring(0, prefixEnds);
+			date = date.substring(prefixEnds);
+		}
+		else {
+			this.era = null;
+		}
+
+		// finally, get the date
+		
+		// now try dividing up by '-'
+		int yearAt = date.indexOf('-');
+		if (yearAt >= 0) {
+			this.year = date.substring(0, yearAt);
+			date = date.substring(yearAt+1);
+
+			int monthAt = date.indexOf('-');
+			if (monthAt >= 0) {
+				this.month = date.substring(0, monthAt);
+				this.day = date.substring(monthAt+1);
+			}
+			else {
+				this.month = date;
+				this.day   = null;
+			}
+		}
+		else {
+			// try dividing by YYYYMMDD - any part of which may be missing...
+			if (date.length() >= 4) {
+				this.year = date.substring(0, 4);
+				if (date.length() >= 6) {
+					this.month = date.substring(4, 6);
+					this.day = date.substring(6);
+				}
+				else {
+					this.month = date.substring(4);
+					this.day   = null;
+				}
+			}
+			// we just have a day in this case...
+			else {
+				this.year = date;
+				this.month = null;
+				this.day = null;
+			}
+		}
+	}
+
+	public String toString()
+	{	if (this.year == null) {
+			return "";
+		}
+
+		StringBuffer buffer = new StringBuffer();
+
+		if (this.era != null) {
+			buffer.append(era);
+		}
+		buffer.append(year);
+		if (this.month != null) {
+			buffer.append("-");
+			buffer.append(this.month);
+			if (this.day != null) {
+				buffer.append("-");
+				buffer.append(this.day);
+
+				if (this.hour != null) {
+					buffer.append("T");
+					buffer.append(this.hour);
+					buffer.append(":");
+					buffer.append(this.minute);
+					if (this.second != null) {
+						buffer.append(":");
+						buffer.append(this.second);
+
+						if (this.timezone != null) {
+							buffer.append(this.timezone);
+						}
+					}
+				}
+			}
+		}
+		return buffer.toString();
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java	(revision 14150)
@@ -0,0 +1,494 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.PrintWriter;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Support the descriptive metadata of a given document
+ *
+ *  The descriptive metadata can occur multiple times, so this 'wrapper' is
+ *  in fact a holder for a number of individual metadata sets.  Individual
+ *  metadata groups are found in METSNamespace items.
+ *
+ *  
+ */
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+public class METSDescriptive
+{ 
+    MultiMap map;
+    String   ID;
+    String   name;
+    
+    public METSDescriptive(String ID, String name)
+    { 
+	this.map  = new MultiMap();
+	this.ID   = ID;
+	this.name = name;
+    }
+    
+    public String getID()
+    { 
+	return this.ID;
+    }
+    
+    public String getName()
+    { 
+	return this.name;
+    }
+    
+    public void addNamespace(METSNamespace namespace)
+    { 
+	this.map.put(namespace.getName(), namespace);
+    }
+    
+    public METSNamespace getNamespace(String namespace)
+    {
+	return (METSNamespace) this.map.get(namespace);
+    }
+    
+    /**
+     *  Get the version of a metadata namespace that can be used for
+     *  adding new values to the document.  Remember, each namespace may
+     *  occur more than once, but only one of the occurrences can be 'open'.
+     *
+     *  @param <code>String</code> the name of the namespace
+     */
+    public METSNamespace getOpenNamespace(String namespace)
+    { 
+	if (this.map.getAll(namespace) == null) {
+	    return null;
+	}
+	
+	Iterator namespaces = ((List) this.map.getAll(namespace)).iterator();
+	
+	while (namespaces.hasNext()) {
+	    METSNamespace namespaceData = (METSNamespace) namespaces.next();
+	    
+	    if (namespaceData.isEditable()){
+		return namespaceData;
+	    }
+	}
+	return null;
+    }
+    
+    /**
+     *  Add a piece of metadata to this document.  Existing values are preserved.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata   
+     *  @param <code>String</code> the value of the metadata
+     */
+    public void addMetadata(String namespace, String label, String value)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData == null ||
+	    namespaceData.isEditable() == false) {
+	    namespaceData = NamespaceFactory.initNamespace(namespace);
+	    this.addNamespace(namespaceData);
+	}
+	namespaceData.addMetadata(label, value);
+    }
+    
+    /**
+     *  Add a piece of metadata to this document.  Existing values in editable
+     *  parts of the document's metadata are destroyed.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata   
+     *  @param <code>String</code> the value of the metadata
+     */
+    public void setMetadata(String namespace, String label, String value)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData == null ||
+	    namespaceData.isEditable() == false) {
+	    namespaceData = NamespaceFactory.initNamespace(namespace);
+	    this.addNamespace(namespaceData);
+	}
+	namespaceData.setMetadata(label, value);
+    }
+    
+    /**
+     *  Remove the metadata for a particular label.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata
+     */
+    public void removeMetadata(String namespace, String label)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData != null && namespaceData.isEditable() == true) {
+	    namespaceData.removeMetadata(label);
+	}
+    }
+    
+    /**
+     *  Fetch a <code>List</code> of the values matching the given label in
+     *  the requisite namespace.
+     *
+     *  @param <code>String</code> namespace
+     *  @param <code>String</code> label of metadata to match
+     *
+     *  @return <code>List</code> the corresponding values found.
+     */
+    public List getMetadata(String namespace, String label)
+    {
+	// Simple case - no metadata
+	if (!this.map.containsKey(namespace)){
+	    return null;
+	}
+	
+	// if there's just one instance of the requisite namespace,
+	// then again it is a simple case
+	if (this.map.getAll(namespace).size() == 1){
+	    return this.getNamespace(namespace).getMetadata(label);
+	}
+	
+	// otherwise, step through and accumulate
+	List accumulatedValues = new ArrayList();
+	Iterator namespaceIterator = this.map.getAll(namespace).iterator();
+	while (namespaceIterator.hasNext()){
+	    METSNamespace localNamespace = (METSNamespace) namespaceIterator.next();
+	    
+	    List localList = localNamespace.getMetadata(label);
+	    if (localList != null && localList.size() > 0){
+		accumulatedValues.addAll(localList);
+	    }
+	}
+	
+	// if the result of the accumulation is empty, return a null item
+	if (accumulatedValues.size() == 0){
+	    return null;
+	}
+	
+	return accumulatedValues;
+    }
+    
+    protected void parse_xmlData(METSNamespace namespace, Element element) 
+    {
+	if (namespace==null) {
+	    System.err.println("namespace is null!!!!");
+	}
+	NodeList children = element.getChildNodes();
+	
+	for (int c = 0; c < children.getLength(); c++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child  = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    // if (childName.equals("gsdl2:Metadata")) {
+	    if (childName.equals("gsdl3:Metadata")) {
+		String metaName = child.getAttribute("name");
+		Text text = (Text) child.getFirstChild();
+		String metaValue = text.getData();
+		
+		addMetadata(namespace.getName(),metaName, metaValue);
+	    }
+	}
+    }
+    
+    public void parse_mdWrap(Element element)
+    {
+	// deal with mets:mdWrap
+	
+	METSNamespace namespace = NamespaceFactory.parseXML(element);
+	
+	NodeList children = element.getChildNodes();
+	
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child  = (Element) children.item(c);
+	    String childName = child.getNodeName();
+
+	    if (childName.equals("mets:xmlData")) {
+		parse_xmlData(namespace, child);
+	    } else {
+		System.err.println("Warning: unrecognised tag " + childName 
+				   + "in mets:mdWrap");
+	    }
+	}
+    }
+
+    /**
+     *  Parse an XML Element as a METS Descriptive Metadata section
+     *
+     *  @param <code>Element</code> the XML element which represents the dmdSec itself
+     */
+    public static METSDescriptive parseXML(Element element) 
+    {
+	// Note: no parsing of attributes required, we just move onto
+	// the metadata namespaces/sections themselves
+
+	// deal with mets:dmdSec
+	String ID = element.getAttribute("ID");
+	String label = element.getAttribute("GROUPID");
+	
+	METSDescriptive thisDescriptive = new METSDescriptive(ID, label);
+
+	//thisDescriptive = new METSDescriptive(ID, label);
+	
+	NodeList children = element.getChildNodes();
+
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    if (childName.equals("mets:mdRef")) {
+		//for External Descriptive Metadata
+		METSNamespace namespace = NamespaceFactory.parseXML(element);
+		System.err.println("*** mets::mdRef does not appear to be implemented yet");
+	    } else if (childName.equals("mets:mdWrap")) {
+		thisDescriptive.parse_mdWrap(child);
+	    } else {
+		// TODO: raise an error!
+		System.out.println("Error: METSDescriptive: unrecognised tag "+childName);
+	    }
+	}
+	return thisDescriptive;
+    }
+    /**
+     *  Write the document metadata to a <code>PrintWriter</code> in METS
+     *  XML format.
+     *  
+     *  @param <code>PrintWriter</code> the writer to use for output
+     */
+    public void write(PrintWriter output)
+    { 
+	String tag = XMLTools.getOpenTag("mets", "dmdSec");
+	tag = XMLTools.addAttribute(tag, "ID", this.ID);
+	tag = XMLTools.addAttribute(tag, "GROUPID", this.name);
+	output.println(tag);
+	
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()){
+	    Object nextKey = keys.next();
+	    if (nextKey == null){
+		continue;
+	    }
+	    String key = nextKey.toString();
+	
+	    // get every copy of the current namespace name - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+	    
+	    // namespaces will write themselves...
+	    while (namespaces.hasNext()){
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+	
+		namespace.write(output);
+	    }
+	}
+	
+	tag = XMLTools.getCloseTag("mets", "dmdSec");
+	output.println(tag);
+    }
+    
+    public void copy_metadata(METSDescriptive dst) 
+    {
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()) {
+	    Object nextKey = keys.next();
+	    if (nextKey == null) {
+		continue;
+	    }
+	    String key = nextKey.toString();
+	    
+	    // get every copy of the current namespace name - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+      
+	    while (namespaces.hasNext()) {
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+		
+		// for each metadata name 
+
+		Iterator mdnIter = namespace.getMetadataNames();
+		while (mdnIter.hasNext()) {
+
+		    String md_name = mdnIter.next().toString();
+		    
+		    List values = getMetadata(key, md_name);
+		    
+		    if (values != null) {	
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    dst.addMetadata("gsdl3",md_name,value);
+			}
+		    }
+		}
+	    }
+	}
+    }
+
+
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	// check if this node is in the
+	int sqlId = -1;
+	
+	//    System.out.println("Writing " + connection.toString());
+	
+	GS3SQLSelect select = new GS3SQLSelect("metadata");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetaID", "=", this.ID);
+	GS3SQLWhereItem item = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    GS3SQLAction action;
+	    
+	    if (resultSet.first()) {
+		// the node already exists - no need to do anything as such
+		GS3SQLUpdate update = new GS3SQLUpdate("metadata");
+		update.setWhere(where);
+		action = update;
+	    }
+	    else { // Set result set to null, just in case first() didn't work above...
+		// It is a new node and needs writing
+		action = new GS3SQLInsert("metadata");
+		
+		action.addValue("DocID", document.getID().toString());
+		action.addValue("MetaID", this.ID);
+	    }
+	    
+	    // always set the group identifier
+	    action.addValue("GroupID", this.name);
+	    
+	    // execute the action as required
+	    statement.execute(action.toString());
+	    
+	    // get the resultSet again
+	    // now execute the select statement again to get the new identifier
+	    // 'MetadataRef'
+	    // System.out.println(select.toString());
+	    resultSet = statement.executeQuery(select.toString());
+	    
+	    if (resultSet.first()) {
+		// get the reference for this item...
+		sqlId = resultSet.getInt("MetadataRef");
+	    }
+	    statement.close();
+	}
+	catch (SQLException sql){
+	    System.err.println("METSDescriptive.writeSQL(): "+sql);
+	}
+	if (sqlId == -1) {
+	    return false;
+	}
+	
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()){
+	    Object nextKey = keys.next();
+	    if (nextKey == null){
+		continue;
+	    }
+	    String key = nextKey.toString();
+	    
+	    // get every current namespace - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+	    
+	    // namespaces will write themselves...
+	    while (namespaces.hasNext()){
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+		
+		if (!namespace.writeSQL(sqlId, connection)){
+		    return false;
+		}
+	    }
+	}
+	return true;
+    }
+    
+    public static METSDescriptive readSQL(DocumentInterface document, GS3SQLConnection connection,
+					  ResultSet resultSet)
+    {
+	try {
+	    String ID   = resultSet.getString("MetaID");
+	    String name = resultSet.getString("GroupID");
+	    
+	    // create the metadata block object
+	    METSDescriptive descriptive = new METSDescriptive(ID, name);
+	    
+	    // get its metadata reference to retrieve namespaces
+	    int metadataRef = resultSet.getInt("MetadataRef");
+	    
+	    // query the database for matching namespaces for this metadata block
+	    GS3SQLSelect select = new GS3SQLSelect("namespaces");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(metadataRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    select.setWhere(where);
+	    
+	    Statement statement = connection.createStatement();
+	    ResultSet namespaceSet = statement.executeQuery(select.toString());
+	    
+	    // parse through the namespaces, calling the namespace class to create instances as it will
+	    if (namespaceSet.first()) {
+		do {
+		    METSNamespace namespace = METSNamespace.readSQL(connection, namespaceSet);
+		    if (namespace != null) {
+			descriptive.addNamespace(namespace);
+		    }
+		    else {
+			System.err.println("Null namespace output");
+		    }
+		}
+		while (namespaceSet.next());
+	    }
+	    statement.close();
+	    return descriptive;
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSDescriptive.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return null;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java	(revision 14150)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class METSDescriptiveSection
+{ 
+    METSLocation location;
+    String       type;
+    String       otherType;
+    
+    public METSDescriptiveSection(String type)
+    {	
+	this.location  = null;
+	this.type      = type;
+	this.otherType = null;
+    }
+	
+    public METSDescriptiveSection(METSLocation location, String type)
+    {	
+	this.location  = location;
+	this.type      = type;
+	this.otherType = null;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java	(revision 14150)
@@ -0,0 +1,266 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSDescriptiveSet
+{ 
+    Map children;
+    Map childrenById;
+    DescriptiveIdentifierFactory identifierFactory;
+    
+    class DescriptiveIdentifierFactory extends AbstractIdentifierFactory
+    { 
+	public DescriptiveIdentifierFactory()
+	{ 
+	    super("DM");
+	}
+    }
+    
+    public METSDescriptiveSet()
+    { 
+	this.children = new HashMap();
+	this.childrenById = new HashMap();
+	
+	this.identifierFactory = new DescriptiveIdentifierFactory();
+	
+	this.createDescriptive("default");
+    }
+    
+    /**
+     *  Create a new descriptive child, adding it automatically to this descriptive set.
+     *
+     *  @param <code>String</code> a name to describe the descriptive set.
+     */
+    public METSDescriptive createDescriptive(String name)
+    { 
+	METSDescriptive descriptive = new METSDescriptive(this.identifierFactory.getNextIdentifier(), name);
+	
+	this.children.put(name, descriptive);
+	this.childrenById.put(descriptive.getID(), descriptive);
+	return descriptive;
+    }
+    
+    /**
+     *  Add a descriptive item to this set.
+     *
+     *  @param <code>METSDescriptive</code> the descriptive set
+     */
+    public void addDescriptive(METSDescriptive metadata)
+    { 
+	// insert the item into both maps...
+	this.children.put(metadata.getName(), metadata);
+	this.childrenById.put(metadata.getID(), metadata);
+	
+	// when a descriptive block is added, check that it hasn't been used before...
+	//this.identifierFactory.noteIdentifier(metadata.getID());
+	this.identifierFactory.noteIdentifier(metadata.getName());
+    }
+    
+    /**
+     *  Get a descriptive item by its name(GROUPID in METS file)
+     */
+    public METSDescriptive getDescriptive(String name)
+    { 
+	
+	return (METSDescriptive) this.children.get(name);
+    }
+    
+    /**
+     *  Get a descriptive item by its identifier
+     */
+    public METSDescriptive getDescriptiveById(String id)
+    { 
+	return (METSDescriptive) this.childrenById.get(id);
+    }
+    
+    public void addNamespace(String descriptiveName, METSNamespace namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	if (descriptive != null){
+	    descriptive.addNamespace(namespace);
+	}
+    }
+    
+    public METSNamespace getNamespace(String descriptiveName, String namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	if (descriptive != null){
+	    return descriptive.getNamespace(namespace);
+	}
+	return null;
+    }
+    
+    public METSNamespace getOpenNamespace(String descriptiveName, String namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    return descriptive.getOpenNamespace(namespace);
+	}
+	return  null;
+    }
+    
+    public void addMetadata(String descriptiveName, String namespace, String label, String value)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.addMetadata(namespace, label, value);
+	}
+    }
+    
+    public void setMetadata(String descriptiveName, String namespace, String label, String value)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.setMetadata(namespace, label, value);
+	}
+    }
+    
+    public void removeMetadata(String descriptiveName, String namespace, String label)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.removeMetadata(namespace, label);
+	}
+    }
+    
+    public void removeAllMetadata(String namespace, String label)
+    { 
+	Iterator descriptiveIter = this.children.values().iterator();
+	
+	while (descriptiveIter.hasNext()) {
+	    METSDescriptive descriptive = (METSDescriptive) descriptiveIter.next();
+	    
+	    descriptive.removeMetadata(namespace, label);
+	}
+    }
+    
+    public List getMetadata(String descriptiveName, String namespace, String label)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    return descriptive.getMetadata(namespace, label);
+	}
+	return null;
+    }
+    
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDescriptive group = (METSDescriptive) groups.next();
+	    
+	    group.write(writer);
+	}
+    }
+    
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDescriptive group = (METSDescriptive) groups.next();
+	    
+	    if (!group.writeSQL(document, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    public static METSDescriptiveSet parseXML(NodeList dmdSecs) 
+    {
+	METSDescriptiveSet set = new METSDescriptiveSet();
+	
+	for (int g = 0; g < dmdSecs.getLength(); g ++) {
+	    METSDescriptive metadata = METSDescriptive.parseXML((Element) dmdSecs.item(g));
+	    
+	    set.addDescriptive(metadata);
+	}
+    
+	System.err.println("******** Away to do transfer!!!!!!");
+	  
+	METSDescriptive top_level_metadata = set.getDescriptive("1");
+	METSDescriptive tl2_metadata = set.getDescriptive("1");
+	METSDescriptive opo_metadata = set.getDescriptive("1.1");
+	METSDescriptive default_metadata = set.getDescriptive("default");
+	
+	if (top_level_metadata != null) {
+	    if (default_metadata != null) {
+		top_level_metadata.copy_metadata(default_metadata);
+	    }
+	    else {
+		System.err.println("Warning: Could not find default metadata");
+	    }
+	}
+	else {
+	    System.err.println("Warning: Could not find top level metadata for groupID=1");
+	}
+	return set;
+    } 
+    
+    public static METSDescriptiveSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    {
+	METSDescriptiveSet set = new METSDescriptiveSet();
+	
+	GS3SQLSelect select = new GS3SQLSelect("metadata");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+
+	// start going through the matching metadata blocks
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSDescriptive descriptive = METSDescriptive.readSQL(document, connection, resultSet);
+		    if (descriptive != null) {
+			set.addDescriptive(descriptive);
+		    }
+		    else {
+			System.out.println("Null descriptive");
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSDescriptiveSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	
+	
+	return set;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java	(revision 14150)
@@ -0,0 +1,681 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSDivision extends AbstractStructure
+{
+    String       order;       // see the METS documentation for the meaning of the different
+    String       orderLabel;  // label types.
+    String       userLabel;
+    String       type;        // e.g. section, chapter, physical or logical, etc.
+    List         metadataRefs;
+    List         adminRefs;
+    List         fileRefs;
+    
+    public static final String DIVISION_TYPE  = "Division";
+    public static final String DIVISION_TAG   = "div";
+    public static final String FPTR_TAG       = "fptr";
+    
+    public METSDivision(String ID, String order, String orderLabel, String userLabel, String type)
+    { 
+	super(ID);
+	
+	this.order = order;
+	this.orderLabel = orderLabel;
+	this.userLabel = userLabel;
+	this.type = type;
+	this.fileRefs = new ArrayList();
+	this.metadataRefs = new ArrayList();
+    }
+    
+    public String getStructureType()
+    { 
+	return DIVISION_TYPE;
+    }
+    
+    public String getLabel()
+    { 
+	return this.orderLabel;
+    }
+    
+    /**
+     *  Find all the divisions that match a given list of file group
+     *  identifiers...
+     *
+     *  @param <code>List</code> the list of identifiers to find
+     *  @param <code>List</code> the list into which to place any
+     *         matching divisions.
+     */
+    public void findDivisionsForFiles(List listOfFileIdentifiers, List resultList)
+    { 
+	Iterator iterator = this.fileRefs.iterator();
+	
+	// get the references on this division, and mark if necessary
+	while (iterator.hasNext()) {
+	    String reference = iterator.next().toString();
+	    
+	    Iterator fileIterator = listOfFileIdentifiers.iterator();
+	    while (fileIterator.hasNext()){
+		String fileRef = fileIterator.next().toString();
+		
+		if (fileRef.equals(reference)){
+		    resultList.add(this);
+		}
+	    }
+	}
+	
+	// get the child groups, and check each in turn...
+	Iterator groups = this.children.values().iterator();
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    group.findDivisionsForFiles(listOfFileIdentifiers, resultList);
+	}    
+    }
+    
+    /**
+     *  Get the namespace from the given set of descriptive blocks...
+     */
+    public METSNamespace findNamespace(String namespaceName, boolean isOpenNamespace, 
+				       METSDescriptiveSet descriptiveSet)
+    { 
+	METSNamespace namespace = null;
+	
+	// Special case - no existing metadata references
+	if (this.metadataRefs.size() == 0) {
+	    // Create new Descriptive and add it to the descriptive set
+	    METSDescriptive descriptive = new METSDescriptive("DM"+this.ID, this.orderLabel);
+	    descriptiveSet.addDescriptive(descriptive);
+	    
+	    // Create the namespace and add it to the Descriptive
+	    namespace = NamespaceFactory.initNamespace(namespaceName);
+	    if (namespace != null){
+		descriptive.addNamespace(namespace);
+	    }
+	    
+	    return namespace;
+	}
+	
+	Iterator descriptiveIterator = this.metadataRefs.iterator();
+	
+	while (descriptiveIterator.hasNext()){
+	    String descriptiveRef = descriptiveIterator.next().toString();
+	    
+	    METSDescriptive descriptive = descriptiveSet.getDescriptiveById(descriptiveRef);
+	    
+	    if (isOpenNamespace) {
+		namespace = descriptive.getOpenNamespace(namespaceName);
+	    }
+	    else {
+		namespace = descriptive.getNamespace(namespaceName);
+	    }
+	    
+	    if (namespace != null)
+		break;
+	}
+	
+	// Special case - no such namespace in any descriptive list...
+	if (namespace == null){
+	    // take the first descriptive area...
+	    METSDescriptive descriptive = descriptiveSet.getDescriptiveById(this.metadataRefs.get(0).toString());
+	    
+	    // create the namespace, and add it to the METSDescriptive
+	    namespace = NamespaceFactory.initNamespace(namespaceName);
+	    if (namespace != null){
+		descriptive.addNamespace(namespace);
+	    }
+	}
+	
+	return namespace;
+    }
+    
+    /**
+     *  Add a new file reference to the structure
+     *
+     *  @param <code>METSFileID</code> the file to be referenced.
+     */
+    public void addFileReference(METSFileID fileRef)
+    { 
+	this.addFileReference(fileRef.toString());
+    }
+    
+    public void addFileReference(String fileRef)
+    { 
+	this.fileRefs.add(fileRef);
+    }
+    
+    /**
+     *  Add a new metadata reference to the structure
+     */
+    public void addMetadataReference(String metadataRef)
+    { 
+	// TODO: correct the type
+	this.metadataRefs.add(metadataRef);
+    }
+
+    /**
+     *  Get all the metadata references
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    public List getMetadataReferences()
+    { 
+	return this.metadataRefs;
+    }
+
+    /**
+     *  Get all the default (first) metadata reference...
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    //TODO: check for this first *open* metadata item...
+    public String getDefaultMetadataReference()
+    { 
+	if (this.metadataRefs.size() == 0) {
+	return null;
+    }
+    return this.metadataRefs.get(0).toString();
+    }
+
+    /**
+     *  Get all the default (first) first filegroup reference...
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    public String getDefaultFileReference()
+    { 
+	if (this.fileRefs.size() == 0) {
+	return null;
+    }
+    return this.fileRefs.get(0).toString();
+    }
+
+    /**
+     *  Get a child or descendant division
+     *
+     *  @param <code>String</code> the path to the child division
+     *  @return <code>METSDivision</code> the division found.
+     */
+    public METSDivision getDivision(String divisionLabel)
+    { 
+	// if the division ID equals this, then process it
+	if (divisionLabel.equals(this.orderLabel)) {
+	    return this;
+	}
+
+	// if we just shouldn't be here, then error...
+	if (!divisionLabel.startsWith(this.orderLabel + ".")) {
+	    return null;
+	}
+
+	// get the next division in the division path...
+	String subdivisionPath = divisionLabel.substring(this.orderLabel.length() + 1);
+	String nextDivision = getTopDivisionName(subdivisionPath);
+	String nextLabel;
+
+	// first try the current division name with an appended '.' and the next child id...
+	METSDivision child = (METSDivision) this.children.get(this.orderLabel + "." + nextDivision);
+	if (child == null){
+	    // if that didn't work, try the next division on its own
+	    child = (METSDivision) this.children.get(nextDivision);
+	    
+	    // if neither worked, then abort...
+	    if (child == null) {
+		return null;
+	    }
+	    else {
+		// set the next identifier
+		nextLabel = METSDivision.getSubDivisionName(divisionLabel);
+	    }
+	} 
+	else {
+	    // set the next identifier...
+	    nextLabel = divisionLabel;
+	}
+	
+	return child.getDivision(nextLabel);
+    }
+    
+    /**
+     *  A convenience method for splitting a division name - this gives the name
+     *  without the leading division label...  To be used when division identifiers
+     *  are given as '.' separated paths and division names do not contain the
+     *  full path of their parents.
+     *
+     *  @return <code>String</code>
+     */
+    public static String getSubDivisionName(String divisionName)
+    { 
+	int at = divisionName.indexOf('.');
+
+	if (at >= 0){
+	    return divisionName.substring(at+1);
+	}
+	else {
+		return null;
+	}
+    }
+
+    /**
+     *  Get the parent division name.  Division identifiers
+     *  concatenate the ancestor's names with intervening
+     *  '.' characters.
+     *
+     *  @return <code>String</code> the division name
+     */
+    public static String getParentName(String divisionName)
+    { 
+	int at = divisionName.lastIndexOf('.');
+
+	if (at >= 0) {
+	    return divisionName.substring(0, at);
+	} 
+	return "";
+    }
+    
+    /**
+     *  Get the top division name.
+     *
+     *  @return <code>String</code> the division name
+     */
+    public static String getTopDivisionName(String divisionName)
+    { 
+	int at = divisionName.indexOf('.');
+
+	if (at >= 0){
+	    return divisionName.substring(0, at);
+	}
+	else {
+	    return divisionName;
+	}
+    }
+
+    /*  Parse an XML Element as a METSDivision section
+     *
+     *  @param <code>Element</code> the XML element which represents
+     *  the mets:div itself 
+     */
+    
+    public static METSDivision parseXML(Element element) 
+    {
+ 	METSDivision thisDivision = null;
+ 
+ 	String divId = element.getAttribute("ID");
+ 	String divType = element.getAttribute("TYPE");
+ 	String divOrder = element.getAttribute("ORDER");
+ 	String divOrderLabel = element.getAttribute("ORDERLABEL");
+ 	String divUserLabel = element.getAttribute("LABEL");
+ 	String divMetaRef = element.getAttribute("DMDID");
+ 	
+ 	thisDivision = new METSDivision(divId, divOrder,divOrderLabel, divUserLabel, divType);
+ 		
+ 	thisDivision.addMetadataReference(divMetaRef);
+ 
+ 	NodeList children = element.getChildNodes();
+ 
+ 	for (int c = 0; c < children.getLength(); c++){
+ 	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+ 	    Element child = (Element) children.item(c);
+ 	    String childName = child.getNodeName();
+ 
+ 	    if (childName.equals("mets:fptr")) {
+ 		
+ 		METSDivision.fptr_parseXML(thisDivision, child);
+ 	    } else if (childName.equals("mets:div")) {
+ 		METSDivision division = METSDivision.parseXML(child);
+ 		thisDivision.addDivision(division);
+ 	    } else {
+ 		System.err.println("Error: mets:div expected mets:div or mets:fptr, but got " + childName);
+ 	    } 
+ 	}
+ 	return thisDivision;
+    }
+ 
+ 
+    /**
+     * Parse an XML element as a  METSFptr Section
+     * @param <code>Element</code> the XML element which represents the mets:fptr itself
+     */
+    private static void fptr_parseXML(METSDivision thisDivision, Element element) 
+    {
+ 	String fileRef = element.getAttribute("FILEID");
+ 	thisDivision.addFileReference(fileRef);
+    }
+ 
+    /**
+
+
+    /**
+    *  Write the METS file in an XML to a text-output sink.
+    *
+    *  @param <code>PrintWriter</code> the destination of the output
+    */
+    public boolean write(PrintWriter writer)
+    { 
+	// Get the main tag for the division
+	String tag = XMLTools.getOpenTag("mets", DIVISION_TAG);
+	tag = XMLTools.addAttribute(tag, "ID", this.ID.toString());
+	tag = XMLTools.addAttribute(tag, "TYPE", this.type);
+	tag = XMLTools.addAttribute(tag, "ORDER", this.order);
+	if (this.orderLabel != null){
+	    tag = XMLTools.addAttribute(tag, "ORDERLABEL", this.orderLabel);
+	}
+	if (this.userLabel != null){ 
+	    tag = XMLTools.addAttribute(tag, "LABEL", this.userLabel);
+	}
+	
+	// Add a metadata reference attribute
+	if (this.metadataRefs.size() > 0){
+	    String metaList = this.writeList(this.metadataRefs);
+	    tag = XMLTools.addAttribute(tag, "DMDID", metaList);
+	}
+	
+	// write the central tag
+	writer.print("   ");  //indentation
+	writer.println(tag);
+
+	// write the list of file pointers for this structural element
+	if (this.fileRefs.size() > 0) {
+	    String fileList = this.writeTagList("mets", "fptr", "FILEID", this.fileRefs);
+	    writer.print("      ");  //indentation
+	    writer.println(fileList);
+	}
+	
+	// write out any children in turn
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.write(writer)) {
+		return false;
+	    }
+	}
+    
+	// close the div element
+	writer.print("   ");  //indentation
+	writer.println(XMLTools.getCloseTag("mets", DIVISION_TAG));
+
+	return true;
+    }
+
+    /**
+     *  A convenience method for writing a <code>List</code> as a series of tags...
+     */
+    private String writeTagList(String namespace, String tagname, 
+				String attribute, List list)
+    { 
+	StringBuffer listString = new StringBuffer();
+	
+	Iterator iterator = list.iterator();
+	while (iterator.hasNext()){
+	    String item = iterator.next().toString();
+	    
+	    String tag = XMLTools.getOpenTag(namespace, tagname);
+	    tag = XMLTools.addAttribute(tag, attribute, item);
+	    tag = XMLTools.makeSingleton(tag);
+	    
+	    listString.append(tag);
+	}
+	return listString.toString();
+    }
+
+    /**
+     *  A convenience method for writing a <code>List</code> as a space-separated <code>String</code>.
+     */
+    private String writeList(List list)
+    { 
+	StringBuffer listString = new StringBuffer();
+	
+	Iterator iterator = list.iterator();
+	while (iterator.hasNext()){
+	    String item = iterator.next().toString();
+	    
+	    if (listString.length() > 0) {
+		listString.append(" ");
+	    }
+	    
+	    listString.append(item);
+	}
+	return listString.toString();
+    }
+    
+    public boolean writeSQL(DocumentID docId, int parentRef, boolean parentIsStructure, 
+			    GS3SQLConnection connection)
+    { 
+	int sqlRef = -1;
+	
+	// set the "where" to find the reference for this division
+	GS3SQLSelect select = new GS3SQLSelect("divisions");
+	select.addField("DivisionRef");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(parentRef), GS3SQLField.INTEGER_TYPE);
+	GS3SQLWhereItem parentItem = new GS3SQLWhereItem("ParentType", "=", parentIsStructure ? METSStructure.STRUCTURE_TYPE : METSDivision.DIVISION_TYPE);
+	GS3SQLWhereItem item = new GS3SQLWhereItem("SectionID", "=", this.ID.toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(parentItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	Statement statement = null;
+	ResultSet resultSet = null;
+	
+	// Do the actual writing
+	GS3SQLAction action;
+	
+	try {
+	    statement = connection.createStatement();
+	    resultSet = statement.executeQuery(select.toString());
+	    if (!resultSet.first()) {
+		
+		GS3SQLInsert insert = new GS3SQLInsert("divisions");
+		insert.addValue("DocID", docId.toString());
+		insert.addValue("SectionID", this.ID.toString());
+		insert.addValue("ParentRef", Integer.toString(parentRef));
+		insert.addValue("ParentType", parentIsStructure == true ? METSStructure.STRUCTURE_TYPE : METSDivision.DIVISION_TYPE);
+		
+		action = insert;
+	    }
+	    else {
+		sqlRef = resultSet.getInt("DivisionRef");
+		
+		GS3SQLUpdate update = new GS3SQLUpdate("divisions");
+		GS3SQLWhere  updateWhere = 
+		    new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE));
+		update.setWhere(updateWhere);
+		action = update;
+	    }
+	}
+	catch (SQLException sql) {
+	    System.err.println(sql + " " + select.toString());
+	    return false;
+	}
+	
+	action.addValue("DivisionType", this.type.toString());
+	action.addValue("LabelOrder", this.order);
+	action.addValue("ShortLabel", this.orderLabel);
+	action.addValue("UserLabel", this.userLabel);
+	
+	try {
+	    statement.execute(action.toString());
+	} catch (SQLException e) {
+	    System.err.println("METSDivision.writeSQL(): "+e);
+	    return false;
+	}
+	
+	// if doing a fresh item, get the new structure reference...
+	if (sqlRef == -1){
+	    // get the new structure reference
+	    try {
+		// read in the structure reference
+		resultSet = statement.executeQuery(select.toString());
+		if (resultSet.first()) {
+		    sqlRef = resultSet.getInt("DivisionRef");
+		}
+	    }
+	    catch (SQLException sqlex) {
+		System.err.println("METSDIVISION.writeSQL(): Unable to retrieve reference for Division " + sqlex);
+		return false;
+	    }
+	}
+	
+	// delete the old file/metadata references
+	try {
+	    GS3SQLWhere referenceWhere = 
+		new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(sqlRef), 
+						    GS3SQLField.INTEGER_TYPE));
+	    
+	    GS3SQLDelete delete = new GS3SQLDelete("divisionfilerefs");
+	    delete.setWhere(referenceWhere);
+	    
+	    statement.execute(delete.toString());
+	    
+	    delete = new GS3SQLDelete("divisionmetarefs");
+	    delete.setWhere(referenceWhere);
+	    statement.execute(delete.toString());
+	
+	    // write the new file references
+	    if (this.fileRefs.size() > 0){
+		Iterator iterator = this.fileRefs.iterator();
+		
+		while (iterator.hasNext()) {
+		    GS3SQLInsert fileinsert = new GS3SQLInsert("divisionfilerefs");
+		    fileinsert.addValue("DocID", docId.toString());
+		    fileinsert.addValue("DivisionRef", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE);
+		    fileinsert.addValue("DivisionType", "Group");
+		    fileinsert.addValue("FileID", iterator.next().toString());
+		    statement.execute(fileinsert.toString());
+		}
+	    }
+	    
+	    // write the metadata references
+	    if (this.metadataRefs.size() > 0){
+		Iterator iterator = this.metadataRefs.iterator();
+		
+		while (iterator.hasNext()) {
+		    GS3SQLInsert metainsert = new GS3SQLInsert("divisionmetarefs");
+		    metainsert.addValue("DocID", docId.toString());
+		    metainsert.addValue("DivisionRef", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE);
+		    metainsert.addValue("DivisionType", "Group");
+		    metainsert.addValue("MetaID", iterator.next().toString());
+		    statement.execute(metainsert.toString());
+		}
+	    }
+	    statement.close();
+	} catch (SQLException e) {
+	    System.err.println("METSDIVISION.writeSQL(): "+e);
+	    return false;
+	}
+	// write out any children in turn
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.writeSQL(docId, sqlRef, false, connection)) {
+		return false;
+	    }
+	}
+	return true;
+    }
+    
+    public static METSDivision readSQL(GS3SQLConnection connection, ResultSet resultSet)
+    {
+	METSLocation metsLocation = null;
+	
+	GS3SQLSelect select = null;
+	try {
+	    // read the basic information
+	    String ID   = resultSet.getString("SectionID");
+	    String type = resultSet.getString("DivisionType");
+	    String order = resultSet.getString("LabelOrder"); 
+	    String orderLabel = resultSet.getString("ShortLabel");
+	    String userLabel  = resultSet.getString("UserLabel");
+      
+	    // construct the division object
+	    METSDivision division = new METSDivision(ID, order, orderLabel, userLabel, type);
+      
+	    // obtain a list of child divisions
+	    int divisionRef = resultSet.getInt("DivisionRef");
+
+	    select = new GS3SQLSelect("divisions");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(divisionRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSDivision.DIVISION_TYPE);
+	    where.add(whereItem);
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet childSet = statement.executeQuery(select.toString());
+      
+	    // circulate through to obtain further children
+	    if (childSet.first())
+		{
+		    do {
+			METSDivision childDivision = METSDivision.readSQL(connection, childSet);
+			division.addDivision(childDivision);
+		    }
+		    while (childSet.next());
+		}
+	    select = new GS3SQLSelect("divisionfilerefs");
+	    select.addField("*");
+	    where = new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(divisionRef), 
+							GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    ResultSet fileSet = statement.executeQuery(select.toString());
+
+	    if (fileSet.first()){
+		do {
+		    String reference = fileSet.getString("FileID");
+		    division.fileRefs.add(reference);
+		}
+		while (fileSet.next());  
+	    }
+	    
+	    select = new GS3SQLSelect("divisionmetarefs");
+	    select.addField("*");
+	    where = new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(divisionRef), 
+							GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    ResultSet metaSet = statement.executeQuery(select.toString());
+
+	    if (metaSet.first()){
+		do {
+		    String reference = metaSet.getString("MetaID");
+		    division.metadataRefs.add(reference);
+		}
+		while (metaSet.next());  
+	    }
+        
+	    statement.close();
+	    return division;
+	}
+	catch (SQLException sqlEx){
+	    System.out.println(sqlEx + " " +select.toString());
+	}    
+	return null;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java	(revision 14150)
@@ -0,0 +1,75 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A METS document representation - the METS objects are represented
+ *  by hand-coded classes rather than JAXB products due to the fact
+ *  that we want wrappers to provide higher-level abstractions than
+ *  the basic METS XML mark-up.
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+
+import java.io.File;
+import java.net.URL;
+
+public class METSDocument
+{
+	// TODO: add use of 'ID'
+	String objectID;
+	String label;
+	String type;
+	// TODO: add use of 'PROFILE';
+
+	METSHeader      header;
+	METSDescriptiveSet descriptiveSet;
+	METSAdmin       admin;
+	METSFileSet     fileSet;
+
+	public METSDocument(String identifier)
+	{
+		this.objectID = identifier; // In Greenstone land, this should be a unique identifier within
+		                            // the current collection
+		this.type = "document"; // Types in METS are just strings - we currently use
+		                        // 'Document' as a standard representation.
+                            // Individual options should be defined in a published
+														// METS Profile - in Greenstone we currently 'cheat'
+														// and bodge the system
+		this.header      = new METSHeader();
+		this.descriptiveSet = new METSDescriptiveSet();
+		this.fileSet     = new METSFileSet();
+	}
+
+	/**
+	 *  Set the document identifier for this document
+	 *
+	 *  @param <code>String</code> the identifier
+	 */
+	public void setDocId(String id)
+	{	this.objectID = id;
+	}
+
+	public String getDocId()
+	{	return this.objectID;
+	}
+
+	public METSHeader getHeader()
+	{	return this.header;
+	}	
+
+	public void setHeader(METSHeader header)
+	{	this.header = header;
+	}
+
+	public void addFile(METSFile file)
+	{	this.fileSet.addFile(file);
+	}
+
+	public void addFile(URL url)
+	{	this.fileSet.addFile(url);
+	}
+
+	public void addFile(METSFile file, String toGroup)
+	{	this.fileSet.addFile(file, toGroup);
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFile.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFile.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFile.java	(revision 14150)
@@ -0,0 +1,352 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import java.net.URL;
+import java.net.URLConnection;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  An individual METS component file for a document.  Many features of METS are not
+ *  currently fully supported/used here, but member variables exist for all of the
+ *  standard members.
+ */
+
+public class METSFile
+{
+    METSFileID    id;
+    METSFilePos   location;
+    String        MIMEType;
+    int           sequenceNo;
+    long          size;
+    METSDate      created;
+    long          checkSum;
+    List          adminRefs;
+    List          describeRefs;
+    METSFileGroup group;
+    
+    /**
+     *  Create a new METSFile object, with the given properties.  The remaining
+     *  fields are filled in with default values.
+     *
+     *  @param <code>String</code> the id of the file
+     *  @param <code>METSFilePos</code> the location of the file
+     *  @param <code>String</code> the MIME type of the file
+     */
+    public METSFile(METSFileID id,  METSFilePos location, String mimeType)
+    {
+	this.id           = id;        // required
+	this.location     = location;  // the location
+	this.MIMEType     = mimeType;  // the MIME type;
+	this.sequenceNo   = -1;       // -1 indicates unset
+	this.size         = -1;        // -1 indicates unset
+	this.checkSum     = -1;        // -1 indicates unset
+	this.created      = null;
+	this.adminRefs    = new ArrayList();
+	this.describeRefs = new ArrayList();
+	this.group        = null;
+    }
+    
+    public boolean equals(URL url)
+    { 
+	return this.location.equals(url);
+    }
+    
+    /**
+     *  Set the parent group
+     *
+     *  @param <code>METSFileGroup</code> the new holding file group
+     */
+    public void setGroup(METSFileGroup group)
+    { 
+	this.group = group;
+    }
+    
+    /**
+     *  Get the parent group of this file
+     *
+     *  @return <code>METSFileGroup</code> the holding file group
+     */
+    public METSFileGroup getGroup()
+    { 
+	return this.group;
+    }
+    
+    /**
+     *  Get the identifier of this file
+     *
+     *  @return <code>METSFileID</code> the file identifier
+     */
+    public METSFileID getID()
+    { 
+	return this.id;
+    }
+    
+    /**
+     *  Set the identifier of this file
+     *
+     *  @param <code>METSFileID</code> the file identifier
+     */
+    public void setID(METSFileID id)
+    { 
+	this.id = id;
+    }
+    
+    /**
+     *  @return <code>String</code> the MIME (content) type of the file
+     */
+    public String getMIMEType()
+    { 
+	return this.MIMEType;
+    }
+    
+    /**
+     *  @return <code>URL</code> the location of the file.  This will often be in
+     *          "file://" form.
+     */
+    public URL getLocation()
+    { 
+	return this.location.getLocation();
+    }
+    
+    /**
+     *  Take a url and make a METSFile object out of it - works out
+     *  details such as the Mime type, identifiers and other necessary
+     *  information to call the constructor for METSFile
+     *
+     *  @param <code>URL</code> the url of the object - this may be a 
+     *                          reference to a file on the local filestore
+     *
+     *  @return <code>METSFile</code> the METS file object for the corresponding
+     *                                document component.
+     */
+    public static METSFile makeMETSFile(URL url, String mimeType)
+    {	
+	METSFile reply = new METSFile(new METSFileID(), new METSFilePos(url), mimeType);
+	return reply;
+    }
+    
+    public static METSFile makeMETSFile(URL url)
+    { 
+	String mimeType;
+	
+	if (url.getProtocol().equals("file")){
+	    // TODO: Work out the MIME type
+	    mimeType = URLConnection.getFileNameMap().getContentTypeFor(url.toString().substring(7));
+	    if (mimeType == null) {
+		mimeType = "text/plain";
+	    }
+	} else { 
+	    // TODO: look up the MIME type through the pertinent connection
+	    mimeType = HTTPTools.getMIMEType(url);
+	}
+	// TODO: make the identifier...
+	
+	return makeMETSFile(url, mimeType);
+    }
+    
+    /**
+     *  Write the METS file in an XML to a text-output sink
+     *
+     *  @param <code>PrintWriter</code> the destination of the output
+     */
+    public void write(PrintWriter writer)
+    { 
+	String tag = XMLTools.getOpenTag("mets", "file");
+	tag = XMLTools.addAttribute(tag, "MIMETYPE", this.MIMEType);
+	tag = XMLTools.addAttribute(tag, "ID", this.id.toString());
+	writer.print("      ");   //indentation
+	writer.println(tag);
+	
+	tag = XMLTools.getOpenTag("mets", "FLocat");
+	tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	
+	tag = XMLTools.makeSingleton(tag);
+	writer.print("         ");  //indentation
+	writer.println(tag);
+	
+	writer.print("      ");   //indentation
+	writer.println(XMLTools.getCloseTag("mets", "file"));
+    }
+    
+    
+    public boolean writeSQL(int groupReference, GS3SQLConnection connection)
+    { 
+	// check if this file is in the table already...
+	GS3SQLAction action;
+	
+	GS3SQLSelect select = new GS3SQLSelect("files");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(groupReference),
+							GS3SQLField.INTEGER_TYPE);
+	GS3SQLWhere  where  = new GS3SQLWhere(whereItem);
+	whereItem = new GS3SQLWhereItem("FileID","=", this.id.toString());
+	where.add(whereItem);
+	select.setWhere(where);
+
+	Statement statement = null;
+	
+	// if not, then make an insert action
+	try {
+	    statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(select.toString());
+	    if (!results.first()){
+		GS3SQLInsert insert = new GS3SQLInsert("files");
+		
+		insert.addValue("FileGroupRef", Integer.toString(groupReference), GS3SQLField.INTEGER_TYPE);
+		insert.addValue("FileID", this.id.toString());
+		
+		action = insert;
+	    }
+	    else {
+		GS3SQLUpdate update = new GS3SQLUpdate("files");
+		
+		update.setWhere(where);
+		action = update;
+	    }
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFile.writeSQL(): "+ex);
+	    return false;
+	}
+	action.addValue("FileLocType", this.location.getType());
+	action.addValue("FileLocation", this.location.getLocation().toString());
+	action.addValue("MIMEType", this.MIMEType);
+	
+	try {
+	    statement.execute(action.toString());
+	    statement.close();
+	    
+	} catch (SQLException e) {
+	    System.err.println("METSFile.writeSQL():"+e);
+	    return false;
+	}
+	return true;
+    }
+    
+    public static METSFile readSQL(GS3SQLConnection connection, ResultSet parentSet)
+    { 
+	try {
+	    String locType = parentSet.getString("FileLocType");
+	    String location = parentSet.getString("FileLocation");
+	    String mimeType = parentSet.getString("MIMEType");
+	    String id = parentSet.getString("FileID");
+	
+	    METSFileID metsID = new METSFileID(id);
+	    METSFilePos metsFilePos = new METSFilePos(locType, location);
+	    
+	    METSFile reply = new METSFile(metsID, metsFilePos, mimeType);
+	    return reply;
+	}
+	catch (SQLException ex){
+	    System.out.println("METSFile.readSQL(): "+ex);
+	}
+	catch (java.net.MalformedURLException urlEx){
+	    System.out.println("METSFile.readSQL(): "+urlEx);
+	}
+	return null;
+    }
+    
+    public static METSFile parse_file(Element element, METSFileGroup group, String parseFilePath)
+    {
+	NodeList children = element.getChildNodes();
+
+	METSFile thisFile = null;
+	
+	for (int c = 0; c < children.getLength(); c++){
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+		continue;
+	    }  
+	    Element child = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    if (childName.equals("mets:file")){
+		METSFile.parse_file(child, group, parseFilePath);
+	    } else if (childName.equals("mets:FLocat")) {
+		String mimeType = element.getAttribute("MIMETYPE");
+		String fileId = element.getAttribute("ID");   
+		
+		METSFilePos thisFilePos = METSFile.parse_flocateXML(child, parseFilePath, fileId);
+	   	thisFile = new METSFile(new METSFileID(fileId), thisFilePos, mimeType);    
+		group.addFile(thisFile);
+	    } else {
+		System.err.println("Warning: unrecognised tag " +childName);
+	    }
+	}
+	return thisFile;
+    }
+    
+    public static METSFilePos parse_flocateXML(Element element, String parseFilePath, String fileId)
+    {
+	METSFilePos thisFilePos = null;
+	
+	// get most of the information from the child FLocat node
+	String locationType = element.getAttribute("LOCTYPE");
+	String href = element.getAttribute("xlink:href");
+	//String fileId = element.getAttribute("ID");
+	try {
+	    thisFilePos = new METSFilePos(locationType, href);
+	    
+	    if (fileId.startsWith("default.")) {
+		//retrieve the string after ..import/
+		int importPosition = href.indexOf("import/")+7;
+		href.substring(importPosition);
+		if (fileId.equals("default.1")) {
+		    href = "file:" + parseFilePath+ "doctxt.xml";
+		} else {
+		    href = "file:" + parseFilePath + href.substring(importPosition);
+		}
+	    }
+            thisFilePos = new METSFilePos(locationType, href);
+	} catch (java.net.MalformedURLException ex) {
+	    // TODO: raise error
+	    System.err.println("METSFile.parse_flocateXML(): "+ex);
+	}
+	return thisFilePos;
+    }
+    
+    
+    /**
+     *  Get the location of this file as a URL
+     */
+    public URL getURL()
+    { 
+	return this.location.getLocation();
+    }
+    
+    /**
+     *  Get modified file date
+     */
+    public long getModifiedDatestamp()
+    { 
+	return this.location.getModifiedDatestamp();
+    }
+    
+    /**
+     *  Overridden <code>toString</code> for convenience - returns the location of the file.
+     *
+     *  @return <code>String</code> the location of the file as a string
+     */
+    public String toString()
+    {
+	return this.location.toString();
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java	(revision 14150)
@@ -0,0 +1,408 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.net.URL;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+import java.io.PrintWriter;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+/**
+ *  Child groups are indicated by colons.  E.g. Chapter1:Section1 would indicate
+ *  the group "Section1" inside the "Chapter1" group...
+ */
+
+public class METSFileGroup
+{ 
+    List   children;
+    List   childGroups;
+    String id;
+    
+    public static final String SECTION_PARENT = "Section";
+    public static final String GROUP_PARENT = "Group";
+    
+    public METSFileGroup(String id)
+    { 
+	this.children = new ArrayList();
+	this.childGroups = new ArrayList();
+	this.id = id;
+    }
+    
+    /**
+     *  Get the modified datestamp that applies for this file group.  An empty group
+     *  will return <code>0</code>.
+     *
+     *  @return <code>long</code> the date modified as milliseconds from the 1st January 1970
+     */
+    public long getModifiedDatestamp()
+    { 
+	long reply = 0; // empty groups reply '0'
+	
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    long fileStamp = file.getModifiedDatestamp();
+	    if (fileStamp > reply) {
+		reply = fileStamp;
+	    }
+	}
+	
+	childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    long fileStamp = fileGroup.getModifiedDatestamp();
+	    if (fileStamp > reply) {
+		reply = fileStamp;
+	    }
+	}
+	
+	return reply;
+    }
+    
+    /**
+     *  Find all the occurrences of a given file, and place them
+     *  in a <code>List</code>.
+     */
+    public void findGroups(URL findFile, List resultList)
+    { 
+	// find in this particular group
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    if (file.equals(findFile))
+		{ resultList.add(this.id);
+		}
+	}
+	
+	// iterate over the child groups
+	childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    
+	    fileGroup.findGroups(findFile, resultList);
+	}
+    }
+    
+    /**
+     *  Get a sub group of this METSFileGroup.
+     *
+     *  @param <code>String</code> the name of the subgroup
+     *  @return <code>METSFileGroup</code> the child group, which will
+     *          be <code>null</code> if the group is not known.
+     */
+    public METSFileGroup getSubgroup(String id)
+    { 
+	String childId;
+	
+	int dotAt = id.indexOf(':');
+	if (dotAt == -1){
+	    childId = null;
+	}
+	else{ 
+	    childId = id.substring(dotAt+1);
+	    id = id.substring(0, dotAt);
+	}
+	
+	// iterate over the child groups
+	Iterator childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup group = (METSFileGroup) childIter.next();
+	    
+	    if (group.id.equals(id)) {
+		if (childId == null) {
+		    return group;
+		}
+		else {
+		    return group.getSubgroup(childId);
+		}
+	    }
+	}
+	return null;
+    }
+    
+    /**
+     *  Add a filegroup to this group of files.
+     *
+     *	@param <code>METSFileGroup</code> the filegroup.
+     */
+    public void addGroup(METSFileGroup group)
+    { 
+	this.childGroups.add(group);
+    }
+    
+    /**
+     *  Add a file to this group of files.
+     *
+     *	@param <code>METSFile</code> the file in a METS File object.
+     */
+    public void addFile(METSFile file)
+    { 
+	this.children.add(file);
+	file.setGroup(this);
+	if (!file.getID().isDefined()){
+	    file.setID(new METSFileID(this.id + "." + Integer.toString(this.children.size())));
+	}
+    }
+    
+    /**
+     *  Get the nth child of the group
+     *
+     *  @param <code>int</code> the index into the group
+     *  @return <code>METSFile</code> the corresponding child.  This may be <code>null</code>
+     *          particularly if the index given falls outside of the valid range of child
+     *          indexes.
+     */
+    public METSFile getFile(int index)
+    { 
+	if (index < 0 || index >= this.children.size()){
+	    return null;
+	}
+	
+	return (METSFile) this.children.get(index);
+    }
+    
+    /**
+     *  Get all the files in the group as a <code>List</code>
+     *
+     *  @return <code>List</code> the file childen of the group.
+     */
+    public List getFiles()
+    { 
+	return this.children;
+    }
+    
+    /**
+     *  Get all the subgroups as a <code>List</code>
+     */
+    public List getSubgroups()
+    { 
+	return this.childGroups;
+    }
+    
+    /**
+     *  Get the number of files in the group
+     *
+     *  @return <code>int</code> the count
+     */
+    public int noOfFiles()
+    { 
+	return this.children.size();
+    }
+    
+    /**
+     *  Get the number of subgroups.
+     */
+    public int noOfSubgroups()
+    { 
+	return this.childGroups.size();
+    }
+    
+    /**
+     *  @return <code>String</code> the name of this group
+     */
+    public String getId()
+    { 
+	return this.id;
+    }
+    
+    public static METSFileGroup parse_groupXML(Element element, METSFileSet set, String parseFilePath, METSFileGroup group)
+    {
+ 	//deal with mets:fileGrp
+ 	NodeList children = element.getChildNodes();
+ 	
+ 	for (int c = 0; c < children.getLength(); c++) {
+ 	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+ 	    Element child = (Element) children.item(c);
+ 	    String childName = child.getNodeName();
+	    
+ 	    if (childName.equals("mets:fileGrp")) {
+ 		String groupId = child.getAttribute("ID");
+ 		METSFileGroup childGroup = new METSFileGroup(groupId);
+		
+ 		if (group != null){
+ 		    group.addGroup(childGroup);
+ 		} else {
+ 		    set.addGroup(childGroup);
+ 		}
+ 		METSFile file = METSFile.parse_file(child, childGroup, parseFilePath);
+ 	    }
+ 	}
+ 	return group;
+    }
+
+    /**
+     *  Write the object in an XML METS format.
+     *
+     *  @param <code>PrintWriter</code> the printwriter being used to write the object
+     */
+    public void write(PrintWriter writer)
+    { 
+	String tag = XMLTools.getOpenTag("mets","fileGrp");
+	tag = XMLTools.addAttribute(tag, "ID", this.id);
+	writer.print("   ");   //indentation
+	writer.println(tag);
+	
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    file.write(writer);
+	}
+	writer.print("   ");   //indentation
+	writer.println(XMLTools.getCloseTag("mets","fileGrp"));
+    }
+    
+    public boolean writeSQL(DocumentInterface document, String parentRef, boolean parentIsSection, 
+			    GS3SQLConnection connection)
+    { 
+	int sqlId = -1;
+	// check if this node is in the database already
+	GS3SQLSelect select = new GS3SQLSelect("filegroups");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileGroupID", "=", this.id);
+	GS3SQLWhereItem whereDoc  = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(whereDoc);
+	select.setWhere(where);
+	
+	ResultSet results = null;
+	Statement statement = null;
+	
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(select.toString());
+	    if (!results.first()) {
+		GS3SQLInsert insert = new GS3SQLInsert("filegroups");
+		
+		insert.addValue("DocID", document.getID().toString());
+		insert.addValue("FileGroupID", this.id);
+		insert.addValue("ParentRef", parentRef, GS3SQLField.INTEGER_TYPE);
+		insert.addValue("ParentType", parentIsSection ? SECTION_PARENT : GROUP_PARENT);
+		
+		statement.execute(insert.toString());
+	    }
+	    else {
+		// TODO: update the data for this file group...
+	    }
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFileGroup.writeSQL(): "+ex);
+	    return false;
+	}
+	
+	// get the filegroup reference now
+	try {
+	    results = statement.executeQuery(select.toString());
+	    if (results.first()) {
+		sqlId = results.getInt("FileGroupRef");
+	    } else {
+		statement.close();
+		return false;
+	    }
+	    statement.close();
+	} catch (SQLException sqlex) {
+	    System.err.println("METSFileGroup.writeSQL(): "+sqlex);
+	    return false;
+	}
+	// iterate over the child groups
+	Iterator childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    
+	    if (!fileGroup.writeSQL(document, Integer.toString(sqlId), false, connection)){
+		System.err.println("METSFileGroup.writeSQL(): Couldn't write FileGroup");
+		return false;
+	    }
+	}
+	
+	// iterate over the child files
+	childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();	    
+	    if (!file.writeSQL(sqlId, connection)){
+		System.err.println("METSFileGroup.writeSQL(): Couldn't write File");
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSFileGroup readSQL(DocumentInterface document, GS3SQLConnection connection,
+					ResultSet resultSet)
+    {
+	try {
+	    String ID         = resultSet.getString("FileGroupID");
+	    String parentType = resultSet.getString("ParentType");
+	    String parentRef  = resultSet.getString("ParentRef");
+	    
+	    // create the metadata block object
+	    METSFileGroup group = new METSFileGroup(ID);
+	    
+	    // get its metadata reference to retrieve divisions
+	    int groupRef = resultSet.getInt("FileGroupRef");
+	    
+	    // query the database for matching groups which are children of this one
+	    GS3SQLSelect select = new GS3SQLSelect("filegroups");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(groupRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSFileGroup.GROUP_PARENT);
+	    where.add(whereItem);
+	    select.setWhere(where);
+	    
+	    Statement statement = connection.createStatement();
+	    
+	    // parse through the child groups
+	    ResultSet childSet = statement.executeQuery(select.toString());
+	    if (childSet.first()) {
+		do {
+		    METSFileGroup fileGroup = METSFileGroup.readSQL(document, connection, childSet);
+		    if (fileGroup != null) {
+			group.addGroup(fileGroup);
+		    }
+		}
+		while (childSet.next());
+	    }
+	    // now scan for file members
+	    select = new GS3SQLSelect("files");
+	    select.addField("*");
+	    whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(groupRef), GS3SQLField.INTEGER_TYPE);
+	    where = new GS3SQLWhere(whereItem);
+	    select.setWhere(where);
+	    ResultSet childFileSet = statement.executeQuery(select.toString());
+	    if (childFileSet.first()) {
+		do {
+		    METSFile file = METSFile.readSQL(connection, childFileSet);
+		    if (file != null) {
+			group.addFile(file);
+		    }
+		}
+		while (childFileSet.next());
+	    }
+	    statement.close();
+	    return group;
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSFileGroup.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return null;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java	(revision 14150)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A type-safe wrapper for METS File identifiers.
+ */
+
+public class METSFileID
+{	
+    String id;
+    
+    public METSFileID()
+    {	
+	this.id = null;
+    }
+    
+    public METSFileID(String id)
+    {	
+	this.id = id;
+    }
+    
+    public String toString()
+    {	
+	return this.id;
+    }
+    
+    public boolean isDefined()
+    {	
+	return this.id != null;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java	(revision 14150)
@@ -0,0 +1,27 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ *  Notes: Greenstone doesn't currently use/support the SEQ or AREA 
+ *  file pointer types hence, one can use only a simple file reference
+ */
+
+public class METSFilePointer
+{
+	public static final String SIMPLE = "";
+	public static final String SEQUENCE = "seq";
+	public static final String AREA = "area";
+
+	String type;
+	String ID;
+	String fileId;
+	List   children;
+
+	public METSFilePointer(String ID, String fileId)
+	{	this.ID = ID;
+		this.fileId = fileId;
+		this.type = METSFilePointer.SIMPLE;
+	}	
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java	(revision 14150)
@@ -0,0 +1,41 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.File;
+import java.net.URL;
+
+public class METSFilePos
+{
+  METSLocation location;
+
+  public METSFilePos(File file) throws java.net.MalformedURLException
+  { this.location = new METSLocation(file);
+  }
+
+  public METSFilePos(URL url)
+  { this.location = new METSLocation(url);
+  }
+
+  public METSFilePos(String locationType, String location) throws java.net.MalformedURLException
+  { this.location = new METSLocation(locationType, location);
+  }
+  
+  public URL getLocation()
+  { return this.location.getLocation();
+  }
+
+  public String getType()
+  { return this.location.getType();
+  }
+
+  public long getModifiedDatestamp()
+  { return this.location.getModifiedDatestamp();
+  }
+
+  public String toString()
+  { return this.location.getLocation().toString();
+  }
+
+  public boolean equals(URL filePos)
+  { return this.location.equals(filePos);
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java	(revision 14150)
@@ -0,0 +1,13 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSFileRef
+{	String label;
+
+	public METSFileRef(String label)
+	{	this.label = label;
+	}
+
+	public String toString()
+	{	return this.label;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java	(revision 14150)
@@ -0,0 +1,344 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A class to hold all the alternative file structures
+ *  for a particular METS Document
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.File;
+import java.io.PrintWriter;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSFileSet
+{ 
+    Map    fileGroups;
+    String reference;
+    FileIdentifierFactory identifierFactory;
+    FileGroupIdentifierFactory groupIdentifierFactory;
+    
+    class FileGroupIdentifierFactory extends AbstractIdentifierFactory
+    {
+	public static final String FILEGROUP_PRELUDE = "FG";
+	
+	public FileGroupIdentifierFactory()
+	{
+	    super("FILEGROUP_PRELUDE");
+	}
+    }
+    
+    public METSFileSet()
+    { 
+	this.fileGroups = new HashMap();
+	this.fileGroups.put("default", new METSFileGroup("default"));
+	this.reference = null;
+	this.identifierFactory = new FileIdentifierFactory();
+	this.groupIdentifierFactory = new FileGroupIdentifierFactory();
+    }
+    
+    public long getModifiedDatestamp()
+    { 
+	long reply = 0;
+	
+	Iterator groupsIter = this.fileGroups.values().iterator();
+	while (groupsIter.hasNext()) {
+		METSFileGroup group = (METSFileGroup) groupsIter.next();
+		long groupStamp = group.getModifiedDatestamp();
+		if (groupStamp > reply){
+		    reply = groupStamp;
+		}
+	    }
+	return reply;
+    }
+    
+    
+    /**
+     *  Get a new FileGroup to lie within this fileset.  The group is not initialised with
+     *  a parent, etc. - all that is done is the allocation of a unique group identifier.
+     *
+     *  @return <code>METSFileGroup</code> the new group.
+     */
+    public METSFileGroup createGroup()
+    { 
+	METSFileGroup group = new METSFileGroup(this.groupIdentifierFactory.getNextIdentifier());
+	return group;
+    }
+    
+    public METSFile createFile(METSFilePos filePos, String type)
+    { 
+	METSFile file = new METSFile(new METSFileID(this.identifierFactory.getNextIdentifier()), filePos, type);
+	return file;
+    }
+    
+    /**
+     *  Get the group that corresponds to the given name...
+     *
+     *  @param <code>String</code> the name of the group.
+     *  @return <code>METSFileGroup</code> the group object - this will be
+     *          <code>null</code> if the group is not found
+     */
+    public METSFileGroup getGroup(String name)
+    { 
+	if (!this.fileGroups.containsKey(name)){
+	    return null;
+	}
+	return (METSFileGroup) this.fileGroups.get(name);
+    }
+    
+    /**
+     *  Get the Nth file from the default group...
+     *
+     *  @param <code>int</code> the index into the default group to use...
+     *  @return <code>METSFile</code> the file.
+     */
+    public METSFile getFile(int index)
+    { 
+	METSFileGroup group = this.getGroup("default");
+
+	if (group == null) {
+	    return null;
+	}
+	return group.getFile(0);
+    }
+    
+    /**
+     *  Add a file to the default file group
+     */
+    public void addFile(METSFile file)
+    { 
+	METSFileGroup defaultGroup = (METSFileGroup) this.fileGroups.get("default");
+	defaultGroup.addFile(file);
+    }
+    
+    /**
+     *  Add a file to the default file group
+     *
+     *  @return <code>METSFile</code> an object that wraps the given url into
+     *          a METS file object
+     */
+    public METSFile addFile(URL url)
+    { 
+	METSFile file = METSFile.makeMETSFile(url);
+	this.addFile(file);
+	return file;
+    }
+    
+    public void addFile(METSFile file, String toGroup)
+    { 
+	METSFileGroup destGroup = (METSFileGroup) this.fileGroups.get(toGroup);
+	
+	if (destGroup == null){
+	    destGroup = new METSFileGroup(toGroup);
+	    this.fileGroups.put(destGroup.getId(), destGroup);
+	}
+	
+	destGroup.addFile(file);
+    }
+    
+    public void addFile(URL url, String toGroup)
+    { 
+	METSFile file = METSFile.makeMETSFile(url);
+	this.addFile(file, toGroup);
+    }
+    
+    /**
+     *  Add a group at the top level of the METS File set.
+     *
+     *  @param <code>METSFileGroup</code> the file group to add.
+     */
+    public void addGroup(METSFileGroup group)
+    { 
+	this.fileGroups.put(group.getId(), group);
+    }
+    
+    /**
+     *  Get all the groups that contain a given URL/file.
+     *
+     *  @param <code>URL</code> the location of the file.
+     *  @return <code>List</code> the list of group references that contain the file.
+     */
+    public List findGroups(URL file)
+    { 
+	List resultList = new ArrayList();
+	
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    group.findGroups(file, resultList);
+	}
+	return resultList;
+    }
+    
+    /*
+     *
+     *
+     */
+    public static METSFileSet parseXML(NodeList fileSecs, METSFileSet fileSet, String parseFilePath) 
+    {
+ 	//METSFileSet set = new METSFileSet();> 
+ 	// this is in effect a group without a sense of 'self'...
+ 	for (int c = 0; c < fileSecs.getLength(); c++) {
+ 	    if (fileSecs.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+	    
+ 	    METSFileGroup groups = METSFileGroup.parse_groupXML((Element)fileSecs.item(c), fileSet, parseFilePath, null);
+ 	}
+ 	return fileSet;
+    }
+    
+
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	writer.println("<mets:fileSec>");
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    group.write(writer);
+	}
+	writer.println("</mets:fileSec>");
+    }
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	// write the set if the reference does not already exist...
+	if (this.reference == null){
+	    // Insert the file section into the database
+	    GS3SQLInsert insert = new GS3SQLInsert("filesection");
+	    insert.addValue("DocID", document.getID().toString());
+	    insert.addValue("FileSecID", "test"); // TODO: remove magic string
+	    
+	    Statement statement = null;
+	    try {
+		statement = connection.createStatement();
+		statement.execute(insert.toString());
+	    } catch (SQLException e) {
+		System.err.println("METSFileSet.writeSQL(): "+e);
+		return false;
+	    }
+	    
+	    // find the file section number
+	    GS3SQLSelect select = new GS3SQLSelect("filesection");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileSecID", "=", "test");
+	    GS3SQLWhereItem whereDoc  = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    where.add(whereDoc);
+	    select.setWhere(where);
+	    
+	    try {
+		ResultSet set = statement.executeQuery(select.toString());
+		if (set.first()) {
+		    int sectionRef = set.getInt("FileSectionRef");
+		    
+		    this.reference = Integer.toString(sectionRef);
+		}
+		statement.close();
+	    }
+	    catch (SQLException ex){
+		System.out.println("METSFileSet.writeSQL(): "+ex);
+		return false;
+	    }
+	}
+	
+	if (this.reference == null) {
+	    return false;
+	}
+	// write out the children
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    if (!group.writeSQL(document, this.reference, true, connection)){
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSFileSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	METSFileSet set = new METSFileSet();    
+	
+	// Get file sections from the filesection table (currently redundant)
+	GS3SQLSelect select = new GS3SQLSelect("filesection");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+
+	Statement statement = null;
+	
+	// Get the identifier for this file set, etc.
+	int fileSetRef = -1;
+	try {
+	    statement = connection.createStatement();
+	    ResultSet sections = statement.executeQuery(select.toString());
+	    if (sections.first()) {
+		fileSetRef = sections.getInt("FileSectionRef");
+		set.reference = Integer.toString(fileSetRef);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	    
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFileSet.readSQL() "+ex);
+	    return null;
+	}
+	// Get child file groups
+	select = new GS3SQLSelect("filegroups");
+	select.addField("*");
+	whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());    
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(fileSetRef), 
+					GS3SQLField.INTEGER_TYPE);
+	where.add(whereItem);
+	whereItem = new GS3SQLWhereItem("ParentType", "=", METSFileGroup.SECTION_PARENT);
+	where.add(whereItem);
+	select.setWhere(where);
+	// start going through the matching file groups
+	try {
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSFileGroup filegroup = METSFileGroup.readSQL(document, connection, resultSet);
+		    if (filegroup != null) {
+			set.addGroup(filegroup);
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.out.println("METSFileSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return set;
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java	(revision 14150)
@@ -0,0 +1,32 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class METSHeader
+{
+    List agents;
+    METSDate createDate;
+    METSDate lastModDate;
+    
+    public METSHeader()
+    {
+	this.agents = new ArrayList();
+    }
+    
+    public METSHeader(METSAgent defaultAgent)
+    {	
+	this();
+	this.agents.add(defaultAgent);
+    }
+    
+    public void setCreateDate(METSDate createDate)
+    {	
+	this.createDate = createDate;
+    }
+    
+    public void setModifiedDate(METSDate modifiedDate)
+    {	
+	this.lastModDate = modifiedDate;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java	(revision 14150)
@@ -0,0 +1,59 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.File;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+public class METSLocation
+{
+  String type;
+  URL    location;
+
+  public METSLocation(String type, String location) throws java.net.MalformedURLException
+  { this.type = type;
+    this.location = new URL(location);
+  }
+
+  public METSLocation(URL url)
+  { this.type = "URL";
+    this.location = url;
+  }
+
+  public METSLocation(File file)
+  { this.type = "URL";
+    this.location = URLTools.getFileURL(file);
+  }
+
+  public boolean equals(URL url)
+  { return this.location.equals(url);
+  }
+
+  public URL getLocation()
+  { return this.location;
+  }
+
+  public long getModifiedDatestamp()
+  { if (this.location != null) {
+      if (this.location.getProtocol().equals("file")) {
+	File file = new File(this.location.getPath());
+	return file.lastModified();
+      }
+      else {
+	try {
+	  URLConnection connection = this.location.openConnection();
+	  return connection.getLastModified();
+	}
+	catch (java.io.IOException io)
+	{ // TODO: report error
+	}
+      }
+    }
+    return System.currentTimeMillis();
+  }
+
+  public String getType()
+  { return this.type;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java	(revision 14150)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSMetaParams
+{
+	
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java	(revision 14150)
@@ -0,0 +1,232 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.Iterator;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Implement a single namespace
+ */
+
+public abstract class METSNamespace
+{
+    protected String       type;        // the type of the metadata
+    protected String       otherType;   // if a non-standard type, then...
+    protected String       name;        // e.g. "DC" for Dublin Core
+    protected METSLocation location;    // the location of the corresponding file
+    protected String       group;       // the group of metadata namespaces to which
+    // this namespace belongs
+    protected String       id;          // the ID for this namespace
+
+    public METSNamespace(String name)
+    { 
+	this.name        = name;
+	this.location    = null;
+	this.group       = null;
+	this.id          = null;
+    }
+    
+    public METSNamespace(String name, METSLocation location)
+    { 
+	this.location    = location;
+	this.name        = name;
+	this.group       = null;
+	this.id          = null;
+    }
+    
+    /**
+     *  Get the name of the namespace...
+     *
+     *  @return <code>String</code> the name of the namespace
+     */
+    public String getName()
+    { 
+	return this.name;
+    }
+    
+    /**
+     *  Get the group of namespaces to which this namespace belongs.
+     *
+     *  @return <code>String</code> the name of the namespace group
+     */
+    public String getGroup()
+    {
+	return this.group;
+    }
+    
+    /**
+     *  Set the group of namespaces to which this namespace belongs.
+     *
+     *  @param <code>String</code> the name of the namespace group
+     */
+    public void setGroup(String group)
+    { 
+	this.group = group;
+    }
+    
+    /**
+     *  Get the ID for this namespace.
+     *
+     *  @return <code>String</code> the id of the namespace
+     */
+    public String getID()
+    {
+	return this.id;
+    }
+    
+    /**
+     *  Set the id of this namespace.
+     *
+     *  @param <code>String</code> the name of the namespace group
+     */
+    public void setID(String id)
+    {
+	this.id = id;
+    }
+
+    /**
+     *  <p>Indicate whether this namespace is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+    
+    public abstract boolean addMetadata(String label, String value);
+    public abstract boolean setMetadata(String label, String value);
+    public abstract boolean removeMetadata(String label);
+    public abstract boolean removeMetadata(String label, String value);
+    public abstract List    getMetadata(String label);
+    public abstract Iterator getMetadataNames();
+    public abstract boolean write(PrintWriter writer);
+    //  public abstract boolean writeSQL(GS3SQLConnection connection);
+
+    /**
+     *  Write out the metadata to an SQL database through a <code>GS3SQLConnection</code>.
+     *
+     *  @param <code>int</code> the sql identifier of the parent object (in "metadata") table.
+     *  @param <code>GS3SQLConnection</code> the SQL database to use.
+     */
+    public boolean writeSQL(int parentId, GS3SQLConnection connection)
+    { 
+	GS3SQLAction action;
+
+	// If the namespace is not null, then set it...
+	if (this.id != null) {
+	    // use an update action in this case...
+	    GS3SQLUpdate update = new GS3SQLUpdate("namespaces");
+	    update.addValue("NamespaceID", this.id);
+
+	    // set up the where clause
+	    GS3SQLWhere where = 
+		new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", this.id, 
+						    GS3SQLField.INTEGER_TYPE));
+	    update.setWhere(where);
+	    action = update;
+	}
+	else {
+	    GS3SQLInsert insert = new GS3SQLInsert("namespaces");
+	    action = insert;
+	}
+
+	if (this.location != null) {
+	    action.addValue("FileLoc", this.location.getLocation().toString());
+	    action.addValue("FileType", "URL");
+	}
+	else {
+	    // no location stuff
+	}
+	action.addValue("MetadataRef", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE);
+	action.addValue("NamespaceType", this.name);
+
+	// Execute the action
+	Statement statement = null;
+	try {
+	    statement = connection.createStatement();
+	    statement.execute(action.toString());
+	} catch (SQLException e) {
+	    System.err.println("METSNamespace.writeSQL(): "+e);
+	    return false;
+	}
+	// then get the namespace reference number if needsbe...
+	if (this.id == null) {
+	    GS3SQLSelect select = new GS3SQLSelect("namespaces");
+	    select.addField("NamespaceRef");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("NamespaceType", "=", this.name);
+	    where.add(where);
+	    try {
+		ResultSet result  = statement.executeQuery(select.toString());
+      		if (result.last()) {
+		    this.id = Integer.toString(result.getInt("NamespaceRef"));
+		}
+		statement.close();
+	    }
+	    catch (SQLException sqlex){
+		this.id = null;
+		System.err.println("METSNamespace.writeSQL(): "+sqlex);
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSNamespace readSQL(GS3SQLConnection connection, ResultSet resultSet)
+    {
+	METSLocation metsLocation = null;
+	
+	try {
+	    String name = resultSet.getString("NamespaceType");
+	    String id = resultSet.getString("NamespaceRef");
+	    String location = resultSet.getString("FileLoc");
+	    String type = resultSet.getString("FileType");
+	    if (location != null && type != null) {
+		metsLocation = new METSLocation(type, location);
+	    }
+	    
+	    METSNamespace namespace = NamespaceFactory.initNamespace(name, metsLocation);
+	    namespace.id = id;
+
+	    int namespaceRef = resultSet.getInt("NamespaceRef");
+      
+	    GS3SQLSelect select = new GS3SQLSelect("mdvalues");
+	    select.addField("*");
+	    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", Integer.toString(namespaceRef),
+								    GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet valuesSet = statement.executeQuery(select.toString());
+	    if (valuesSet.first()) {
+		do {
+		    String label = valuesSet.getString("Label");
+		    String value = valuesSet.getString("Value");
+
+		    namespace.addMetadata(label, value);
+		}
+		while (valuesSet.next());
+	    }
+	    statement.close();
+	    return namespace;
+	}
+	catch (java.net.MalformedURLException urlEx){
+	    System.err.println("METSNamespace.readSQL(): "+urlEx);
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSNamespace.readSQL(): "+sqlEx);
+	}    
+	return null;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java	(revision 14150)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ *  Notes: Greenstone doesn't currently use/support the SEQ file pointer type;
+ *         hence, one can use only 'area' or 
+ */
+
+public class METSPointer
+{
+	String ID;
+	METSLocation location; // Or something similar...?
+
+	public METSPointer ()
+	{
+	}	
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java	(revision 14150)
@@ -0,0 +1,323 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSStructure extends AbstractStructure
+{
+    String label;
+    String type;
+    METSDivision divGroup;
+
+    public static final String STRUCTURE_TYPE = "Structure";
+  
+    public METSStructure(String id, String label, String type)
+    { 
+	super(id);
+	
+	this.label    = label;
+	this.type     = type;
+	this.divGroup = null;
+    }
+
+    /**
+     *  Show what sort of <code>AbstractStructure</code> this structure is...
+     *
+     *  @return <code>String The identifying string for a <code>METSStructure</code> type.
+     */
+    public String getStructureType()
+    { return STRUCTURE_TYPE;
+    }
+  
+    /**
+     *  Find all the divisions that match a given list of file group
+     *  identifiers...
+     *
+     *  @param <code>List</code> the list of identifiers to find
+     *  @param <code>List</code> the list into which to place any
+     *         matching identifiers.
+     */
+    public void findDivisionsForFiles(List listOfFileIdentifiers, List resultList)
+    { Iterator groups = this.children.values().iterator();
+    while (groups.hasNext())
+	{ METSDivision group = (METSDivision) groups.next();
+
+	group.findDivisionsForFiles(listOfFileIdentifiers, resultList);
+	}    
+    }
+
+    /**
+     *  Add a division further into this structure.
+     *
+     *  @param <code>String</code> the identifier label for the division
+     *         within which the division is to be added.
+     *  @param <code>METSDivision</code> the division to add.   
+     */
+    public void addSubDivision(String divisionLabel, METSDivision division)
+    { String topDivision = METSDivision.getTopDivisionName(divisionLabel);
+
+    METSDivision child = (METSDivision) this.children.get(topDivision);
+    if (child == null)
+	{ return;
+	}
+
+    child = child.getDivision(divisionLabel);
+
+    if (child != null) {
+	child.addDivision(division);
+    }
+    }
+
+    /**
+     *  Get a subdivision of a given identifier...
+     * 
+     *  @param <code>String</code> the identifier of the division
+     *  @return <code>METSDivision</code> the division, which will be
+     *          <code>null</code> if it is not found.
+     */
+    public METSDivision getDivision(String divisionLabel)
+    { String topDivision = METSDivision.getTopDivisionName(divisionLabel);
+
+    if (!topDivision.equals("All"))
+	System.out.println("Top division is " + topDivision);
+    METSDivision child = (METSDivision) this.children.get(topDivision);
+    if (child == null)
+	{ System.out.println("No child found");
+	Iterator groups = this.children.keySet().iterator();
+	while (groups.hasNext()) {
+	    System.out.println(groups.next().toString());
+	}
+	return null;
+	}
+
+
+    child = child.getDivision(divisionLabel);
+
+    return child;
+    }
+
+
+    /*
+     *
+     * parse an XML Element as a METSStructure Section
+     *
+     * @param <code>Element</code> the XML element which represents
+     *  the mets:structMap itself 
+     */
+    
+    public static METSStructure parseXML(Element element) 
+    {
+ 	String structId = element.getAttribute("ID");
+ 	String structLabel = element.getAttribute("LABEL");
+	String structType = element.getAttribute("TYPE");
+ 
+ 	METSStructure thisStruct = new METSStructure (structId, structLabel, structType);
+ 	
+ 	NodeList divSecs = element.getChildNodes();
+ 	if (divSecs != null) {
+ 	    for (int c = 0; c < divSecs.getLength(); c++) {
+ 		if (divSecs.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		    continue;
+ 		}
+ 		Element divElement = (Element) divSecs.item(c);
+ 		String divName = divElement.getNodeName();
+ 		if (divName.equals("mets:div")) {
+ 		    METSDivision division = METSDivision.parseXML(divElement);
+ 		    thisStruct.addDivision(division);
+ 		} else {
+ 		    System.err.println("Error: mets:structMap was expecting mets:div but got "+divName);
+ 		}
+ 	    }
+ 	}
+ 	return thisStruct;
+    }
+ 
+    /**
+     *  Write this structure in XML(METS) format to a <code>PrintWriter</code>
+     */
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	String tag = XMLTools.getOpenTag("mets", "structMap");
+	tag = XMLTools.addAttribute(tag, "ID", this.ID.toString());
+	tag = XMLTools.addAttribute(tag, "TYPE", this.type);
+	tag = XMLTools.addAttribute(tag, "LABEL", this.label);
+	
+	writer.println(tag);
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    group.write(writer);
+	}
+	writer.println("</mets:structMap>");
+    }
+    
+    
+    /**
+     *  Write this METSStructure to an SQL database....
+     *
+     *  @param <code>DocumentInterface</code> the enclosing document.
+     *  @param <code>GS3SQLConnection</code> the SQL database connection.
+     */
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	int sqlRef = -1;
+	ResultSet results = null;
+	Statement statement = null;
+	
+	// Prepare query to see if this structure has already been written
+	GS3SQLSelect select = new GS3SQLSelect("structure");
+	select.addField("StructureRef");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhereItem item = new GS3SQLWhereItem("StructureID", "=", this.ID.toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	// attempt to execute the query & get the current structure's reference
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(select.toString());
+	    if (results.first()){
+		sqlRef = results.getInt("structureRef");
+	    }
+	}
+	catch (SQLException sqlEx) {
+	    System.err.print("METSStructure.writeSQL(): "+sqlEx);
+	    return false;
+	}
+	
+	// insert a new structure if it wasn't there previously
+	if (sqlRef == -1) {
+	    GS3SQLInsert insert = new GS3SQLInsert("structure");
+	    insert.addValue("DocID", document.getID().toString());
+	    insert.addValue("StructureID", this.ID.toString());
+	    insert.addValue("StructureType", this.type);
+	    insert.addValue("Label", this.label);
+	    
+	    try {
+		statement.execute(insert.toString());
+		
+		// get the new structure reference by re-running the original select object
+		results = statement.executeQuery(select.toString());
+		if (results.first()) {
+		    sqlRef = results.getInt("StructureRef");
+		} 
+	    } catch (SQLException sqlEx) {
+		System.err.print("METSStructure.writeSQL(): "+sqlEx);
+		return false;
+	    }
+	    
+	}
+	else {
+	    GS3SQLUpdate update = new GS3SQLUpdate("structure");
+	    update.setWhere(where);
+	    
+	    update.addValue("StructureType", this.type);
+	    update.addValue("Label", this.label);
+	    
+	    try {
+		statement.execute(update.toString());
+	    } catch (SQLException sqlEx) {
+		System.err.print("METSStructure.writeSQL(): "+sqlEx);
+		return false;
+	    }
+	}
+	
+	// close the statement
+	try {
+	    statement.close();
+	} catch (SQLException e) {
+	    System.err.println("METSStructure.writeSQL(): "+e);
+	}
+	// write out the child groups (Divisions) now...    
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.writeSQL(document.getID(), sqlRef, true, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+    
+    /**
+     *  Read a METSStructure from a database - on entry, the current item in the
+     *  <code>ResultSet</code> holds the row in the database with the structure's
+     *  data.
+     *
+     *  @param <code>DocumentInterface</code> the document which owns the structure
+     *  @param <code>GS3SQLConnection</code> the database connection itself, to load
+     *         child objects through, etc.
+     *  @param <code>ResultSet</code> pointing to the current row in the database.
+     */
+    public static METSStructure readSQL(DocumentInterface document, GS3SQLConnection connection,
+					ResultSet resultSet)
+    {
+	GS3SQLSelect select = null;
+	
+	try {
+	    String ID    = resultSet.getString("StructureID");
+	    String type  = resultSet.getString("StructureType");
+	    String label = resultSet.getString("Label");
+
+	    // create the metadata block object
+	    METSStructure structure = new METSStructure(ID, label, type);
+      
+	    // get its metadata reference to retrieve divisions
+	    int structureRef = resultSet.getInt("StructureRef");
+
+	    // query the database for matching division for this structure block
+	    select = new GS3SQLSelect("divisions");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(structureRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSStructure.STRUCTURE_TYPE);
+	    where.add(whereItem);
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet divisionSet = statement.executeQuery(select.toString());
+
+	    // parse through the divisions
+	    if (divisionSet.first()) {
+		do {
+		    METSDivision division = METSDivision.readSQL(connection, divisionSet);
+		    if (division != null) {
+			structure.addDivision(division);
+		    }
+		}
+		while (divisionSet.next());
+	    }
+	    statement.close();
+	    return structure;
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSStructure.readSQL(): "+ sqlEx + " " + select.toString());
+	    System.exit(1);
+	}
+	return null;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java	(revision 14150)
@@ -0,0 +1,181 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSStructureSet
+{ 
+    Map children;
+
+    public static final String GSDL3_SECTION_STRUCTURE = "Section";
+
+    public METSStructureSet()
+    { 
+	this.children = new HashMap();
+    }
+
+    public void addStructure(METSStructure structure)
+    { 
+	this.children.put(structure.getID(), structure);
+    }
+
+    public METSStructure getStructure(String name)
+    { 
+	return (METSStructure) this.children.get(name);
+    }
+
+    /**
+     *  Find the divisions that contain a list of given files
+     *
+     *  @param <code>List</code> the list of files to find...
+     *  @return <code>List</code> the matching divisions.
+     */
+    public List findDivisionsForFiles(List listOfFileIdentifiers)
+    { 
+	List reply = new ArrayList();
+
+	// iterate across all the children...
+	Iterator structures = this.children.values().iterator();
+	
+	while (structures.hasNext()){
+	    METSStructure structure = (METSStructure) structures.next();
+	    
+	    structure.findDivisionsForFiles(listOfFileIdentifiers, reply);      
+	}    
+	return reply;
+    }
+    
+    /**
+     *  Add a division to an indicated structure.
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>METSDivision</code> the division to be added.
+     */
+    public void addDivision(String structureId, METSDivision division)
+    {
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+	
+	if (structure != null){
+	    structure.addDivision(division);
+	}
+    }
+
+    /**
+     *  Add a division to an indicated division within structure.
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>String</code> the division identifer, within which the division
+     *         is to be placed.
+     *  @param <code>METSDivision</code> the division to be added.
+     */
+    public void addSubDivision(String structureId, String divisionID, METSDivision division)
+    { 
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+
+	if (structure != null){
+	    structure.addSubDivision(divisionID, division);
+	}
+    }
+
+    /**
+     *  Get a division from within a given structure
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>String</code> the division identifer
+     *
+     *  @return <code>METSDivision</code> the division
+     */
+    public METSDivision getDivision(String structureId, String divisionId)
+    { 
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+
+	if (structure != null){
+	    return structure.getDivision(divisionId);
+	}
+	return null;
+    }
+    public static METSStructureSet parseXML(NodeList structMapSecs)
+    {
+ 	METSStructureSet set = new METSStructureSet();
+ 
+ 	for (int c = 0; c < structMapSecs.getLength(); c++) {
+ 	    METSStructure structure = METSStructure.parseXML((Element) structMapSecs.item(c));
+ 	    set.addStructure(structure);
+ 	}
+ 	return set;
+    }
+        
+
+    public void write(PrintWriter writer)
+    { 
+	Iterator structures = this.children.values().iterator();
+
+	while (structures.hasNext()){
+	    METSStructure group = (METSStructure) structures.next();
+	    
+	    group.write(writer);
+	}
+    }
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator structures = this.children.values().iterator();
+    
+	while (structures.hasNext()){
+	    METSStructure group = (METSStructure) structures.next();
+
+	    if (!group.writeSQL(document, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    public static METSStructureSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	METSStructureSet set = new METSStructureSet();
+
+	GS3SQLSelect select = new GS3SQLSelect("structure");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+	
+	// start going through the matching metadata blocks
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSStructure structure = METSStructure.readSQL(document, connection, resultSet);
+		    if (structure != null) {
+			set.addStructure(structure);
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSStructureSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return set;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java	(revision 14150)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.greenstone.gsdl3.gs3build.schema.Schema;
+import org.greenstone.gsdl3.gs3build.schema.SchemaNode;
+
+public class MetaElement
+{
+	Map    attributes;
+	List   children;
+	String name;
+
+	public MetaElement(Element element, Schema schema)
+	{	
+		// basic setup
+		this.attributes = new HashMap();
+		this.children   = new ArrayList();
+		this.name = element.getNodeName();
+		StringBuffer contentBuffer = new StringBuffer();
+		
+		// split the name into namespace/workspace
+		String workSpace = SchemaNode.getNamespace(name);
+		name = SchemaNode.getElementName(name);
+
+		// get attributes
+		NamedNodeMap attributes = element.getAttributes();
+		for (int a = 0; a < attributes.getLength(); a ++)
+		{	Node attribute = attributes.item(a);
+
+			this.attributes.put(attribute.getNodeName(), attribute.getNodeValue());
+		}
+
+		// get children
+		NodeList children = element.getChildNodes();
+		for (int c = 0; c < children.getLength(); c++)
+		{	Node child = children.item(c);
+
+			if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE)
+			{	// make a child
+				if (contentBuffer.length() > 0) {
+					this.children.add(contentBuffer.toString());
+					contentBuffer.setLength(0);
+				}
+				Object childElement = new MetaElement((Element) child, schema);
+				this.children.add(childElement);
+			}
+			else
+			{	// append to content if not an element - this may be wasteful & type of
+				// element should be checked
+ 				contentBuffer.append(child.getNodeValue());
+			}
+		}
+		if (contentBuffer.length() > 0) {
+			this.children.add(contentBuffer.toString());
+		}
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java	(revision 14150)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+
+/**
+ *  A class which provides metadata-centric tools for handling XML objects
+ */
+public class MetaTools
+{
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java	(revision 14150)
@@ -0,0 +1,33 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class MetadataLabel
+{
+  String namespace;
+  String label;
+
+  public MetadataLabel(String label)
+  { int colonAt = label.indexOf(":");
+    String namespace;
+		
+    if (colonAt > 0) {
+      this.namespace = label.substring(0, colonAt);
+      this.label     = label.substring(colonAt+1);
+    }
+    else {
+      this.namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
+      this.label = label;
+    }
+  }
+
+  public String getNamespace()
+  { return this.namespace;
+  }
+
+  public String getLabel()
+  { return this.label;
+  }
+
+  public static String getDefaultNamespace()
+  { return GSDL3Namespace.GSDL3_NAMESPACE_ID;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java	(revision 14150)
@@ -0,0 +1,71 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+public class NamespaceFactory
+{
+    public static METSNamespace initNamespace()
+    {	
+	return new GSDL3Namespace();
+    }
+    
+    public static METSNamespace initNamespace(String name)
+    {	
+	if (name.equals(GSDL3Namespace.GSDL3_NAMESPACE_ID)) {
+	    return new GSDL3Namespace();
+	}
+	return new SimpleNamespace(name);
+    }
+    
+    public static METSNamespace initNamespace(String name, METSLocation location)
+    {	
+	if (name.equals(GSDL3Namespace.GSDL3_NAMESPACE_ID)){
+	    return new GSDL3Namespace(location);
+	}
+	return new SimpleNamespace(name, location);
+    }
+    
+    /**
+     *  TODO: parse the document in based upon its namespace
+     */
+    public static METSNamespace initNamespace(String name, Element element)
+    {	
+	//return null;
+	return new SimpleNamespace(name);
+    }
+    
+    public static METSNamespace parseXML(Element element)
+    {
+	METSNamespace namespace = null;
+	
+	//String mdType = element.getAttribute("MDType");
+	//String mdType =element.getAttribute("MDTYPE");
+
+	String othermdType = element.getAttribute("OTHERMDTYPE");
+	String mimeType = element.getAttribute("mimeType");
+	
+	if (element.getNodeName().equals("mets:mdWrap")) {
+	    namespace = initNamespace(othermdType, element);
+	}
+	else if (element.getNodeName().equals("mets:mdRef")) {
+	    String mdType = element.getAttribute("MDType");
+	    // just a file wrapper on the metadata...
+	    String location = element.getAttribute("xlink:href");
+	    String locationType = element.getAttribute("LOCTYPE");
+	    
+	    try {
+		METSLocation metsLocation = new METSLocation(locationType, location);
+		namespace = initNamespace(mdType, metsLocation);
+	    }
+	    catch (java.net.MalformedURLException ex) {
+		// TODO: Error, should never happen in the case of Greenstone-exported XML
+	    }
+	}
+	return namespace;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java	(revision 14150)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A simple metadata item with a label and one value to fit...
+ */
+
+public class NamespaceItem
+{
+	private String label;
+	private String value;
+
+	public NamespaceItem(String label, String value)
+	{	this.label = label;
+		this.value = value;
+	}
+
+	public String getLabel()
+	{	return this.label;
+	}
+
+	public String getValue()
+	{	return this.value;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java	(revision 14150)
@@ -0,0 +1,259 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+/**
+ *  Implement a generic, ordered namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class OrderedNamespace extends METSNamespace
+{
+    List metadataList;
+    
+    public OrderedNamespace(String name)
+    { 
+	super(name);
+	this.metadataList = new ArrayList();
+    }
+    
+    public OrderedNamespace(String name, METSLocation location)
+    { 
+	super(name, location);
+	this.metadataList = new ArrayList();
+    }
+    
+    public boolean validate(String field, String value)
+    { 
+	return true;
+    }
+    
+    /**
+     *  Add a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean addMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataList.add(new NamespaceItem(label, value));
+	return true;
+    }
+    
+    /**
+     *  Assign a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.  Any existing metadata for that field will
+     *  be destroyed if the new value validates.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean setMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)){
+	    return false;
+	}
+	
+	// TODO:		this.metadataMap.setOnly(label, value);
+	int itemNo = this.findItem(label);
+	if (itemNo >= 0){
+	    this.metadataList.set(itemNo, new NamespaceItem(label, value));
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     *  Remove all metadata values for a given field name
+     *
+     *  @param <code>String</code> the field to delete
+     *
+     *  @return <code>boolean</code> whether the field was actually deleted; 
+     *          will return <code>true</code> if the field was already empty.
+     */
+    public boolean removeMetadata(String label)
+    { 
+	int itemNo;
+	
+	itemNo = this.findItem(label);
+	while (itemNo >= 0){
+	    this.metadataList.remove(itemNo);
+	    itemNo = this.findItem(label);
+	}
+	return true;
+    }
+    
+    /**
+     *  Remove a particular incidence of a given metadata field for a document.
+     *  If an exact match for the given value is not found, nothing changes.
+     *  N.B. if a value occurs twice, only the first incidence of it will be
+     *  deleted from the list.
+     *
+     *  @param <code>String</code> the field to have the value removed
+     *  @param <code>String</code> the value to be removed from a given field
+     *
+     *  @return <code>boolean</code> <code>true</code> if an actual metadata text
+     *          is matched against the given value and is thus deleted.
+     */
+    public boolean removeMetadata(String label, String value)
+    { 
+	int itemNo;
+	
+	itemNo = this.findItem(label);
+	if (itemNo >= 0) {
+	    this.metadataList.remove(itemNo);
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     *  Get the metadata items for a particular label
+     *
+     *  @param <code>String</code> the label to fetch values for - must be devoid
+     *                             of namespace prologue (i.e. "title" rather than
+     *                             e.g. "dc:title").
+     *  
+     *  @return <code>List</code> the list of corresponding values.  May be
+     *          <code>null</code> if no values are found for the metadata
+     */
+    public List getMetadata(String label)
+    { 
+	List resultList = new ArrayList();
+	
+	for (int i = 0; i < this.metadataList.size(); i ++) {
+	    NamespaceItem item = (NamespaceItem) this.metadataList.get(i);
+	    
+	    if (item.getLabel().equals(label)){
+		resultList.add(item.getValue());
+	    }
+	}
+	if (resultList.size() > 0){
+	    return resultList;
+	}
+	return null;
+    }
+    
+    private int findItem(String label)
+    { 
+	for (int i = 0; i < this.metadataList.size(); i ++) {
+	    if (((NamespaceItem) this.metadataList.get(i)).getLabel().equals(label)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    private int findItem(String label, String value)
+    { 
+	for (int i = 0; i < this.metadataList.size(); i ++){
+	    NamespaceItem item = (NamespaceItem) this.metadataList.get(i);
+	    if (item.getLabel().equals(label) && item.getValue().equals(value)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    public Iterator getMetadataNames() 
+    {
+ 	return this.metadataList.iterator();
+    }
+    
+    
+    /**
+     *  Write out the metadata to an XML file through a <code>PrintWriter</code>.
+     *
+     *  @param <code>PrintWriter</code> the writer to use.
+     */
+    public boolean write(PrintWriter writer)
+    { 
+	// if this is a non-file block of metadata, write it out in long hand
+	if (this.location == null){
+	    String tag = XMLTools.getOpenTag("mets", "mdWrap");
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+	    tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+	    if (this.id != null) {
+		this.id = "gsdl"+this.id;
+		tag = XMLTools.addAttribute(tag, "ID", this.id);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println(tag);
+	    
+	    Iterator items = this.metadataList.iterator();
+	    
+	    while (items.hasNext()){
+		NamespaceItem item = (NamespaceItem) items.next();
+		this.writeItem(writer, item);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println("</mets:mdWrap>");
+	}
+	// otherwise, drop the metadata out in a simplified file-reference
+	// form only
+	else
+	    { 
+		String tag = XMLTools.getOpenTag("mets", "mdRef");
+		tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+		tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+		tag = XMLTools.addAttribute(tag, "MDTYPE", this.name);
+		//tag = XMLTools.addAttribute (tag, "MDTYPE", "OTHER");
+		//tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name); 
+		if (this.id != null) {
+		    tag = XMLTools.addAttribute(tag, "ID", this.id);
+		}
+		tag = XMLTools.makeSingleton(tag);
+	    }
+	return true;
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, NamespaceItem item)
+    { 
+	// Do some default sillinesses
+	//writer.write("         "); 
+	writer.write(XMLTools.getOpenTag(this.name, item.getLabel()));
+	
+	writer.write(item.getValue());
+	
+	writer.write(XMLTools.getCloseTag(this.name, item.getLabel()));
+	return true;
+    }
+    
+    /**
+     *  <p>Indicate whether this metadata is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java	(revision 14150)
@@ -0,0 +1,347 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Implement a simple, unordered, namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class SimpleNamespace extends METSNamespace
+{
+    MultiMap metadataMap;
+    
+    public SimpleNamespace(String name)
+    { 
+	super(name);
+	this.metadataMap = new MultiMap();
+    }
+    
+    public SimpleNamespace(String name, METSLocation location)
+    { 
+	super(name, location);
+	this.metadataMap = new MultiMap();
+    }
+    
+    public SimpleNamespace(String name, Element mdWrapTag)
+    { 
+	super(name);
+	this.metadataMap = new MultiMap();
+	
+	NodeList childNodes = mdWrapTag.getChildNodes();
+	for (int c = 0; c < childNodes.getLength(); c ++) {
+	    // a metadata node
+	    if (childNodes.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+		// get the name of the metadata from the node
+		String metadataLabel = childNodes.item(c).getNodeName();
+		
+		// skip blank labels
+		// TODO: raise an error: metadata item without metadata label set
+		if (metadataLabel == null || metadataLabel.length() == 0) {
+		    continue;
+		}
+		
+		// build the value
+		String metadataValue = "";
+		
+		Element childElement = (Element) childNodes.item(c);
+		for (int i = 0; i < childElement.getChildNodes().getLength(); i ++) {
+		    metadataValue = metadataValue + childElement.getChildNodes().item(i).toString();
+		}
+		
+		// trim any leading namespace identifiers in "namespace:label" metadata
+		if (metadataLabel.startsWith(this.name + ":")) {
+		    metadataLabel = metadataLabel.substring(this.name.length() + 1);
+		}
+		
+		this.addMetadata(metadataLabel, metadataValue);
+	    }
+	}
+    }
+    
+    public boolean validate(String field, String value)
+    { 
+	return true;
+    }
+    
+    /**
+     *  Add a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean addMetadata(String label, String value)
+    {
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataMap.put(label, value);
+	return true;
+    }
+    
+    /**
+     *  Assign a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.  Any existing metadata for that field will
+     *  be destroyed if the new value validates.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean setMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataMap.setOnly(label, value);
+	return true;
+    }
+    
+    /**
+     *  Remove all metadata values for a given field name
+     *
+     *  @param <code>String</code> the field to delete
+     *
+     *  @return <code>boolean</code> whether the field was actually deleted; 
+     *          will return <code>true</code> if the field was already empty.
+     */
+    public boolean removeMetadata(String label)
+    { 
+	this.metadataMap.remove(label);
+	return true;
+    }
+    
+    /**
+     *  Remove a particular incidence of a given metadata field for a document.
+     *  If an exact match for the given value is not found, nothing changes.
+     *  N.B. if a value occurs twice, only the first incidence of it will be
+     *  deleted from the list.
+     *
+     *  @param <code>String</code> the field to have the value removed
+     *  @param <code>String</code> the value to be removed from a given field
+     *
+     *  @return <code>boolean</code> <code>true</code> if an actual metadata text
+     *          is matched against the given value and is thus deleted.
+     */
+    public boolean removeMetadata(String label, String value)
+    { 
+	return this.metadataMap.remove(label, value);
+    }
+    
+    /**
+     *  Get the metadata items for a particular label
+     *
+     *  @param <code>String</code> the label to fetch values for
+     *  
+     *  @return <code>List</code> the list of corresponding values.  May be
+     *          <code>null</code>
+     */
+    public List getMetadata(String label)
+    { 
+	return this.metadataMap.getAll(label);
+    }
+    
+    public Iterator getMetadataNames() 
+    {
+ 	return this.metadataMap.keySet().iterator();
+    }
+    
+    /**
+     *  Write out the metadata to an XML file through a <code>PrintWriter</code>.
+     *
+     *  @param <code>PrintWriter</code> the writer to use.
+     */
+    public boolean write(PrintWriter writer)
+    { 
+	// if this is a non-file block of metadata, write it out in long hand
+	if (this.location == null){
+	    String tag = XMLTools.getOpenTag("mets", "mdWrap");
+	    
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+	    tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+	    if (this.id != null) {
+		this.id = "gsdl"+this.id;
+		tag = XMLTools.addAttribute(tag, "ID", this.id);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println(tag);
+	    
+	    tag = XMLTools.getOpenTag("mets", "xmlData");
+	    writer.print("      ");  //indentation 
+	    writer.println(tag);
+	    
+	    Iterator keys = this.metadataMap.keySet().iterator();
+	    while (keys.hasNext()){
+		String thisKey = keys.next().toString();
+		writer.print("         "); //indentation
+		this.writeItem(writer, thisKey);
+	    }
+	    
+	    writer.print("      ");  //indentation 
+	    writer.println(XMLTools.getCloseTag("mets", "xmlData"));
+	    
+	    writer.print("   ");  //indentation
+	    writer.println(XMLTools.getCloseTag("mets", "mdWrap"));
+	}
+	// otherwise, drop the metadata out in a simplified file-reference
+	// form only
+	else
+	    { 
+		String tag = XMLTools.getOpenTag("mets","mdRef");
+		tag = XMLTools.addAttribute(tag, "LOCTYPE", "URL");
+		tag = XMLTools.addAttribute(tag, "xlink:href", location.getLocation().toString());
+		//tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+		//tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+		tag = XMLTools.addAttribute(tag, "MDTYPE", this.name);	
+		if (this.id != null) {
+		    tag = XMLTools.addAttribute(tag, "ID", this.id);
+		}
+		writer.println(tag);
+		writer.print("   ");  //indentation	
+		writer.println("</mets:mdRef>");
+	    }
+	return true;
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, String label)
+    { 
+	if (this.location == null) {
+	    // just place the metadata in a simple wrapper
+	    Iterator values = this.metadataMap.getAll(label).iterator();
+	    
+	    while (values.hasNext()) {
+		String value = values.next().toString();
+		String metaTagName = "Metadata name=" +'"'+ label +'"';
+		//writer.write(XMLTools.getOpenTag(this.name, label));
+		writer.write(XMLTools.getOpenTag(this.name, metaTagName));
+		writer.write(value);
+		
+		//writer.println(XMLTools.getCloseTag(this.name, label));
+		writer.println(XMLTools.getCloseTag(this.name, "Metadata"));
+	    }
+	}
+	else {
+	    String tag = XMLTools.getOpenTag("mets", "mdRef");
+	    tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	    tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", this.getName());
+	    tag = XMLTools.makeSingleton(tag);
+	    writer.print("      "); //indentation 
+	    writer.println(tag);
+	}
+	return true;
+    }
+    
+    /**
+     *  Write out the metadata to an SQL database through a <code>GS3SQLConnection</code>.
+     *
+     *  @param <code>GS3SQLConnection</code> the SQL database to use.
+     */
+    public boolean writeSQL(int parentId, GS3SQLConnection connection)
+    { 
+	// write the general stuff
+	String sqlId;
+	
+	if (!super.writeSQL(parentId, connection)) {
+	    return false;
+	}
+	
+	Statement statement = null;
+	ResultSet results = null;
+	try {
+	    statement = connection.createStatement();
+	    if (this.id == null) {
+		GS3SQLSelect select = new GS3SQLSelect("namespaces");
+		select.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE)));
+		select.addField("NamespaceRef");
+		results = statement.executeQuery(select.toString());
+		if (results.first()) {
+		    sqlId = Integer.toString(results.getInt("NamespaceRef"));
+		} else {
+		    statement.close();
+		    return false;
+		}
+	    }
+	    else {
+		sqlId = this.id;
+	    }
+	    
+	    // clear all the existing metadata items for this namespace
+	    GS3SQLDelete delete = new GS3SQLDelete("mdvalues");
+	    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", sqlId, GS3SQLField.INTEGER_TYPE));
+	    delete.setWhere(where);
+	    statement.execute(delete.toString());
+	    
+	    // write out the metadata for this namespace
+	    Iterator keys = this.metadataMap.keySet().iterator();
+	    while (keys.hasNext()){
+		String thisKey = keys.next().toString();
+		
+		Iterator values = this.metadataMap.getAll(thisKey).iterator();
+		
+		while (values.hasNext()){
+		    String value = values.next().toString();
+		    
+		    GS3SQLInsert insert = new GS3SQLInsert("mdvalues");
+		    insert.addValue("NamespaceRef", sqlId, GS3SQLField.INTEGER_TYPE);
+		    insert.addValue("Label", thisKey);
+		    insert.addValue("Value", value);
+		    statement.execute(insert.toString());
+		}
+	    }
+	    statement.close();
+	}
+	catch (SQLException sql) {
+	    System.out.println("SimpleNamespace.writeSQL(): "+sql);
+	}
+	return true;
+    }
+    
+    
+    /**
+     *  <p>Indicate whether this metadata is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java	(revision 14150)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  TODO: complete a class to create file identifiers
+ */
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class StructureIdentifierFactory extends AbstractIdentifierFactory
+{
+    public static final String STRUCTURE_IDENTIFIER_PRELUDE = "DS";
+    
+    public StructureIdentifierFactory()
+    {
+	super(STRUCTURE_IDENTIFIER_PRELUDE);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java	(revision 14150)
@@ -0,0 +1,166 @@
+package org.greenstone.gsdl3.gs3build.notifier;
+
+import java.io.*;
+import java.net.*;
+import java.sql.*;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public class NotifierManager {
+
+    public NotifierManager() {
+    }
+    
+    public void detectEvents(CollectionManager collManager) {
+	String notifyHost = collManager.getNotifyHost();
+	if (notifyHost == null)
+	    return;
+	
+	System.out.println("detecting events");
+	
+	String collectionName = collManager.getCollectionName();
+	String host = "http://localhost:8080/soap/servlet/rpcrouter";
+	
+	String lastBuild = dateToSqlString(collManager.getBuildDate());
+	System.out.println("last build was " + lastBuild);
+        try {
+	    Statement statement = collManager.getDatabase().createStatement();
+            // detect all new documents. A document is new if and only if
+            // AccessionDate >= CollectionLastRebuiltDate
+	    ResultSet results = statement.executeQuery("SELECT DocID FROM document WHERE DocType != 'GSMETADATA' AND AccessionDate >= " + lastBuild);
+	    notify(collectionName, host, notifyHost, results, collManager.getDatabase(), collManager.getBuildDate(), "new_document");
+	    
+	    // detect modified documents. A document is modified if and only if
+	    // AccessionDate < CollectionLastRebuiltDate (ie, it is not new) and 
+	    // IndexedDate >= CollectionLastRebuiltDate 
+	    results = statement.executeQuery("SELECT DocID FROM document WHERE DocType != 'GSMETADATA' AND AccessionDate < " + lastBuild + " AND IndexedDate >= " + lastBuild);
+	    notify(collectionName, host, notifyHost, results, collManager.getDatabase(), collManager.getBuildDate(), "document_modified");
+	    
+	    statement.close();
+	    // TODO deleted docs?
+        } catch (SQLException e) {
+	    // TODO Auto-generated catch block
+	    e.printStackTrace();
+        }
+    }
+    
+    /**
+     * @param collName
+     * @param hostURL
+     * @param results
+     * @param date
+     * @param eventType TODO
+     * @throws MalformedURLException
+     * @throws IOException
+     * @throws ProtocolException
+     * @throws SQLException
+     */
+    private void notify(String collName, String hostURL, String notifyHost, ResultSet results, GS3SQLConnection database, Date date, String eventType) {
+	try {
+	    System.out.println("notifyHost is " + notifyHost);
+	    URL url = new URL("http://" + notifyHost + "/alerting/service");
+	    System.out.println("trying to send to " + url);
+	    
+	    while(results.next()) {
+		System.out.println(eventType + ": " + results.getString("DocID"));
+	            
+                String documentID = results.getString("DocID");
+                
+		try {
+                    HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // TODO what about proxies?
+                    conn.setUseCaches(false);
+                    conn.setDoInput(true);
+                    conn.setDoOutput(true);
+                    conn.setRequestMethod("POST");
+                    conn.connect();
+                    // Construct data
+                    OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream());
+                    
+                    writer.write("action=receiveEvent&");
+                    writer.write("documentID=");
+                    writer.write(URLEncoder.encode(documentID, "UTF-8"));
+                    writer.write("&type=");
+                    writer.write(URLEncoder.encode(eventType, "UTF-8"));
+                    writer.write("&collectionID=");
+                    writer.write(URLEncoder.encode(collName, "UTF-8"));
+                    writer.write("&hostID=");
+                    writer.write(URLEncoder.encode(hostURL, "UTF-8"));
+                    writer.write("&document_title=");
+                    writer.write(URLEncoder.encode(getDocumentTitle(documentID, database), "UTF-8"));
+                    writer.write("&document_url=");
+                    writer.write(URLEncoder.encode("http://localhost:8080/gsdl3/library?a=d&c=" + collName + "&d=" + documentID, "UTF-8"));
+                    writer.write("&timestamp=");
+                    writer.write(URLEncoder.encode(new SimpleDateFormat("yyyy-MM-dd HH:mm z").format(date), "UTF-8"));
+                    writer.flush();
+                    System.out.println(conn.getResponseCode());
+                    InputStream fromReceiver = conn.getInputStream();
+                    int i;
+                    while ((i = fromReceiver.read()) != -1) {
+                        System.out.write(i);
+                    }
+                    conn.disconnect();
+                } catch (ProtocolException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                } catch (UnsupportedEncodingException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                } catch (IOException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                }
+	    }
+	} catch (SQLException e) {
+	    // TODO Auto-generated catch block
+	    e.printStackTrace();
+	} catch (MalformedURLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * @param documentID
+     * @return
+     * @throws SQLException
+     */
+    private String getDocumentTitle(String documentID, GS3SQLConnection database) {
+        System.out.println("getting title for document " + documentID + " from database");
+
+        try {
+	    Statement statement = database.createStatement();
+            String sqlString = "SELECT mdvalues.value " +
+		"FROM structure s JOIN divisions d ON (s.structureref = d.parentref) " +
+		"                 JOIN divisionmetarefs USING (divisionref) " + 
+		"                 JOIN metadata m USING (metaid) " +
+		"                 JOIN namespaces USING (metadataref) " + 
+		"                 JOIN mdvalues USING (namespaceref) " +
+		"  WHERE mdvalues.label = 'Title' AND s.docid = '" + documentID + "'" + 
+		"                                 AND s.structureType = 'Whole Document' " +
+		"                                 AND d.parenttype = 'Structure' " + 
+		"                                 AND m.docID = s.docid;";
+            ResultSet results = statement.executeQuery(sqlString);
+            if (results.first()) {
+                String title = results.getString("value");
+                System.out.println("title is " + title);
+                return title;
+            }
+	    statement.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        return "";
+    }
+
+    /**
+     * @param date
+     * @return
+     */
+    private String dateToSqlString(Date date) {
+        return new SimpleDateFormat("yyyyMMddHHmmss").format(date);
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java	(revision 14150)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class ChildWrapper
+{
+    int minOccurs;
+    int maxOccurs;
+
+
+    /**
+     *  The default number of occurrences for an element is 1 (one) for
+     *  both minimum and maximum counts.
+     */
+    public ChildWrapper()
+    {
+	this.minOccurs = 1;
+	this.maxOccurs = 1;
+    }
+
+    public void setMinOccurs(int minOccurs)
+    {
+	this.minOccurs = minOccurs;
+    }
+
+    public void setMaxOccurs(int maxOccurs)
+    {
+	this.maxOccurs = maxOccurs;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java	(revision 14150)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.*;
+
+public class EnumConstraint
+{
+    List options;
+
+    public EnumConstraint()
+    { this.options = new ArrayList();
+    }
+
+  public void addOption(String option)
+  {
+    this.options.add(option);
+  }
+
+  public boolean validate(String value)
+  {
+	  return this.options.contains(value);
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java	(revision 14150)
@@ -0,0 +1,34 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class RangeConstraint
+{
+    public static final int RANGE_MIN_EXCLUSIVE = 0x00;
+    public static final int RANGE_MAX_EXCLUSIVE = 0x01;
+    public static final int RANGE_MIN_INCLUSIVE = 0x10;
+    public static final int RANGE_MAX_INCLUSIVE = 0x11;
+
+    int type;
+    double value;
+
+    public RangeConstraint(int type, double value)
+    {
+	    this.type = type;
+	    this.value = value;
+    }
+
+  public boolean validate(double value)
+  {
+  	switch (this.type) {
+	  case RANGE_MIN_EXCLUSIVE:
+	    return (value > this.value);
+  	case RANGE_MAX_EXCLUSIVE:
+	    return (value < this.value);
+   case RANGE_MIN_INCLUSIVE:
+	    return (value >= this.value);
+   case RANGE_MAX_INCLUSIVE:
+	    return (value <= this.value);
+	}
+	return false;
+}
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/Schema.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/Schema.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/Schema.java	(revision 14150)
@@ -0,0 +1,78 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class Schema
+{
+	List children;
+
+	public Schema(Node node)
+	{
+		this.children = new ArrayList();
+
+		NodeList elements = node.getChildNodes();
+
+		System.out.println("Schema has " + elements.getLength() + " children");
+		for (int e = 0; e < elements.getLength(); e++) {
+			System.out.println("Subcomponent " + e);
+			// TODO: actually test what is being parsed here!
+			if (elements.item(e).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+				continue;
+			}
+
+			System.out.println(elements.item(e));
+
+			Element element = (Element) elements.item(e);
+
+			String elementName = element.getLocalName();
+			String fullName = element.getNodeName();
+			String valueName = element.getNodeValue();
+
+			System.out.println(elementName + " " + fullName + " " + valueName);
+
+			if (elementName == null) {
+				elementName = fullName;
+			}
+
+			System.out.println(elementName);
+			elementName = SchemaNode.getElementName(elementName);
+
+			if (elementName.equals("element")) {
+
+				SchemaElement thisElement = new SchemaElement(element);
+/*			
+				NodeList attributes = element.getElementsByTagName("attribute");
+
+				for (int i = 0; i < attributes.getLength(); i++) {
+			    Element attribute = (Element) attributes.item(i);
+					// handle that attribute, create its node and check what you get out
+					try {
+						SchemaAttribute thisAttribute = new SchemaAttribute(attribute);
+						thisElement.addAttribute(thisAttribute);
+					}
+					catch (SchemaException ex) {
+						// do nothing at the moment - though a malformed attribute should be
+						// cause for throwing everything in the air
+					}
+				}*/
+				this.children.add(thisElement);
+			}
+			else {
+				try {
+					Object child = SchemaNodeFactory.makeNode(element);
+					if (child != null) {
+						this.children.add(child);
+					}
+				}
+				catch (SchemaException metaEx)
+				{
+				}
+			}
+		}
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java	(revision 14150)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaAll extends SchemaNode
+{
+	List compulsory;
+
+	public SchemaAll(Element element) throws SchemaException
+	{	super(element);
+	}
+
+	protected List _allocate()
+	{	this.compulsory = new ArrayList();
+		return this.compulsory;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java	(revision 14150)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+public class SchemaAttribute
+{ 
+	public final static String ATTRIBUTE_TYPE_STRING = "String";
+
+  String name;
+  String type;
+  boolean required;
+
+	public SchemaAttribute(Element element) throws SchemaException
+	{
+		// get the name - things are totally unworkable if it doesn't exist!
+		this.name = element.getAttribute("name");
+		if (this.name == null) {
+			throw new SchemaException("All attributes must have names");
+		}
+
+		// ids are quite important in some metadata schema - copy that too
+		String id = element.getAttribute("id");
+
+		// get the required field - we don't actually cope with "prohibited" at the moment!
+		String use = element.getAttribute("use");
+		this.required = (use != null && use.equals("required"));
+
+		// and the type - we don't reflect any further upon this, actually
+		this.type = element.getAttribute("type");
+		if (this.type == null) {
+			this.type = SchemaAttribute.ATTRIBUTE_TYPE_STRING;
+		}
+
+		// Constraints should now be added, but at present that is not done, as we
+		// are non-validating!
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java	(revision 14150)
@@ -0,0 +1,23 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaChoice extends SchemaNode
+{
+	List alternatives;
+
+	public SchemaChoice(Element element) throws SchemaException
+	{
+		super(element);
+	}
+
+	protected List _allocate()
+	{	this.alternatives = new ArrayList();
+		return this.alternatives;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java	(revision 14150)
@@ -0,0 +1,65 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaComplexType implements SchemaType
+{
+	public String name;
+	public List children;
+
+	public SchemaComplexType(Element element) throws SchemaException
+	{
+		this.name = element.getAttribute("name");
+		if (name == null)	
+		{ throw new SchemaException("Type defined without name");
+		}
+
+		this.children = new ArrayList();
+
+		NodeList childNodes = element.getChildNodes();
+		for (int i = 0; i < childNodes.getLength(); i ++) {
+			Node node = childNodes.item(i);
+
+			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+				String nodeName = node.getNodeName();
+
+				if (nodeName == null) {
+					continue;
+				}
+
+				nodeName = SchemaNode.getElementName(nodeName);
+				// a simple-based type
+				if (node.getNodeName().equals("simpleContent")) {
+					// we're really just a simple type with attributes - nothing more to do
+				}
+				// a complex-based type
+				else if (node.getNodeName().equals("complexContent")) {
+					// we just contain elements, no text
+				}
+				// the following pretty much speak for themselves
+				else {
+					try {
+						Object child = SchemaNodeFactory.makeNode((Element) node);
+						if (child != null) {
+							this.children.add(child);
+						}
+					}
+					catch (SchemaException ex)
+					{	
+					}
+				}
+			}
+		}
+
+		SchemaTypeMap.theMap.addType(this);
+	}
+	
+	public String getName()
+	{	return this.name;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java	(revision 14150)
@@ -0,0 +1,104 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.*;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SchemaElement
+{ 
+	// the following may contain duplicates when compared to SchemaAttribute, but this is
+	// deliberate...
+	public static final String ELEMENT_TYPE_STRING = "String";
+
+  String name;
+	String type;
+  List   attributes;
+  List   childElements;
+	int    minOccurs;
+	int    maxOccurs;
+
+	public SchemaElement(Element element)
+	{
+		// get the name of this element
+		this.name = element.getAttribute("name");
+		if (this.name == null) {
+			// TODO: this is a reference object
+			String ref = element.getAttribute("ref");
+
+			if (ref != null) {
+				// TODO: fetch the actual object
+			}
+			else {
+				// TODO: abort
+			}
+		}
+
+		this.type = element.getAttribute("type");
+		if (this.type == null || this.type.length() == 0)
+		{ this.type = SchemaElement.ELEMENT_TYPE_STRING;
+		}
+
+		String temp;
+		temp = element.getAttribute("minOccurs");
+		if (temp != null && temp.length() > 0) {
+			this.minOccurs = Integer.parseInt(temp);
+			this.minOccurs = 1;
+		}
+		temp = element.getAttribute("maxOccurs");
+		if (temp != null && temp.length() > 0) {
+			this.maxOccurs = Integer.parseInt(temp);
+		}
+		else {
+			this.maxOccurs = 1;
+		}
+
+		this.expand(element);
+	}
+
+	private void expand(Element element)
+	{ // TODO
+		// get the first child element
+		NodeList children = element.getChildNodes();
+		if (children != null) {
+			for (int i = 0; i < children.getLength(); i ++)
+			{ if (children.item(i).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+					// TODO: raise an error
+					continue;
+				}
+
+				String name = children.item(i).getNodeName();
+				if (name == null) {
+					continue;
+				}
+				
+				try {
+					Object child = SchemaNodeFactory.makeNode((Element) children.item(i));
+					if (child != null) {
+						this.childElements.add(child);
+					}
+				}
+				catch (SchemaException ex)
+				{
+				}
+			}
+		}
+	}
+
+  public SchemaElement(String name)
+  { this.name = name;
+    this.attributes = new ArrayList();
+    this.childElements = new ArrayList();
+  }
+
+  public void addAttribute(SchemaAttribute attribute)
+  {
+	  this.attributes.add(attribute);
+  }
+
+    public void addElement(SchemaElement element)
+    {
+			this.childElements.add(element);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaException.java	(revision 14150)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class SchemaException extends Exception
+{
+	public SchemaException(String name)
+	{
+		super(name);
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java	(revision 14150)
@@ -0,0 +1,21 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaGroup extends SchemaNode
+{
+	List members;
+
+  public SchemaGroup(Element element) throws SchemaException
+  {
+		super(element);
+  }
+
+	public List _allocate()
+	{	this.members = new ArrayList();
+		return this.members;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java	(revision 14150)
@@ -0,0 +1,68 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+
+public abstract class SchemaNode
+{
+	public SchemaNode(Element element) throws SchemaException
+	{	List childList = this._allocate();
+		this.initialise(element, childList);
+		System.out.println(element.getNodeName());
+	}
+	
+	abstract protected List _allocate();
+
+	protected void initialise(Element element, List childList) throws SchemaException
+	{
+		// TODO: check minimum, maximum occurrences etc.
+
+		NodeList children = element.getChildNodes();
+		if (children != null) {
+			System.out.println("  " + children.getLength() + " children");
+			for (int i = 0; i < children.getLength(); i ++)
+			{ if (children.item(i).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+					Node child = children.item(i);
+
+					String type = child.getNodeName();
+					System.out.println("Child is " + type);
+
+					if (type == null) {
+						throw new SchemaException("Badly defined child - no name");
+					}
+
+					type = SchemaNode.getElementName(type);
+
+					if (type.equals("element")) {
+						SchemaElement elementChild = new SchemaElement((Element) children.item(i));
+						childList.add(elementChild);
+					}
+					else {
+						Object elementChild = SchemaNodeFactory.makeNode((Element) child);
+						if (elementChild != null) {
+							childList.add(elementChild);
+						}
+					}
+					// TODO: do an abstraction between here and SchemaElement
+				}
+			}
+		}
+	}
+
+	public static String getElementName(String fullname)
+	{	if (fullname.indexOf(':') >= 0) {
+			return fullname.substring(fullname.indexOf(':') + 1);
+		}
+		return fullname;
+	}
+
+	public static String getNamespace(String fullname)
+	{	if (fullname.indexOf(':') >= 0) {
+			return fullname.substring(0, fullname.indexOf(':'));
+		}
+		return null;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java	(revision 14150)
@@ -0,0 +1,33 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+public class SchemaNodeFactory
+{
+	public static Object makeNode(Element element) throws SchemaException
+	{
+		String elementName = element.getNodeName();
+
+		System.out.println("Element name " + elementName);
+
+		elementName = SchemaNode.getElementName(elementName);
+
+		if (elementName.equals("choice")) {			
+			return new SchemaChoice(element);
+		}
+		else if (elementName.equals("group")) {
+			return new SchemaGroup(element);
+		}
+		else if (elementName.equals("all")) {
+			return new SchemaAll(element);
+		}
+		else if (elementName.equals("sequence")) {
+			return new SchemaSequence(element);
+		}
+		else if (elementName.equals("element")) {
+			return new SchemaElement(element);
+		}
+		return null;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java	(revision 14150)
@@ -0,0 +1,73 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.io.*;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+public class SchemaParser
+{
+	public SchemaParser(File file, String namespace)
+	{
+	  try {
+			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+			DocumentBuilder builder = factory.newDocumentBuilder();
+			Document document = builder.parse(file);
+
+			// TODO: get all the types in the tree
+
+			// TODO: do a traverse, and thus cope with elements-within-elements if needsbe, but
+			// this shouldn't happen except in a directly defined situation - actually doing the
+			// parsing in part inside each node would work well provided one checked for a node
+			// having already been done...
+			System.out.println("Read");
+
+			NodeList schemas = document.getElementsByTagName("xs:schema");
+			if (schemas.getLength() == 0) {
+				schemas = document.getElementsByTagName("xsd:schema");
+			}
+			if (schemas.getLength() == 0) {
+				schemas = document.getElementsByTagName("schema");
+			}
+
+			System.out.println(" " + schemas.getLength() + " schemas");
+
+			for (int s = 0; s < schemas.getLength(); s ++) {
+				Schema schema = new Schema(schemas.item(s));
+			}
+		}
+		catch (FactoryConfigurationError e) {
+			System.out.println(e);
+		}
+		catch (ParserConfigurationException ex) {
+			System.out.println(ex);
+		}
+		catch (SAXException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println(ex);
+		}
+	}
+
+	public static void main(String args[])
+	{	try
+		{	File file = new File(args[0]);
+
+			SchemaParser parser = new SchemaParser(file, "");			
+		}
+		catch (Exception ex)
+		{ System.out.println(ex);
+		}
+		return;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java	(revision 14150)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.HashMap;
+
+public class SchemaRefMap
+{
+	HashMap map;
+
+	public SchemaRefMap()
+	{
+		this.map = new HashMap();
+	}
+
+	public void addItem(String reference, Object item)
+	{
+		this.map.put(reference, item);
+	}
+
+	public Object getItem(String reference)
+	{	return this.map.get(reference);
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java	(revision 14150)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaSequence extends SchemaNode
+{
+	List sequence;
+
+	public SchemaSequence(Element element) throws SchemaException
+	{	super(element);
+	}
+
+	public List _allocate()
+	{	this.sequence = new ArrayList();
+		return this.sequence;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java	(revision 14150)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SchemaSimpleType implements SchemaType
+{
+	String name;
+
+	public void SchemaSimpleType(Element element) throws SchemaException
+	{
+		this.name = element.getAttribute("name");
+		if (name == null)	
+		{ throw new SchemaException("Type defined without name");
+		}
+
+		NodeList childNodes = element.getChildNodes();
+		for (int i = 0; i < childNodes.getLength(); i ++) {
+			Node node = childNodes.item(i);
+
+			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+			}
+		}
+	}
+
+	public String getName()
+	{	return this.name;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaType.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaType.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaType.java	(revision 14150)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public interface SchemaType
+{
+	public String getName();
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java	(revision 14150)
@@ -0,0 +1,26 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SchemaTypeMap
+{
+	Map map;
+	public static final SchemaTypeMap theMap;
+
+	static {
+		theMap = new SchemaTypeMap();
+	}
+
+	public SchemaTypeMap()
+	{	this.map = new HashMap();
+	}
+
+	public void addType(SchemaType type)
+	{	this.map.put(type.getName(), type);
+	}
+
+	public SchemaType getType(String name)
+	{	return (SchemaType) this.map.get(name);
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/DOMUtils.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/DOMUtils.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/DOMUtils.java	(revision 14150)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+public class DOMUtils
+{
+  public static String getNodeAttribute(Node node, String attribute)
+  {
+    NamedNodeMap atts = node.getAttributes();
+    if (atts == null) {
+      return null;
+    }
+    Node attributeNode = atts.getNamedItem(attribute);
+    if (attributeNode == null) {
+      return null;
+    }
+    return attributeNode.getNodeValue();
+  }
+    
+  public static String getNodeChildText(Node node)
+  {
+    StringBuffer fieldName = new StringBuffer();
+
+    NodeList fieldChildren = node.getChildNodes();
+    for (int f = 0; f < fieldChildren.getLength(); f ++) {
+      if (fieldChildren.item(f).getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	fieldName.append(fieldChildren.item(f).getNodeValue());
+      }
+    }
+    return fieldName.toString();
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/FileTrie.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/FileTrie.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/FileTrie.java	(revision 14150)
@@ -0,0 +1,145 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class FileTrie
+{	String   stem;
+	List     childTries;
+	List     childDocs;
+	FileTrie parent;
+
+	public FileTrie(String stem)
+	{	this.childTries = new ArrayList();
+		this.childDocs  = new ArrayList();
+		this.stem = stem;
+		this.parent = null;
+	}
+
+	public void setParent(FileTrie trie)
+	{	this.parent = trie;
+	}
+
+	public FileTrie getParent()
+	{	return this.parent;
+	}
+
+	public void addChild(FileTrie trie)
+	{	trie.setParent(this);
+		this.childTries.add(trie);
+	}
+
+	public void replaceChild(FileTrie oldChild, FileTrie newChild)
+	{	for (int i = 0; i < this.childTries.size(); i ++)
+		{	FileTrie thisChild = (FileTrie) this.childTries.get(i);
+			if (thisChild == oldChild)
+			{	this.childTries.set(i, newChild);
+				return;
+			}
+		}
+	}
+
+	public void addDocument(DocumentID documentID)
+	{	this.childDocs.add(documentID);
+	}
+
+	public boolean postDocument(String filename, DocumentID documentID)
+	{	int common = 0;
+		int i;
+		
+		while (common < filename.length() &&
+		       common < this.stem.length() &&
+					  filename.charAt(common) == stem.charAt(common))
+		{	common ++;
+		}
+
+		if (common == 0)
+		{ return false;
+		}
+
+		if (common == this.stem.length())
+		{	// post it in this node
+			filename = filename.substring(this.stem.length());
+
+			// select which sub-trie to attempt...
+			for (i = 0; i < this.childTries.size(); i ++)
+			{	FileTrie child = (FileTrie) this.childTries.get(i);
+				
+				if (child.postDocument(filename, documentID))
+				{	break;
+				}
+			}
+
+			if (i == this.childTries.size())
+			{ FileTrie newChild = new FileTrie(filename);
+				this.addChild(newChild);
+
+				newChild.addDocument(documentID);
+			}
+		}
+		else
+		{	// divide this node at the 'common' point...
+			FileTrie parentTrie = new FileTrie(this.stem.substring(0, common));
+			this.parent.replaceChild(this, parentTrie);
+
+			// truncate our root
+			this.stem = this.stem.substring(common);
+
+			// add ourselves to our new parent
+			parentTrie.addChild(this);
+
+			// trim back the file name
+			filename = filename.substring(common);
+
+			// recurse into posting the document
+			this.postDocument(filename, documentID);
+		}
+		return true;
+	}
+
+	public boolean postFile(String filename)
+	{	return this.postDocument(filename, null);
+	}
+
+	public List getDocuments(String filename)
+	{	FileTrie leaf = this.getNode(filename);
+		return leaf.childDocs;
+	}
+
+	public FileTrie getNode(String filename)
+	{	int common = 0;
+		int i;
+		
+		while (common < filename.length() &&
+		       common < this.stem.length() &&
+					 filename.charAt(common) == stem.charAt(common))
+		{	common ++;
+		}
+
+		if (common == 0)
+		{ return null;
+		}
+
+		if (common == this.stem.length())
+		{	if (common == filename.length())
+			{	return this;
+			}
+
+			filename = filename.substring(this.stem.length());
+
+			// select which sub-trie to attempt...
+			for (i = 0; i < this.childTries.size(); i ++)
+			{	FileTrie child = (FileTrie) this.childTries.get(i);
+				
+				child = child.getNode(filename);
+
+				if (child != null)
+				{	return child;
+				}
+			}
+		}
+		return null;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java	(revision 14150)
@@ -0,0 +1,111 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class GS2TextFileHandler
+{
+  String  content;
+  String  line;
+
+  public GS2TextFileHandler(String content)
+  { this.content = content;
+  }
+    
+  public boolean hasMore()
+  { return this.line != null;
+  }
+
+  public boolean hasMoreLines()
+  { return this.content != null;
+  }
+
+  public String getEntry()
+  { return this.getEntry(false);
+  }
+
+  public String getEntry(boolean breakSpace)
+  { String reply;
+    int start, tab = 0;
+    boolean quoted = false;
+
+    if (!this.hasMore()) {
+      return null;
+    }
+
+    start = 0;
+    while (start < this.line.length() &&
+	   this.line.charAt(start) == ' ') {
+      start ++;
+    }
+
+    if (start == this.line.length()) {
+      this.line = null;
+      return null;
+    }
+    
+    if (this.line.charAt(start) == '"') {
+      quoted = true;
+      breakSpace = false;
+      start ++;
+    }
+    tab = start;
+
+    while (tab != this.line.length() &&
+	   this.line.charAt(tab) != '\t' &&
+	   !(quoted && this.line.charAt(tab) == '"') &&
+	   !(this.line.charAt(tab) == ' ' && breakSpace))
+    { tab ++;
+    }
+
+    if (start > 0) {
+      this.line = this.line.substring(start);
+      tab -= start;
+    }
+    
+    if (tab == this.line.length()) {
+      reply = this.line;
+      this.line = null;
+    }
+    else {
+      reply = this.line.substring(0, tab);
+      this.line = this.line.substring(tab+1);
+    }
+
+    reply.trim();
+    
+    return reply;
+  }
+
+  public String getLine()
+  { if (this.content == null) {
+      this.line = null;
+      return null;
+    }
+	
+    do {
+      int eol = this.content.indexOf('\n');
+      if (eol < 0) {
+	this.line = this.content;
+	this.content = null;
+      }
+      else {
+	this.line = this.content.substring(0, eol);
+	this.content = this.content.substring(eol+1);
+	while (this.content.length() > 0 &&
+	       this.content.charAt(0) < ' ')
+	{ this.content = this.content.substring(1);
+	}
+      }
+
+      if (this.line != null) {
+	this.line.trim();
+      }
+      if (this.line.length() > 0 &&
+	  this.line.charAt(this.line.length() - 1) == (char) 13) {
+	this.line = this.line.substring(0, this.line.length()-1);
+	this.line.trim();
+      }
+    } while (this.content != null && this.line != null && this.line.length() == 0);
+    
+    return this.line;
+  }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java	(revision 14150)
@@ -0,0 +1,383 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLBlock
+{
+    Vector content;
+    int    tagpos;
+    int    endpos;
+
+    public HTMLBlock(int docpos)
+    { 
+	this.content = new Vector(1);
+	this.tagpos  = docpos;
+    }
+    
+    public HTMLBlock(int docpos, int endpos)
+    { 
+	this.content = new Vector(1);
+	this.tagpos  = docpos;
+	this.endpos  = endpos;
+    }
+
+    public void addTag(HTMLTag tag)
+    { 
+	if (tag.endPos() > this.endpos){
+	    this.endpos = tag.endPos();
+	}
+	content.addElement(tag);
+    }
+
+    /**
+     *  @deprecccate
+     public void addText(String text)
+     { content.addElement(text);
+     }
+    */
+
+    public void addText(HTMLCText text)
+    { 
+	if (text.endPos() > this.endpos){
+	    this.endpos	= text.endPos();
+	}
+	content.addElement(text);
+    }
+    
+    /**
+     * @return HTML encoded <code>String</code> of the document
+     */
+    public String contentString()
+    { 
+	int member;
+	StringBuffer reply;
+	String type;
+	
+	reply = new StringBuffer();
+	for (member = 1; member < content.size() - 1; member ++){
+	    /*if (content.elementAt(member) instanceof String){
+	      reply.append((String) content.elementAt(member));
+	      }
+	      else 
+	  */
+	    if (content.elementAt(member) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(member)).toString());
+	    }
+	}
+	return reply.toString();
+    }
+    
+    /**
+     * Returns document position of start of the block
+     */
+    public int startPos()
+    { 
+	return this.tagpos;
+    }
+
+    /**
+     * Returns document position of end of the block
+     */
+    public int endPos()
+    { 
+	return this.tagpos;
+    }
+
+    /**
+     * @return head (first) tag of the block
+     */
+    public HTMLTag headTag()
+    { 
+	return ((HTMLTag) content.elementAt(0));
+    }
+
+    /**
+     * @return name (as a <code>String</code>) of the tag at the head of the block
+     */
+    public String headTagName()
+    { 
+	return ((HTMLTag) content.elementAt(0)).tagName();
+    }
+
+    /**
+     * @return name of the tail (last) tag of the block as a <code>String</code>
+     */
+    public String tailTagName()
+    { 
+	return (((HTMLTag) content.elementAt(content.size() - 1)).tagName());
+    }
+	
+	
+    /**
+     *  @return: starting element character position of the item-th item
+     */
+    private int elementStartPos(int item)
+    { 
+	Object element;
+	int	pos = -1;
+		
+	element = this.content.elementAt(item);
+	if (element instanceof HTMLTag){
+	    pos	= ((HTMLTag) element).startPos();
+	}
+	else if (element instanceof HTMLCText){
+	    pos	= ((HTMLCText) element).startPos();
+	}
+	return pos;		
+    }
+    
+    /**
+     *  @return: ending element character position of the item-th item
+     */
+    private int elementEndPos(int item)
+    { 
+	Object	element;
+	int		pos = -1;
+	
+	element = this.content.elementAt(item);
+	if (element instanceof HTMLTag){
+	     pos = ((HTMLTag) element).endPos();
+	}
+	else if (element instanceof HTMLCText){
+	     pos = ((HTMLCText) element).endPos();
+	}
+	return pos;		
+    }
+
+    /**
+     *  @return the HTMLBlock of the indicated subitems
+     */
+    public HTMLBlock subBlock(int startitem, int enditem)
+    { 
+	int       i;
+	Object    element;
+	HTMLBlock reply;
+	reply = new HTMLBlock(this.elementStartPos(startitem),
+			      this.elementEndPos(enditem-1));
+	// copy all the bits
+	for (i = startitem; i < enditem; i++){
+	    element = this.content.elementAt(i);
+	    if (element instanceof HTMLTag){
+		reply.addTag((HTMLTag) element);
+	    }
+	    else if (element instanceof HTMLCText){
+		reply.addText((HTMLCText) element);
+	    }
+	/*
+	  else if (element instanceof String)
+	  {	reply.addText((String) element);
+	  }
+	*/
+	}
+	return reply;
+    }
+
+    /**
+     * Return the position of the given <code>HTMLBlock</code> within this block
+     * @return the tag index of the child block; or < 0 if not a chile
+     */
+    public int find(HTMLBlock subblock)
+    { 
+	int start, end, at, attag;
+	HTMLTag tag;
+	
+	start = 0;
+	end = this.content.size();
+	while (start != end){
+	     /* compare positions */
+	    at = (start + end) / 2;
+	    attag = at;
+	    while (attag < end &&
+		   (content.elementAt(attag) instanceof HTMLTag) == false){
+		attag ++;
+	    }
+	    
+	    if (attag == end){
+		end = at;
+		continue;
+	    }
+	    tag =	(HTMLTag) content.elementAt(attag);
+	    if (tag.startPos() > subblock.startPos()){
+		end = at;				
+	    }
+	    else if (tag.startPos() < subblock.startPos()){
+		start = attag + 1;
+	    }
+	    else { 
+		return attag;
+	    }
+	}
+	return -1;
+    }
+    
+    /**
+     * @return The raw HTML of the block
+     */
+    public String HTMLString()
+    { 
+	int loop;
+	StringBuffer reply;
+
+	reply	= new StringBuffer();
+	for (loop = 0; loop < this.content.size(); loop ++){
+	    /*
+	      if (content.elementAt(loop) instanceof String)
+	      {	reply.append((String) content.elementAt(loop));
+	      }
+	      else 
+	    */
+	    if (content.elementAt(loop) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(loop)).toString());
+	    }
+	    else if (content.elementAt(loop) instanceof HTMLTag){
+		reply.append(((HTMLTag) content.elementAt(loop)).toString());
+	    }
+	}
+	return reply.toString();
+    }
+    
+    /**
+     * @return The text of the block - no HTML tags.  If no text is present, any
+     *         <code>alt</code> information for <code>img</code> tags will be given 
+     *         instead
+     */
+    public String toString()
+    {
+	int member;
+	StringBuffer reply;
+	StringBuffer ireply;
+	
+	reply = new StringBuffer("");
+	ireply = new StringBuffer("");
+	for (member = 0; member < content.size(); member ++){
+	     /*if (content.elementAt(member) instanceof String)
+	       { reply.append((String) content.elementAt(member));
+	       }
+	     */
+	    if (content.elementAt(member) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(member)).toString());
+	    }
+	    else if (content.elementAt(member) instanceof HTMLTag){
+		if (((HTMLTag) content.elementAt(member)).tagName().equals("img")){
+		    ireply.append(((HTMLTag) content.elementAt(member)).idValue("alt"));
+		}
+	    }
+	}
+	reply = HTMLCText.cleanString(reply);
+	ireply = HTMLCText.cleanString(ireply);
+	//		reply = reply.trim();
+	if (reply == null || reply.length() == 0){
+	    return ireply.toString();
+	}
+	return reply.toString();
+    }
+    
+    public HTMLObject elementAt(int at)
+    { 
+	return (HTMLObject) this.content.elementAt(at);
+    }
+
+    /**
+     * @return the number of elements in the block
+     */
+    public int size()
+    { 
+	return this.content.size();
+    }
+
+    /**
+     * @return an enumeration of all elements in the block
+     */
+    public Enumeration elements()
+    { 
+	Enumeration enumer;
+  
+	enumer = new HTMLBlockEnumerator(this);
+	return enumer;
+    }
+
+    /**
+     * @return an enumeration of all tags in the block
+     */
+    public Enumeration tagElements()
+    { 
+	Enumeration enumer;
+	
+	enumer = new HTMLBlockTagEnumerator(this);
+	return enumer;
+    }
+
+    /**
+     * Finalization method
+     */
+    protected void finalize() throws Throwable
+    { 
+	this.content	= null;
+	super.finalize();
+    }
+}
+
+final class HTMLBlockEnumerator implements Enumeration
+{ 
+    private HTMLBlock block;
+    int			member;
+
+    public HTMLBlockEnumerator(HTMLBlock block)
+    { 
+	this.block	= block;
+	this.member	= 0;
+    }
+
+    public Object nextElement()
+    { 
+	Object element;
+
+	element = this.block.content.elementAt(member);
+	member ++;
+	return element;
+    }
+
+    public boolean hasMoreElements()
+    {
+	if (block == null || this.member == this.block.content.size()){
+	    return false;
+	}
+	return true;
+    }
+}
+
+final class HTMLBlockTagEnumerator implements Enumeration
+{
+    private HTMLBlock block;
+    int	member;
+    
+    public HTMLBlockTagEnumerator(HTMLBlock block)
+    { 
+	this.block = block;
+	this.member = 0;
+	while ( this.member < this.block.content.size() &&
+		this.block.content.elementAt(this.member) instanceof HTMLTag == false){
+	    this.member ++;
+	}
+    }
+
+    public Object nextElement()
+    { 
+	Object element;
+	
+	element = this.block.content.elementAt(member);
+	do {
+	    member ++;
+	} while (this.block.content.elementAt(member) instanceof HTMLTag == false);
+	return element;
+    }
+    
+    public boolean hasMoreElements()
+    { 
+	if (block == null || this.member == this.block.content.size()){
+	    return false;
+	}
+	return true;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java	(revision 14150)
@@ -0,0 +1,409 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLBlockList
+{ Vector blocks;
+	String title;
+	protected static HTMLTagVector anchorStartTags;
+	protected static HTMLTagVector anchorEndTags;
+	protected static HTMLTagVector anchorOptEndTags;
+	protected static HTMLTagVector headingStartTags;
+	protected static HTMLTagVector headingEndTags;
+
+	static
+	{	anchorStartTags 	= new HTMLTagVector("a");
+		anchorEndTags			= new HTMLTagVector("a,h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		anchorOptEndTags 	= new HTMLTagVector("div,center,form");
+		headingStartTags 	= new HTMLTagVector("h1,h2,h3,h4,h5,h6");
+		headingEndTags	 	= new HTMLTagVector("h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+	}
+
+	// opening   opentag  incontent	result
+	//    T					T					T				*       {if opening, incontent must be false}
+	//    T					T					F				T
+	//    T					F					T				*       {if opening, incontent must be false}
+	//    T					F					F				F
+	//    F					T					T				T
+	//    F					T					F				F				{if getting an opening tag
+	//		F					F					T				T
+	//		F					F					F				T
+	private boolean inTagList(HTMLTag tag, Vector tags, boolean opening)
+	{ int tagref;
+
+		for (tagref = 0; tagref < tags.size(); tagref ++)
+		{ if ((tag.xtagName().equals((String) tags.elementAt(tagref))) &&
+		      (tag.tagIsOpening() || !opening))
+			{ return true;
+			}
+		}
+		return false;
+	}
+
+	//
+	// Common initialisation; parse the document for text surrounded by
+	// pairs of <starttag>....<endtag/starttag> combos
+	//
+	protected void initialise(HTMLDoc doc, HTMLTagVector starttags, HTMLTagVector endtags, HTMLTagVector optendtags)
+	{ boolean intag;
+		boolean incontent;
+		HTMLCText		text;
+		HTMLTag			tag;
+		HTMLTag 		starttag;
+		HTMLBlock 	block;
+		HTMLBlock		docBlock;
+		Enumeration	elements;
+		Object			element;
+
+		this.blocks = new Vector(1);
+		block				= null;
+		intag				= false;
+		incontent		= false;
+		starttag		= null;
+
+		docBlock		= doc.getCodedContent();
+		elements		= docBlock.elements();
+
+		while (elements.hasMoreElements())
+		{ element	= elements.nextElement();
+
+			// if it's a tag
+			if (element instanceof HTMLTag)
+			{ tag = (HTMLTag) element;
+			
+				// end the fragment if getting a closing tag
+				if (intag)
+				{ if (endtags.matches(tag, false) ||
+							(incontent && optendtags.matches(tag, false)))
+					{ block.addTag(starttag.endTag());
+						intag = false;
+						incontent = false;
+					}
+					else
+					{ block.addTag(tag);
+					}
+				}
+
+				// !intag is checked for explicitly, as it may have been
+				// switched off by a non-partner opening tag above: eg:
+				//   <H2>A Heading<H3>A Smaller Heading</H3>
+				//                ^^^^
+				//                here
+
+				// start fragment if the tag is a starter
+				if (!intag)
+				{ if (starttags.matches(tag, true))
+					{ block = new HTMLBlock(tag.startPos(), tag.endPos());
+						this.blocks.addElement((Object) block);
+						block.addTag(tag);
+						starttag = tag;
+						intag = true;
+						incontent = false;
+					}
+				}
+			}	
+			// if the parse returned text
+			else
+			{ text	= (HTMLCText) element;
+
+				if (intag)
+				{ block.addText(text);
+					if (!text.nullString())
+					{ incontent = true;
+					}
+				}
+			}
+		}
+
+		// if a tag wasn't "finished");
+		if (intag)
+		{ block.addTag(starttag.endTag());
+		}
+
+		this.title = doc.urlString();
+	}
+
+	protected HTMLBlockList()
+	{
+	}
+	
+	//
+	// constructor for given vectors of tags: startlist elements always start a new
+	// block [ending an open one if necessary]; endlist elements always end an open 
+	// block; optendlist elements close a block iff a block has been started, and
+	// some non-blank content (ie. some "real" text, not just returns/spaces) has
+	// been read since the beginning of the block
+	//
+	public HTMLBlockList(HTMLDoc doc, Vector starttags, Vector endtags, Vector optendtags)
+	{ HTMLTagVector _starttags;
+		HTMLTagVector _endtags;
+		HTMLTagVector _optendtags;
+
+		_starttags	= new HTMLTagVector(starttags);
+		_endtags		= new HTMLTagVector(starttags);
+		_optendtags	= new HTMLTagVector(starttags);
+		this.initialise(doc, _starttags, _endtags, _optendtags);
+	}
+	
+	//
+	// constructor for explicit lists; a tagVector is created for each list;
+	// see above for list roles
+	//
+	public HTMLBlockList(HTMLDoc doc, String startlist, String endlist, String optendlist)
+	{ HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		starttags		= new HTMLTagVector(startlist);
+		endtags			= new HTMLTagVector(endlist);
+		optendtags	= new HTMLTagVector(optendlist);
+		this.initialise(doc, starttags, endtags, optendtags);
+	}
+
+	//
+	// constructor; get all HTMLBlocks for h{level} through to h1;
+	// eg hlevel==3 test h3,h2,h1
+	//
+	public HTMLBlockList(HTMLDoc doc, int hlevel)
+	{ int loop;
+		StringBuffer levelstring;
+		HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		levelstring = new StringBuffer("h1");
+		for (loop = 2; loop <= hlevel; loop ++)
+		{ levelstring.append(",h");
+		  levelstring.append(Integer.toString(loop));
+		}
+
+		starttags		= new HTMLTagVector(levelstring.toString());
+		levelstring.append(",table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		endtags			= new HTMLTagVector(levelstring.toString());
+		optendtags	= new HTMLTagVector("div,center,form");
+
+		this.initialise(doc, starttags, endtags, optendtags);
+	}
+
+	/**
+	 * Static method returning list of anchors of a document
+	 */
+	public static synchronized HTMLBlockList anchorlist(HTMLDoc doc)
+	{ HTMLBlockList anchors;
+		HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		anchors = new HTMLBlockList();
+
+		anchors.initialise(doc, anchorStartTags, anchorEndTags, anchorOptEndTags);
+
+		return anchors;
+	}
+
+	//
+	// return how many tags are in the block list
+	//
+	public int size()
+	{ if (this.blocks == null)
+		{ return 0;
+		}
+
+		return this.blocks.size();
+	}
+
+	/**
+	 * @return number of blocks in the block list
+	 * @deprecated
+	 */
+	public int tagCount()
+	{	return this.size();
+	}
+
+	//
+	// return the position of the tag in the document
+	//
+	public int tagPos(int tag)
+	{ HTMLBlock block;
+
+		if (this.blocks == null)
+		{ return 0;
+		}
+		block = (HTMLBlock)	this.blocks.elementAt(tag);
+		return block.startPos();
+	}
+
+	//
+	// return the nth block in the list
+	//
+	public HTMLBlock tagBlock(int tag)
+	{ HTMLBlock block;
+
+		block = (HTMLBlock) this.blocks.elementAt(tag);
+		return block;
+	}
+
+	//
+	// return the index of the next blocklist of the same level
+	//
+	public int tagNext(int attag)
+	{ HTMLBlock block;
+		String		headTag;
+		int				nextTag;
+
+		block = (HTMLBlock) this.blocks.elementAt(attag);
+		headTag = block.headTagName();
+		nextTag	= attag+1;
+
+		while (nextTag < this.blocks.size())
+		{	block	=	(HTMLBlock) this.blocks.elementAt(nextTag);
+			if (block.headTagName().equals(headTag))
+			{	break;
+			}
+			nextTag ++;
+		}
+		return nextTag;
+	}
+
+	//
+	// return the number of children at the next heading level
+	// 
+	public int tagChildren(int fromtag, int level)
+	{ int sibling;
+		int descendent;
+		int children;
+		int	atlevel;
+		HTMLTag tag;
+
+		// get current tag level; level 6 headings cannot have
+		// children
+		if (level == 6)
+		{ return 0;
+		}
+
+		// now, if the next tag at this level is the next tag in
+		// the document, then we can't have any children
+		if (fromtag > 0)
+		{ sibling	= this.tagNext(fromtag-1);
+		}
+		else
+		{	sibling	= this.size();
+		}
+		if (sibling == fromtag)
+		{ children = 0;
+		}
+		// if the next tag at this level is *not* next, then we have
+		// at least one child (ie the next tag in sequence), even if
+		// it is not at the next heading level down.
+		//
+		// if no further children occur at the next level down, we
+		// have only one child.  If any further children occur at the
+		// next level down, then they are counted, except where it is
+		// the next tag in sequence, which is automatically accounted
+		// for
+		else
+		{	children	= 1;
+			for (descendent = fromtag+1; descendent < sibling; descendent ++)
+			{ tag	= this.tagBlock(descendent).headTag();
+				
+				if (tag.tagLevel() ==	level + 1)
+				{	children ++;
+				}
+			}
+		}
+		return children;
+	}
+
+
+	public int tagChildren(int attag)
+	{ int sibling;
+		int descendent;
+		int children;
+		int	atlevel;
+		HTMLTag tag;
+
+		// get current tag level; level 6 headings cannot have
+		// children
+		atlevel		=	this.tagBlock(attag).headTag().tagLevel();
+		if (atlevel == 6)
+		{ return 0;
+		}
+
+		// now, if the next tag at this level is the next tag in
+		// the document, then we can't have any children
+		sibling		= this.tagNext(attag);
+		if (sibling == attag + 1)
+		{ children = 0;
+		}
+		// if the next tag at this level is *not* next, then we have
+		// at least one child (ie the next tag in sequence), even if
+		// it is not at the next heading level down.
+		//
+		// if no further children occur at the next level down, we
+		// have only one child.  If any further children occur at the
+		// next level down, then they are counted, except where it is
+		// the next tag in sequence, which is automatically accounted
+		// for
+		else
+		{	children	= 1;
+			for (descendent = attag + 2; descendent < sibling; descendent ++)
+			{ tag	= this.tagBlock(descendent).headTag();
+				
+				if (tag.tagLevel() ==	atlevel + 1)
+				{	children ++;
+				}
+			}
+		}
+		return children;
+	}
+
+	//
+	// write out a structured output onto System.out
+	//
+  public void sysoutList()
+	{ int item;
+		HTMLBlock block;
+
+		if (this.blocks == null)
+		{ return;
+		}
+
+		for (item = 0; item < this.blocks.size(); item ++)
+		{ block = (HTMLBlock) this.blocks.elementAt(item);
+			System.out.println("  @"+ block.startPos()+":"+block.headTagName() +":"+ block);
+		}
+	}
+
+	//
+	// write out HTMLBlocks as an Enumeration
+	//
+	public Enumeration elements()
+	{ return new HTMLBlockListEnumerator(this);
+	}
+}
+
+final class HTMLBlockListEnumerator implements Enumeration
+{ private HTMLBlockList blocklist;
+	private int						member;
+
+	public HTMLBlockListEnumerator(HTMLBlockList blocklist)
+	{ this.blocklist	= blocklist;
+		this.member			= 0;
+	}
+
+	public Object nextElement()
+	{ Object element;
+
+		element = this.blocklist.blocks.elementAt(member);
+		member ++;
+		return element;
+	}
+
+	public boolean hasMoreElements()
+	{ if (this.blocklist == null || this.member == this.blocklist.blocks.size())
+		{ return false;
+		}
+		return true;
+	}
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLCText.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLCText.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLCText.java	(revision 14150)
@@ -0,0 +1,105 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLCText implements HTMLObject
+{ String text;
+  int    start;
+  int    end;
+
+  public static final String HTML_TEXT_TYPE = "Text";
+
+  public HTMLCText(String text, int start, int end)
+  { this.text  = text;
+    this.start = start;
+    this.end   = end;
+  }
+
+  public String getHTMLType()
+  { return HTML_TEXT_TYPE;
+  }
+
+  public int startPos()
+  { return this.start;
+  }
+
+  public int endPos()
+  { return this.end;
+  }
+
+  public boolean isEmpty()
+  { return emptyString(this.text);
+  }
+
+  public static boolean emptyString(String s)
+  { int c;
+
+    for (c = 0; c < s.length(); c ++)
+    { if (s.charAt(c) > 32)
+      { return false;
+      }
+    }
+    return true;
+  }
+
+  public static StringBuffer cleanString(StringBuffer buffer)
+  { int c;
+    int w;
+
+    w = 0;	
+    for (c = 0; c < buffer.length(); c ++)
+    { if (buffer.charAt(c) >= 32)
+      { if (buffer.charAt(c) == 32)
+	{ if (w > 0 && buffer.charAt(w-1) > 32)
+	  { buffer.setCharAt(w, buffer.charAt(c));
+	    w ++;
+	  }
+	}
+        else
+	{ if (c != w)
+	  { buffer.setCharAt(w, buffer.charAt(c));
+	  }
+	  w ++;
+	}
+      }
+      else
+      { if (w > 0 && buffer.charAt(w-1) > 32)
+	{ buffer.setCharAt(w,' ');
+	  w ++;
+	}
+      }
+    }
+    buffer.setLength(w);
+    return buffer;
+  }
+
+  /**
+   * nullString; used by initialisation for checking for
+   * non-blank content (ie. text other than just spaces/returns)
+   * see constructor (ie HTMLBlockList) methods below for role of
+   * this in the larger plan of things
+   */
+  public boolean nullString()
+  { int ch;
+
+    if (text != null)
+    { for (ch = 0; ch < text.length(); ch ++)
+      { if (text.charAt(ch) > ' ')
+	{ return false;
+	}
+      }
+    }
+    return true;
+  }
+
+  public String toString()
+  { return this.text;
+  }
+
+  public String toString(boolean cleaned)
+  { if (cleaned == false)
+    { return toString();
+    }
+    else
+    { return HTMLEntity.decodeText(this.text);
+    }
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLContext.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLContext.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLContext.java	(revision 14150)
@@ -0,0 +1,201 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLContext
+{ boolean valid;
+	StringBuffer	text;
+	
+	public static	HTMLTagVector breakTags = new HTMLTagVector("a,h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+
+	private int offset (String string, int direction)
+	{	int offset;
+
+		if (direction > 0)
+		{	offset = string.indexOf('.');
+		}
+		else
+		{	offset = string.lastIndexOf('.');
+		}
+		return offset;
+	}
+
+	public HTMLContext(HTMLBlock parent, HTMLBlock child, int scope)
+	{ int pos;
+		int start, startoffset;
+		int	end, endoffset;
+		int	length;
+		String	string;
+		HTMLTag	tag;
+
+		this.text	= null;
+
+		// if we can't match it, give up!
+		pos = parent.find(child);
+		if (pos < 0)
+		{ this.valid	= false;
+			return;
+		}
+
+		// get start!
+		if (pos > 0)
+		{ start = pos - 1;
+		}
+		else
+		{	start	= 0;
+		}	
+		startoffset	= -1;
+		length	= 0;
+		while (	start > 0)
+		{	
+		  /*
+		  if (parent.elementAt(start) instanceof String)
+			{	string = (String) parent.elementAt(start);
+
+				startoffset	= offset(string, -1);
+				if (startoffset > 0)
+				{	break;
+				}
+
+				length += string.length();
+				
+				if (length >= scope)
+				{	break;
+				}
+				}
+				else 
+		  */
+			if (parent.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.elementAt(start)).toString();
+
+				startoffset	= offset(string, -1);
+				if (startoffset > 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.elementAt(start) instanceof HTMLTag)
+			{	// do some checks
+				tag = (HTMLTag) parent.elementAt(start);
+
+				if (breakTags.matches(tag, false))
+				{	startoffset	= -1;
+					break;
+				}
+			}
+			start --;
+		}
+
+		// get end!
+		pos = pos + child.size();
+		if (pos < parent.size())
+		{ end	= pos + 1;
+		}
+		else
+		{	end	= parent.size();
+		}	
+		endoffset	= -1;
+		length	= 0;
+		string	= null;
+		while (end < parent.size())
+		{	if (parent.content.elementAt(start) instanceof String)
+			{	string = (String) parent.content.elementAt(start);
+
+				endoffset	= offset(string, +1);
+				if (endoffset >= 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(start)).toString();
+
+				endoffset	= offset(string, +1);
+				if (endoffset >= 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLTag)
+			{	tag = (HTMLTag) parent.elementAt(start);
+
+				if (breakTags.matches(tag, false))
+				{	endoffset	= -1;
+					end	= end - 1;
+					break;
+				}
+			}
+			System.out.println("String:"+string);
+			end ++;
+		}
+
+		this.text	= new StringBuffer();
+		if (startoffset >= 0)
+		{	string	= null;
+
+			if (parent.content.elementAt(start) instanceof String)
+			{	string = (String) parent.content.elementAt(start);
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(start)).toString();
+			}
+			
+			if (string != null)
+			{ this.text.append(string.substring(startoffset+1));
+			}
+			start ++;
+		}
+
+		for (pos = start; pos <= end; pos ++)
+		{ if (pos == end && endoffset >= 0)
+			{	break;
+			}
+
+			if (parent.content.elementAt(pos) instanceof String)
+			{	this.text.append((String) parent.content.elementAt(pos));
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLCText)
+			{	this.text.append(((HTMLCText) parent.content.elementAt(pos)).toString());
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLTag)
+			{ // don't add it!
+			}
+		}
+
+		if (pos == end && endoffset > 0)	// this deliberately ignore endoffset == 0
+																		 	// 'cos nothing should happen anyhow!
+		{	string	= null;
+
+			if (parent.content.elementAt(pos) instanceof String)
+			{	string = (String) parent.content.elementAt(pos);
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(pos)).toString();
+			}
+		
+			if (string != null)
+			{	this.text.append(string.substring(0, endoffset));
+			}
+		}
+	}
+
+	public String toString()
+	{	if (this.text == null)
+		{ return "";
+		}
+		return this.text.toString();
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java	(revision 14150)
@@ -0,0 +1,658 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.*;
+import java.io.*;
+import java.applet.*;
+import java.util.*;
+
+/** 
+ *  HTMLDOC: handles an individual HTML document
+ */
+
+public class HTMLDoc
+{ 
+    String        content;
+    String        title;
+    URL           url;
+    int           pos;
+    int           status;
+    HTMLBlockList blocklist;
+    List          urls_out;
+    List          urls_in;
+    Object        note;
+    HTMLLoader    loader;
+    HTMLBlock     coded;
+
+    /**
+     * Constructor which loads a given document; the ref indicates from where!
+     */
+    public HTMLDoc(URL url, int ref)
+    { 
+	URLConnection connect;
+	int length, i, c;
+	byte inbuffer[];
+	InputStream in;
+
+	this.url = url;
+	this.pos = 0;
+	this.blocklist = null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+	this.note = null;
+	this.status = 0;
+
+	if (ref >= 0){
+	    int attempts;
+	    
+	    attempts = 0;
+	    loader = new HTMLLoader(url);
+	    loader.load();
+	    if (ref == 0){
+		while(this.ready() == false){
+		    try {
+			Thread.sleep(100);
+		    }
+		    catch (InterruptedException ex){
+			
+		    }
+		}
+	    }
+	}
+	else {
+	    this.content = "";
+	}
+    }
+    
+    public HTMLDoc(URL url)
+    {	
+	this(url, 0);
+    }
+    
+    /**
+     * @return <code>true</code> when a document has finished loading.
+     */
+    public synchronized boolean ready()
+    {
+	if (this.loader == null){
+	    return true;
+	}
+	
+	boolean loaded 	= loader.loaded();
+	
+	if (loaded == true){
+	    // reload if necessary
+	    if (this.loader.error == true){
+		if (this.status < 3 && this.loader.notFound == false){
+		    this.status ++;
+		    this.loader = new HTMLLoader(this.url);
+		    loader.load();
+		    return false;
+		}
+		else {
+		    this.status 	= 4;
+		    this.content	= null;
+		    this.loader		= null;
+		    return true;
+		}
+	    }
+	    this.content 	= loader.HTML();
+	    this.loader		= null;
+	}
+	return loaded;
+    }
+    
+    /**
+     * Create a document from a file
+     */
+    public HTMLDoc(URL url, String file)
+    { 
+	FileInputStream _in;
+	byte[] data = new byte[128];
+	int databytes;
+	StringBuffer content;
+	
+	try {
+	    _in = new FileInputStream(file);
+	    if (_in != null){
+		content = new StringBuffer();
+		do{  
+		    databytes = _in.read(data);
+		    if (databytes > 0){
+			content.append(new String(data, 0, databytes));
+		    }		
+		} while (databytes >= 0);
+		this.content		= content.toString();
+	    }
+	}
+	catch (IOException ex){
+	    this.content = null;
+	    System.out.println(ex);
+	}
+	this.url = url;
+	this.pos = 0;
+	this.status = 0;
+	this.note = null;
+	this.blocklist	= null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+    }
+
+    /**
+     * Create a url-only document: the document body is not stored
+     */
+    public HTMLDoc(String url)
+    {
+	if (url != null){
+	    try{
+		this.url = new URL(url);
+	    }
+	    catch (MalformedURLException ex){
+		this.url = null;
+	    }
+	}
+	else {
+	    this.url = null;
+	}
+	this.pos = 0;
+	this.status = 0;
+	this.note = null;
+	this.content = null;
+	this.blocklist = null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+    }
+    
+    static public HTMLDoc fromString(String url, String string)
+    {	
+	HTMLDoc reply;
+	
+	reply = new HTMLDoc(url);
+	reply.content = string;
+	return reply;
+    }
+    
+    static public boolean validExtension(String url)
+    {	
+	String extension;
+	
+	// get file extension
+	extension = URLString.extension(url);
+	if (extension != null){
+	    extension = extension.toLowerCase();
+	}
+	
+	if (extension == null ||
+	    extension.indexOf("htm") >= 0 || 
+	    extension.indexOf("asp") >= 0){
+	    return true;
+	}
+	return false;
+    }
+    
+    static boolean validExtension(URL url)
+    {	
+	return validExtension(url.toString());
+    }
+
+    /**
+     * Uncache a single document
+     */
+    public static HTMLDoc uncacheDoc(String filebase, String url)
+    {	
+	String 	pathname;
+	String	filename;
+	HTMLDoc	doc;
+	
+	pathname = URLString.pathName(url);
+	if (pathname == null){
+	    return null;
+	}
+	
+	filename = filebase + pathname.replace('/', File.pathSeparatorChar);
+	doc = new HTMLDoc(URLString.toURL(url), filename);
+	
+	return doc;
+    }
+
+    /**
+     * @return URL (ie. Java URL object) of the document
+     */
+    public URL getUrl()
+    { 
+	return this.url;
+    }
+
+    // Report url to AppletContext
+    public void show(AppletContext ac)
+    { 
+	URL url;
+
+	url = this.getUrl();
+	if (url != null){
+	    ac.showDocument(url);
+	}
+    }
+    
+    // Get a HTMLBlock list of the wanted text in an HTMLdoc
+    public void getSelectContent(int level)
+    { 
+	HTMLBlockList list;
+
+	list = new HTMLBlockList(this, level);
+	this.blocklist = list;
+    }
+    
+    /**
+     * return the selected content
+     */
+    public HTMLBlockList selectContent()
+    { 
+	return this.blocklist;
+    }
+
+    /**
+     * get header blocks as selected content
+     */
+    public HTMLBlockList headerBlocks()
+    {	
+	if (this.blocklist == null){
+	    this.getSelectContent(6);
+	}
+	return this.blocklist;
+    }
+    
+    /**
+     * return a tagged-up select with an <A NAME=> at the beginning of each Hx
+     */
+    public String taggedSelectContent()
+    {
+	int heading, offset, pos;
+	String reply;
+	
+	if (this.blocklist == null || 
+	    this.blocklist.size() == 0){
+	    return this.content;
+	}
+	
+	reply = "";
+	offset	= 0;
+	for (heading = 0; heading < this.blocklist.size(); heading ++){
+	    reply = reply + 
+		this.content.substring(offset, this.blocklist.tagPos(heading)) +
+		"<A NAME=\"_L"+heading+"\"></A>";
+	    offset = this.blocklist.tagPos(heading);
+	}
+	reply = reply + this.content.substring(offset, this.content.length());
+	return reply;
+    }
+    
+    /**
+     * Return enumeration of headings in document
+     */
+    public Enumeration headings()
+    {
+	if (this.blocklist == null){
+	    this.getSelectContent(6);
+	}
+	return this.blocklist.elements();
+    }
+    
+    /**
+     *	@return HTMLBlock of the child text underneath a heading
+     */
+    public HTMLBlock headingChildText(int headref)
+    { 
+	HTMLBlock pageblock;
+	HTMLBlock headblock;
+	HTMLBlock childblock;
+	int blockstart, blockend;
+	
+	pageblock = this.getCodedContent(); // get all the coded bits of the page
+	headblock = this.headerBlocks().tagBlock(headref);
+	
+	blockstart = pageblock.find(headblock);
+	blockstart += headblock.size();
+	if (headref == this.headerBlocks().size() - 1) {
+	    blockend = pageblock.size();
+	}
+	else {
+	    blockend = pageblock.find(this.headerBlocks().tagBlock(headref+1));
+	}
+	childblock = pageblock.subBlock(blockstart, blockend);
+	
+	return childblock;
+    }
+    
+    /**
+     * Add incoming url to list of incoming references
+     */
+    public void addUrlIn(String url)
+    {
+	if (this.urls_in == null){
+	    this.urls_in = new ArrayList();
+	}
+	this.urls_in.add(url);
+    }
+    
+    /**
+     * @return a String of all incoming urls
+     */
+    public String urlsInString()
+    { 
+	if (this.urls_in != null){
+	    return this.urls_in.toString();
+	}
+	return "";
+    }
+    
+    /**
+     * @return an enumeration of all incoming urls as Strings
+     */
+    public Iterator urlsIn()
+    {
+	if (this.urls_in == null){
+	    return null;
+	}
+	return this.urls_in.iterator();
+    }
+    
+    /**
+     * @return the <code>StringVector</code> of all incoming urls
+     */
+    public List urlsInList()
+    {	
+	return this.urls_in;
+    }
+
+    /**
+     * @return <code>true</code> if the given url is linked to by this
+     *  document
+     */
+    public boolean linkedTo(URL url)
+    {	
+	if (this.urls_out != null &&
+	    this.urls_out.indexOf(url.toString()) >= 0){
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     * See above: returns <code>true</code> if the document is linked to
+     * by this document.
+     */
+    public boolean linkedTo(HTMLDoc doc)
+    {
+	return linkedTo(doc.getUrl());
+    }
+
+    /**
+     * Add outgoing url to list of outgoing references
+     */
+    public void addUrlOut(String url)
+    { 
+	if (this.urls_out == null){
+	    this.urls_out = new ArrayList();
+	}
+	this.urls_out.add(url);
+    }
+    
+    /**
+     * @return a String of all outgoing urls
+     */
+    public String urlsOutString()
+    { 
+	if (this.urls_out != null){
+	    return this.urls_out.toString();
+	}
+	return "";
+    }
+    
+    /**
+     * @return the <code>StringVector</code> of all outgoing urls
+     */
+    public List urlsOutList()
+    {	
+	return this.urls_out;
+    }
+
+    /**
+     * Return an enumeration of outbound urls
+     */
+    public Iterator urlsOut()
+    { 
+	if (this.urls_out == null){
+	    return null;
+	}
+	return this.urls_out.iterator();
+    }
+
+    private static int urlListPos(List urlList, String url)
+    {	
+	for (int i = 0; i < urlList.size(); i ++){
+	    if (((String) urlList.get(i)).equals(url)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    // Rename internally the content of a list
+    private void __renameUrl(List urlList, String oldUrl, String newUrl)
+    {	
+	if (urlList != null){
+	    int oldPos = urlListPos(urlList, oldUrl);
+	    
+	    if (oldPos >= 0){
+		if (urlListPos(urlList, newUrl) >= 0){
+		    // delete oldUrl: now duplicate
+		    urlList.remove(oldPos);
+		}
+		else {
+		    // rename oldUrl
+		    urlList.set(oldPos, newUrl);
+		}
+	    }
+	}
+    }
+    
+    /**
+     * Rename inbound/outbound url references
+     */
+    public void renameUrl(String oldUrl, String newUrl)
+    {	
+	this.__renameUrl(this.urls_in, oldUrl, newUrl);
+	this.__renameUrl(this.urls_out, oldUrl, newUrl);
+    }
+    
+    /**
+     * Save document
+     */
+    public boolean save(String filename, boolean select)
+    {	
+	FileOutputStream fout;
+	PrintWriter pout;
+	
+	String flattened = URLString.flattenedUrl(filename);
+	
+	try{	
+	    HTMLDocAnchorList anchorList = new HTMLDocAnchorList(this);
+
+	    anchorList.flattenLinks();
+			
+	    fout = new FileOutputStream(flattened);
+	    pout = new PrintWriter(fout);
+	    if (select == true) {
+		pout.print(this.taggedSelectContent());
+	    }
+	    else {
+		pout.print(this.content);
+	    }
+	    pout.close();
+	    fout.close();
+	}
+	catch (IOException ex) {
+	    System.out.println(ex);
+	    return false;
+	}
+	return true;
+    }
+    
+    /**
+     * Save document content
+     */
+    public boolean save(String base)
+    {
+	return this.save(base, false);
+    }
+    
+    /**
+     * Save selected content
+     */
+    public boolean saveSelect(String base)
+    {	
+	return this.save(base, true);
+    }
+
+    /**
+     * Get title of page
+     */
+    public void getTitle()
+    { 
+	HTMLDocTitle doctitle;
+
+	if (this.content != null){
+	    doctitle = new HTMLDocTitle(this);
+	    this.title	= doctitle.Title();
+	}
+    }
+    
+    /**
+     * @return: title of page
+     */
+    public String titleString()
+    {
+	if (this.title == null){
+	    this.getTitle();
+	}
+	return this.title;
+    }
+    
+    /**
+     * @return a <code>String</code> of the url of the page
+     */
+    public String urlString()
+    {
+	return this.url.toString();
+    }
+
+    /**
+     * 	Replace the HTML text of the page with the provided HTML string
+     *	content.  This is an extremely dangerous call to use, as dependent
+     *	objects such as tag objects will be immediately rendered inaccurate
+     *	in terms of pointers held to the original text.
+     */
+    protected void setContent(String newContent)
+    { 
+	this.content = newContent;
+    }
+
+    /**
+     * @return a <code>String</code> of the HTML text of the page
+     */
+    public String getContent()
+    { 
+	return this.content;
+    }
+
+    /**
+     * @return a <code>HTMLBlock</code> of the page content
+     */
+    public HTMLBlock getCodedContent()
+    {	
+	// if we haven't already got it, code the bugger up right now!
+	if (this.coded == null){
+	    HTMLParser parser;
+	    String next;
+	    HTMLBlock coded;
+	    HTMLTag tag;
+	    HTMLCText text;
+
+	    if (this.content == null){
+		return null;
+	    }
+	    
+	    parser = new HTMLParser(this);
+	    coded = new HTMLBlock(0, this.content.length());
+
+	    parser.startParse();
+	    next = parser.fullParse();
+	    while (next != null){
+		if (HTMLTag.isTag(next)){
+		    tag = new HTMLTag(next, parser.lastParse(), parser.atParse());
+		    coded.addTag(tag);
+		}
+		else {
+		     text = new HTMLCText(next, parser.lastParse(), parser.atParse());
+		     coded.addText(text);
+		}
+		next = parser.fullParse();
+	    }
+	    this.coded	= coded;
+	}
+	return this.coded;
+    }
+    
+    /**
+     * @return <code>true</code> if this document is not errored from loading
+     * etc.
+     */
+    public boolean valid()
+    {	
+	return (this.status < 4 && this.content != null && this.content.equals("null") == false);
+    }
+    
+    /**
+     * HTMLString: returns content
+     */
+    public String HTMLString()
+    { 
+	return this.content;
+    }
+    
+    /**
+     * Return text of document as ascii text;
+     */
+    /*
+      public String toString()
+      {	StringBuffer	reply = new StringBuffer();
+      int						parastart = 0;
+
+      return reply.toString();
+      }
+    */
+
+    /**
+     * Set current note on this page
+     * @return old note object
+     */
+    public Object setNote(Object note)
+    {
+	Object oldnote;
+	
+	oldnote	= this.note;
+	this.note = note;
+	return oldnote;
+    }
+    
+    /**
+     * @return current note <code>Object</code> on this page
+     */
+    public Object getNote()
+    {
+	return this.note;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java	(revision 14150)
@@ -0,0 +1,160 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+public class HTMLDocAnchorList extends HTMLBlockList
+
+{ HTMLDoc doc;
+
+	public HTMLDocAnchorList(HTMLDoc doc)
+
+	{ super();
+
+		this.doc = doc;	// remember the document reference
+
+
+
+		super.initialise(doc, anchorStartTags, anchorEndTags, anchorOptEndTags);
+
+	}
+
+
+
+	/**
+	 *	Return the HTMLBlock containing the named local anchor and it's
+	 *	child (contained) tags.
+	 *
+	 *	@param <code>String</code> name of the local anchor to find.
+	 *	@return <code>HTMLBlock</code> the corresponding block of HTML
+	 *					contained by the anchor.  If the local anchor does not
+	 *					exist, this value will be <code>null</code>.
+	 */
+	public HTMLBlock localAnchor(String name)
+
+	{	int block;
+
+		String	localname;
+
+
+
+		for (block = 0; block < this.size(); block ++)
+
+		{ localname = this.tagBlock(block).headTag().idValue("name");
+
+			if (localname == null)
+
+			{ continue;
+
+			}
+
+			if (name.equalsIgnoreCase(localname))
+
+			{	return this.tagBlock(block);
+
+			}
+
+		}
+
+		return null;
+
+	}
+
+	private int indexOf(String base, String subString, int from)
+	{	int i, j;
+
+		for (i = from; i < base.length() - subString.length(); i ++)
+		{	if (Character.toLowerCase(base.charAt(i)) == subString.charAt(0))
+			{	j = 0;
+				for (j = 1; j < subString.length(); j ++)
+				{	if (Character.toLowerCase(base.charAt(i+j)) != subString.charAt(j))
+					{	break;
+					}
+				}
+
+				if (j == subString.length())
+				{	return i;
+				}
+			}
+		}
+		return -1;
+	}
+
+	public void flattenLinks()
+	{	int 					block;
+		String				url;
+		char					endsWith;
+		StringBuffer	newContent = new StringBuffer(doc.getContent());
+		boolean				changed = false;
+
+//		System.out.println("Flattening");
+
+		for (block = 0; block < this.size(); block ++)
+		{ HTMLTag tag;
+			int			tagPos;
+			int			valueStart, valueEnd;
+
+			tag = this.tagBlock(block).headTag();	// get the <A> tag itself
+
+			url = tag.idValue("href"); 						// if there is no HREF member, abandon this one
+			if (url == null)
+			{ System.out.println("No href");
+				continue;
+			}
+	
+			tagPos	= tag.startPos();
+			tagPos	=	indexOf(doc.getContent(), "href", tagPos);	// get the HREF itself
+			if (tagPos > tag.endPos() || tagPos < 0)					// if none in the tag, skip it
+			{	System.out.println("Didn't get href");
+				continue;
+			}
+			tagPos += "href".length();												// skip past it
+
+			while (	doc.getContent().charAt(tagPos) <= ' ' ||		// skip pass equals, etc.
+							doc.getContent().charAt(tagPos) == '=')
+			{	tagPos ++;
+			}
+
+			if (doc.getContent().charAt(tagPos) == '"' ||				// get the demarcating whitespace/quotes
+					doc.getContent().charAt(tagPos) == '\'')
+			{	endsWith	= doc.getContent().charAt(tagPos);
+				tagPos ++;
+			}
+			else
+			{	endsWith  = ' ';
+			}
+			valueStart = tagPos;
+
+			//			System.out.println("Flattening " + doc.getContent().length() + " " + tagPos);
+
+			while (tagPos < doc.getContent().length() &&
+			       doc.getContent().charAt(tagPos) != endsWith)	// read it in
+			{	if (doc.content.charAt(tagPos) == '?' ||				// replace special characters			
+						doc.content.charAt(tagPos) == '&')
+				{	newContent.setCharAt(tagPos, '_');						//   and note that we've done so
+					changed	= true;
+				}
+				tagPos ++;
+			}
+			valueEnd = tagPos;
+
+			tagPos = indexOf(doc.getContent(), ".html", valueStart);
+			if (tagPos >= 0 && tagPos < valueEnd)
+			{	// shuffle around the html bit
+				while (tagPos < valueEnd - 5)
+				{	newContent.setCharAt(tagPos, newContent.charAt(tagPos + 5));
+					tagPos ++;
+				}
+				newContent.setCharAt(tagPos ++, '.');
+				newContent.setCharAt(tagPos ++, 'h');
+				newContent.setCharAt(tagPos ++, 't');
+				newContent.setCharAt(tagPos ++, 'm');
+				newContent.setCharAt(tagPos ++, 'l');
+			}
+		}
+
+		if (changed == true)																// replace the document as needsbe
+		{	doc.setContent(newContent.toString());
+		}
+	}
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java	(revision 14150)
@@ -0,0 +1,25 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+public class HTMLDocHeadingList extends HTMLBlockList
+
+{ public HTMLDocHeadingList(HTMLDoc doc)
+
+	{ super();
+
+
+
+		super.initialise(doc, headingStartTags, headingEndTags, anchorOptEndTags);
+
+	}
+
+
+
+	public int headingLevel(int element)
+
+	{	return this.tagBlock(element).headTag().tagLevel();
+
+	}
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java	(revision 14150)
@@ -0,0 +1,187 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+import java.util.*;
+
+
+
+public class HTMLDocLocalHeading
+
+{	String		heading;
+
+	HTMLBlock	block;
+
+
+
+	public HTMLDocLocalHeading(HTMLDoc doc, String local)
+
+	{	HTMLBlock 				nextheading, lastheading, anchor;
+
+		Enumeration				headings;
+
+		HTMLDocAnchorList	anchors;
+
+		HTMLBlock					docBlock;
+
+		int								_start, _end, _pos;
+
+
+
+		// initialise (to undefined)
+
+		this.heading	= null;
+
+		this.block		= null;
+
+
+
+		// get anchors
+
+		anchors	= new HTMLDocAnchorList(doc);
+
+		anchor	= anchors.localAnchor(local);
+
+		if (anchor == null)
+
+		{	return;
+
+		}
+
+
+
+		// get headings position; lastheading will be previous, nextheading will be next
+
+		headings		= doc.headings();
+
+		lastheading	= null;
+
+		nextheading	= null;
+
+
+
+		while (headings.hasMoreElements())
+
+		{	nextheading	= (HTMLBlock) headings.nextElement();
+
+			if (nextheading.startPos() > anchor.startPos())
+
+			{	break;
+
+			}
+
+			
+
+
+
+			lastheading	= nextheading;
+
+		}
+
+
+
+		// didn't get any actual headings; abort.
+
+		if (nextheading == null)
+
+		{	return;
+
+		}
+
+
+
+		// got to the end
+
+		if (nextheading == lastheading)
+
+		{	this.block = nextheading;
+
+		}
+
+		else
+
+		{	// now see where anchor is; try to find null textual content between anchor
+
+			// and heading
+
+			docBlock	= doc.getCodedContent();
+
+			_start		= docBlock.find(anchor);
+
+			_end			= docBlock.find(nextheading);
+
+			for (_pos = _start; _pos < _end; _pos ++)
+
+			{	// HTML tag
+
+				if (docBlock.elementAt(_pos) instanceof HTMLTag)
+
+				{	
+
+				}
+
+				// HTML text - if non-blank, abort
+
+				else
+
+				{	if (((HTMLCText) docBlock.elementAt(_pos)).nullString() == false)
+
+					{	break;
+
+					}
+
+				}
+
+			}
+
+
+
+			// assign
+
+			if (_pos != _end)
+
+			{	this.block	=	lastheading;
+
+			}
+
+			else
+
+			{	this.block	= nextheading;
+
+			}
+
+		}
+
+		this.heading	= this.block.toString();
+
+	}
+
+
+
+	public String toString()
+
+	{	return this.heading;
+
+	}
+
+
+
+	public HTMLBlock block()
+
+	{	return this.block;
+
+	}
+
+
+
+	protected void finalize() throws Throwable
+
+	{	this.heading	= null;
+
+		this.block		= null;
+
+		super.finalize();
+
+	}
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java	(revision 14150)
@@ -0,0 +1,89 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLDocTitle
+{ StringBuffer title;
+	private static HTMLTagVector titleStartTags;
+	private static HTMLTagVector titleEndTags;
+	private static HTMLTagVector titleOptEndTags;
+
+	static
+	{	titleStartTags 	= new HTMLTagVector("title,h1,h2,h3,h4,h5,h6");
+		titleEndTags 		= new HTMLTagVector("title,h1,h2,h3,h4,h5,h6,body,head,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		titleOptEndTags	= new HTMLTagVector("div,center,form");
+	}
+
+	/**
+	 * Constructor: generates a title for the supplied <code>HTMLDoc</code>.
+	 */
+	public HTMLDocTitle(HTMLDoc doc)
+	{ boolean				intitle;
+		boolean				incontent;
+		HTMLCText			text;
+		HTMLTag				tag;
+		HTMLBlock			parser;
+		Enumeration		elements;
+		Object				element;
+		
+		this.title 	= null;
+		intitle			= false;
+		incontent		= false;
+		
+		parser		=	doc.getCodedContent();
+		if (parser == null)
+		{	return;
+		}
+		elements	= parser.elements();
+
+		while (elements.hasMoreElements())
+		{ //System.out.println(read + ":" +  read.charAt(0));
+			element	= elements.nextElement();
+			
+			if (element instanceof HTMLTag)
+			{ tag = (HTMLTag) element;
+
+				if (intitle)
+				{	if (titleEndTags.matches(tag, false) ||
+							(incontent && titleOptEndTags.matches(tag, false))) // if closing
+					{ intitle = false;
+						break;
+					}
+					else // else ignore it
+					{
+					}
+				}
+				else
+				{	if (titleStartTags.matches(tag, true) == true)
+					{ intitle			= true;
+						incontent		= false;
+						this.title	= new StringBuffer();
+					}
+				}
+			}
+			else
+			{ text	= (HTMLCText) element;
+
+				if (intitle == true)
+				{	this.title.append(text.toString());
+					if (!text.nullString())
+					{ incontent	= true;
+					}
+				}
+			}
+		}
+		if (this.title != null)
+		{ this.title = HTMLCText.cleanString(this.title);
+		}
+	}
+
+	/**
+	 * @return the Title as a <code>String</code>.
+	 */
+	public String Title()
+	{	if (this.title == null)
+		{ return null;
+		}
+		return this.title.toString();
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java	(revision 14150)
@@ -0,0 +1,346 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.*;
+import java.util.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+
+public class HTMLEntity
+{ String  name;
+  int     code;
+  static Vector entities = null;
+  static Vector	nmapping = null;
+
+  public HTMLEntity(String name, int code)
+  { int start, end, at;
+    HTMLEntity compareEntity;
+
+    if (HTMLEntity.entities == null)
+    { HTMLEntity.entities = new Vector();
+    }
+    if (HTMLEntity.nmapping == null)
+    { HTMLEntity.nmapping = new Vector();
+    }
+
+    this.name	= name;
+    this.code	= code;
+
+    entities.addElement(this);
+    start = 0;
+    end   = nmapping.size();
+    while (start != end)
+    { at = (start + end) / 2;
+	
+      compareEntity = 
+	(HTMLEntity) entities.elementAt(((Integer) nmapping.elementAt(at)).intValue());
+      if (compareEntity.code > code)
+      { end = at;
+      }
+      else
+      { start = at + 1;
+      }
+    }
+    if (end == nmapping.size())
+    { nmapping.addElement(new Integer(nmapping.size()));
+    }
+    else
+    { nmapping.insertElementAt(new Integer(nmapping.size()), start);
+    }
+  }
+
+  public String toString()
+  { Character c;
+
+    c = new Character((char) this.code);
+    return c.toString();
+  }
+
+  public static String codeString(char c)
+  { StringBuffer 	reply;
+    String        codestring;
+
+    if (c < 128 || c == '&' || c == '<' || c == '>')
+    { return null;
+    }
+    codestring = Integer.toString((int) c);
+    reply      = new StringBuffer("&#");
+    reply.append(codestring);
+    reply.append(";");
+
+    return reply.toString();
+  }
+
+  public static String nameCodeString(char c)
+  { int start, end, at, compare;
+    HTMLEntity compareEntity;
+
+    start = 0;
+    end		= nmapping.size();
+    while (start != end)
+    { at	= (start + end) / 2;
+      
+      compareEntity = 
+	(HTMLEntity) entities.elementAt(((Integer) nmapping.elementAt(at)).intValue());
+      if (compareEntity.code == (int) c)
+      { StringBuffer reply;
+
+        reply = new StringBuffer("&");
+	reply.append(compareEntity.name);
+	reply.append(";");
+	return reply.toString();
+      }
+      else if (compareEntity.code < c)
+      {	end	= at;
+      }
+      else
+      {	start	= at + 1;
+      }
+    }
+    return codeString(c);
+  }
+
+  public static void getEntities(String source)
+  { HTMLEntity 	newentity;
+    int         pos;
+    int					start;
+    String			name;
+    int					code;
+    
+    if (source == null)
+    { return;
+    }
+
+    start = 0;
+    while (start < source.length())
+    { pos = start;
+      while (source.charAt(pos) != ':')
+      { pos ++;
+      }
+      name = source.substring(start, pos);
+
+      // skip colon and reset
+      pos ++;
+      start = pos;
+
+      while(pos < source.length() && source.charAt(pos) > ' ')
+      { pos ++;
+      }
+      code = Integer.parseInt(source.substring(start, pos));
+
+      newentity = new HTMLEntity(name, code);
+      
+      while ( pos < source.length() && source.charAt(pos) < ' ')
+      { pos ++;
+      }
+      start = pos;
+    }
+  }
+
+  public static void getEntities(InputStream in)
+  { getEntities(DocumentLoader.getAsString(in));
+  }
+
+  public static void getEntities(File file)
+  { FileInputStream in;
+
+    try
+    { in = new FileInputStream(file);
+      getEntities(in);
+      in.close();
+    }
+    catch (IOException io)
+    { System.out.println("No entities");
+    }
+  }
+
+  public static void getEntities()
+  { ClassLoader loader;
+    InputStream	in;
+
+    in = ClassLoader.getSystemResourceAsStream("org\\greenstone\\gsdl3\\gs3build\\util\\Entities.map");
+    if (in == null)
+    { System.out.println("Unable to load Entities.map from org.greenstone.gsdl3.gs3build.util");
+    }
+    getEntities(in);
+  }
+
+  public static String encodeText(String source, boolean named)
+  { StringBuffer reply;
+    int          start,at;
+
+    if (source == null)
+    { return null;
+    }
+
+    if (HTMLEntity.entities == null)
+    {	getEntities();
+    }
+
+    reply	= new StringBuffer();
+    at 		= 0;
+    start	= 0;
+    while (at < source.length())
+    { if (source.charAt(at) > 128)
+      {	if (at != start)
+	{ reply.append(source.substring(start, at));
+	}
+
+        if (named)
+	{ reply.append(nameCodeString(source.charAt(at)));
+	}
+	else
+	{ reply.append(codeString(source.charAt(at)));
+	}
+	start	= at + 1;
+      }
+      at ++;
+    }
+		
+    return reply.toString();
+  }
+
+  public static String decodeText(String source)
+  { int start, end, loff, roff;
+    int sstart, send, test;
+    int startoff, endoff, baseoff;
+    boolean matched;
+    String  tail;
+    StringBuffer reply;
+
+    if (source == null)
+    { return null;
+    }
+
+    if (HTMLEntity.entities == null)
+    {	getEntities();
+    }
+
+    reply   = new StringBuffer();
+    baseoff = 0;
+
+    tail = source;
+    do
+    { // get next ampersand
+	    startoff = tail.indexOf('&');
+      if (startoff < 0 || startoff == tail.length() - 1)
+      { break;
+      }
+
+      // if it's followed by a hash, evaluate as a number
+      if (tail.charAt(startoff + 1) == '#')
+      { if (startoff < tail.length() - 2 &&
+      			tail.charAt(startoff+2) >= '0' &&
+            tail.charAt(startoff+2) <= '9')
+      	{ int code;
+
+        	loff = startoff + 2;
+        	while (tail.charAt(loff) >= '0' &&
+          				tail.charAt(loff) <= '9')
+        	{ loff ++;
+          }
+
+          code = Integer.parseInt(tail.substring(startoff+2, loff));
+        	reply.append(tail.substring(0, startoff));
+					reply.append(new Character((char) code));
+
+          if (tail.charAt(loff) == ';')
+          { loff ++;
+          }
+          loff = loff - startoff - 1;
+        }
+        else
+        { loff = 0;
+        	reply.append(tail.substring(0, startoff+1));
+        }
+      }
+      else
+      {	start = 0;
+        end   = HTMLEntity.entities.size();
+        loff  = 0;
+
+	while (startoff+1+loff < tail.length() && start != end)
+	{ sstart = start;
+	  send   = end;
+          test	 = (sstart + send) >> 1;
+
+	  matched	= false;
+		      
+	  roff	= startoff + 1 + loff;
+
+	  while (sstart != send)
+	  { test = (sstart + send) >> 1;
+
+	    if (tail.charAt(roff) < ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.charAt(loff))
+	    { send = test;
+	    }
+	    else if (tail.charAt(roff) > ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.charAt(loff))
+	    { sstart	= test + 1;
+	    }
+	    else
+	    { break;
+	    }
+	  }
+
+	  //					System.out.println(sstart+">"+send+":"+loff);
+
+	  if (sstart != send) // found a match
+	  { sstart = test;
+	    while (sstart >= start &&
+		   tail.charAt(roff) ==
+		   ((HTMLEntity) HTMLEntity.entities.elementAt(sstart)).name.charAt(loff))
+	    { sstart --;
+	    }
+	    sstart ++;
+
+	    send = test + 1;
+	    while (send < end &&
+		   tail.charAt(roff) ==
+		   ((HTMLEntity) HTMLEntity.entities.elementAt(send)).name.charAt(loff))
+	    { send ++;
+	    }
+
+//						System.out.println(sstart+"!"+send+":"+loff);
+
+	    start = sstart;
+	    end	= send;
+	  }
+	  else
+	  { loff	= 0;
+	    reply.append(tail.substring(0, startoff+1));
+	    break;
+	  }
+
+//					System.out.println(start+">"+end+":"+loff);
+	  
+	  loff ++;
+	  if (end == start + 1 &&
+	      ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.length() == loff)
+	  { reply.append(tail.substring(0, startoff));
+	    reply.append(((HTMLEntity) HTMLEntity.entities.elementAt(test)).toString());
+	    if (roff < tail.length() - 1 && tail.charAt(roff+1) == ';')
+	    { loff ++;
+	    }
+	    break;
+	  }
+	}
+      }
+
+      tail = tail.substring(startoff + 1 + loff);
+    } while (tail != null && tail.length() > 0);
+    
+    if (tail != null)
+    { reply.append(tail);
+    }
+    return reply.toString();
+  }
+
+  public static void main(String args[])
+  { String reply;
+
+    getEntities();
+    
+    reply = decodeText(args[0]);
+    System.out.println(reply);
+  }
+}
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java	(revision 14150)
@@ -0,0 +1,208 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.*;
+import java.util.*;
+
+class HTMLLoadManager implements Runnable
+{	Vector		 	toLoad;
+	Vector			loading;
+	Vector			loaded;
+	boolean			stopFlag;
+	boolean			started;
+	Thread			thread;
+	static int	running;
+
+	static
+	{	running	= 0;
+	}
+
+	/**
+	 *	Vanilla constructor of an object to load HTML documents in the background
+	 */
+	public HTMLLoadManager()
+	{	this.toLoad		= new Vector();
+		this.loading 	= new Vector();
+		this.loaded		= new Vector();
+		this.thread		= null;
+		this.stopFlag	= false;
+		this.started	= false;
+	}
+
+	/**
+	 *	Check if a given document is loading
+	 *
+	 *	@return <code>boolean</code> whether the document is loading or not
+	 */
+	public boolean isLoading(String docURL)
+	{	int i;
+
+		for (i = 0; i < this.toLoad.size(); i ++)
+		{	if (this.toLoad.elementAt(i).toString().equals(docURL))
+			{	return true;
+			}
+		}
+
+		for (i = 0; i < this.loading.size(); i ++)
+		{	if (((HTMLDoc) this.loading.elementAt(i)).urlString().equals(docURL))
+			{	return true;
+			}
+		}
+
+		for (i = 0; i < this.loaded.size(); i ++)
+		{	if (((HTMLDoc) this.loaded.elementAt(i)).urlString().equals(docURL))
+			{	return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 *	Start this object as a thread
+	 */
+	public synchronized void start()
+	{	if (this.thread == null)
+		{	this.thread	= new Thread(this);
+			this.thread.start();		// start the thread
+			this.stopFlag	= false;	// reset the stop flag as it could have been reset
+			this.started 	= true;		// note that we're running
+			running ++;
+		}
+	} 
+
+	/**
+	 *	The main thread function; it calls a number of private service functions to
+	 *  achieve it's affect.  Once the stop flag is set, it runs until all loading
+	 *  documents are loaded, then stops.
+	 */
+	public void run()
+	{ int i;
+
+		while (this.stopFlag == false || this.loading.size() > 0)
+		{	// slumber while there's nothing to do
+			while (this.toLoad.size() == 0 && this.loading.size() == 0)
+			{	try
+				{
+					thread.sleep(100);
+				}
+				catch (InterruptedException ex)
+				{
+				}
+			}
+
+			// first check for finished documents
+			i = 0;
+			while (i < this.loading.size())
+			{	HTMLDoc doc = (HTMLDoc) this.loading.elementAt(i);
+
+				if (doc.ready())
+				{	this.deschedule(i);
+				}
+				else
+				{	i ++;
+				}
+			}
+
+			// then check for a document to load
+			while (this.stopFlag == false &&
+						 this.toLoad.size() > 0 &&
+						 this.loading.size() < 10)
+			{	this.scheduleNext();
+			}
+		}
+		this.started = false;
+		running --;
+	}
+
+	/**
+	 *	Returns the total number of loader managers running at this time
+	 *
+	 *	@return <code>int</code> the number of managers running.
+	 */
+	public synchronized int running()
+	{	return running;
+	}
+
+	private synchronized void scheduleNext()
+	{	// pop url from list to load
+		URL docURL	= (URL) this.toLoad.elementAt(0);
+		this.toLoad.removeElementAt(0);
+
+		// generate new document and add it to the "loading" vector;
+		// the document content is loaded in another thread
+		HTMLDoc	doc = new HTMLDoc(docURL, 1);
+		this.loading.addElement(doc);
+	}
+
+	private synchronized void deschedule(int i)
+	{	// get the document and place it in the loaded list
+		HTMLDoc	doc = (HTMLDoc) this.loading.elementAt(i);
+		this.loaded.addElement(doc);
+
+		// remove it from the loading list
+		this.loading.removeElementAt(i);
+	}
+
+	/**
+	 *	Get a loaded document, not blocking if nothing is ready yet.
+	 *
+	 *  @return <code>HTMLDoc</code> the document; NB. this may be <code>null</code> as
+	 *                               it is not guaranteed that there is currently a loaded
+	 *                               document.
+	 */
+	public synchronized HTMLDoc nextLoaded()
+	{	HTMLDoc reply;
+
+		// if nothing is loaded, return null
+		if (this.loaded.size() == 0)
+		{	return null;
+		}
+
+		// get the first element from the loaded vector and return it,
+		// removing it from the vector in the meantime
+		reply = (HTMLDoc) this.loaded.elementAt(0);
+		this.loaded.removeElementAt(0);
+		return reply;
+	}
+
+	/**
+	 *	Request the loading of a document; this does not block but schedules
+	 *  the document for loading.
+	 *
+	 *  @param <code>URL</code> the URL of the document to load
+	 */
+	public synchronized void load(URL docURL)
+	{	// check if it's being loaded already; if so do nothing
+		if (this.isLoading(docURL.toString()) == false)
+		{	this.toLoad.addElement(docURL);
+
+			// self-start if the loading thread isn't running
+			if (this.started == false)
+			{	this.start();
+			}
+		}
+	}
+
+	/**
+	 *	Get the total number of documents loading/loaded through this loader
+	 *
+	 *	@param <code>int</code> the number of documents
+	 */
+	public synchronized int loadingDocs()
+	{	return this.toLoad.size() + this.loading.size() + this.loaded.size();
+	}
+
+	/**
+	 *	Instruct the manager to stop loading documents
+	 */
+	public synchronized void stopLoading()
+	{	this.stopFlag = true;
+	}
+
+	protected void finalize() throws Throwable
+	{	this.thread 	= null;
+		this.loading 	= null;
+		this.toLoad		= null;
+		this.loaded		= null;
+		super.finalize();
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java	(revision 14150)
@@ -0,0 +1,353 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+import java.net.*;
+
+import java.io.*;
+
+
+
+public class HTMLLoader implements Runnable
+
+{ URL 					from;
+
+	StringBuffer	HTML;
+
+	boolean				loaded;
+
+	boolean				error;
+
+	boolean				notFound;
+
+	Thread				thread;
+
+	static int		running;
+
+
+
+	/**
+
+	 * Constructor to load given URL
+
+	 */
+
+	public HTMLLoader(URL from)
+
+	{ this.from 	= from;
+
+		this.HTML 	= new StringBuffer();
+
+		this.loaded	= false;
+
+		this.error	= false;
+
+		this.thread = null;
+	}
+
+
+
+	/**
+
+	 * @return <code>0</code> if finished running; any other value if
+
+	 *         still executing
+
+	 */
+
+	public static synchronized int running()
+
+	{ return running;
+
+	}
+
+
+
+	/**
+
+	 * Starts the thread; the method <code>load</code> is a synonym for this
+
+	 */
+
+	public synchronized void start()
+
+	{ if (thread == null)
+
+		{ thread = new Thread(this);
+
+			thread.start();
+
+			running	++;
+
+		}
+
+//		System.out.println("Running: "+running);
+
+	}
+
+
+
+	/**
+
+	 * Do the actual loading; this will multi-thread with calling task
+
+	 */
+
+	public void run()
+
+	{	URLConnection connect;
+
+		int						length, i, c;
+
+		byte					inbuffer[];
+
+		InputStream		in;
+
+
+
+		inbuffer = new byte[128];
+
+
+
+		this.notFound	= false;
+//		System.out.println("Running loader");
+
+
+
+		// get connection and test length
+
+		try
+
+		{	connect = this.from.openConnection();
+
+		}
+
+		catch (IOException ioex)
+
+		{ this.loaded	= true;
+
+			this.error	=	true;
+
+			running --;
+			System.out.println(ioex);
+
+			return;
+
+		}
+
+
+		length = connect.getContentLength();
+
+		if (length == 0)
+
+		{ this.loaded = true;
+
+			running --;
+			System.out.println("Empty");
+
+			return;
+
+		}
+
+
+
+		// get an inputstream to the document body
+
+		try
+
+		{ in = connect.getInputStream();
+
+		}
+
+		catch (IOException ioex)
+
+		{ this.error	= true;
+
+			this.loaded	= true;
+
+			running --;
+			System.out.println("Open failed" + ioex);
+
+			return;
+
+		}
+
+
+
+		if (length < 0)
+
+		{	if (connect.getHeaderField(0) != null &&
+
+					connect.getHeaderField(0).indexOf("404") >= 0)
+
+			{	this.notFound	= true;
+
+				this.error		= true;
+
+				this.loaded		= true;
+
+				running --;
+				System.out.println("Not found");
+
+				return;
+
+			}
+
+		}
+
+//		System.out.println("Ready to read " + length);
+
+
+
+		// read in the document body
+
+		i = 0;
+
+		c = 0;
+
+		while (true || i < length)
+
+		{ // read in 128 characters (abort on IO Exception)
+
+			try
+
+			{ c = in.read(inbuffer);
+
+			}
+
+			catch (IOException ioex)
+
+			{ break;
+
+			}
+
+			// System.out.println("read " + c);
+
+
+
+			// if no characters were read, end, otherwise add it to the body
+
+			if (c != -1)
+
+			{ if (c > 0)
+
+				{ this.HTML.append(new String(inbuffer, 0, c));
+
+					i += c;
+
+				}
+
+			}
+
+			else
+
+			{ break;
+
+			}
+
+			// if an incomplete buffer was read (ie no characters remain), end
+
+/*			if (c < inbuffer.length && i >= length)
+
+			{ break;
+
+			}*/
+
+		}
+
+
+
+		// Close file
+
+		try
+
+		{	in.close();
+
+		}
+
+		catch (IOException ioex)
+
+		{	//System.out.println("Closed except "+ioex);
+
+		}
+
+	
+		this.loaded = true;
+
+  	running --;
+
+	}
+
+
+
+	/**
+
+	 * Do the actual loading (calls a Start on the Thread)
+
+	 */
+
+	public void load()
+
+	{	this.start();
+
+//		this.run();
+
+	}
+
+
+
+	/**
+
+	 * @return HTML text as a <code>String</code>.
+
+	 */
+
+	public synchronized String HTML()
+
+	{ if (this.error)
+
+		{ return "Error";
+
+		}
+
+		return this.HTML.toString();
+
+	}
+
+
+
+	/**
+
+	 * @return  <code>true</code> if the file has successfully loaded
+
+	 */
+
+	public boolean loaded()
+
+	{ return this.loaded;
+
+	}
+
+
+
+	protected void finalize() throws Throwable
+
+	{	this.thread = null;
+
+		this.from		= null;
+
+		this.HTML		= null;
+
+		super.finalize();
+
+	}
+
+
+
+	static
+
+	{ running = 0;
+
+	}
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java	(revision 14150)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLLocalAnchor
+{	
+	public static int anchorRef(HTMLBlockList anchors, String name)
+	{ int loop;
+		String	idvalue;
+
+		for (loop = 0; loop < anchors.size(); loop ++)
+		{	idvalue = anchors.tagBlock(loop).headTag().idValue("name");
+			if (idvalue != null && idvalue.equalsIgnoreCase(name))  //TODO: check if this should be case senstitive
+			{ return loop;
+			}
+		}
+		return -1;
+	}
+
+	public static int header(HTMLDoc doc, String local)
+	{	HTMLBlockList anchors;
+		int						heading;
+		HTMLBlock			headingblock;
+		int						anchor;
+		HTMLBlock			anchorblock;
+
+		anchors	= HTMLBlockList.anchorlist(doc);
+		anchor	= anchorRef(anchors, local);
+		if (anchor < 0)
+		{	return -1;
+		}
+
+		anchorblock	= anchors.tagBlock(anchor);
+		for (heading = 0; heading < doc.headerBlocks().size(); heading ++)
+		{	headingblock	= doc.headerBlocks().tagBlock(heading);
+
+			if (headingblock.startPos() > anchorblock.endPos())
+			{	break;
+			}
+		}
+		return heading - 1;
+	}
+	
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLObject.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLObject.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLObject.java	(revision 14150)
@@ -0,0 +1,5 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public interface HTMLObject
+{ public String getHTMLType();
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLParser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLParser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLParser.java	(revision 14150)
@@ -0,0 +1,243 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.applet.*;
+
+public class HTMLParser
+{ 
+    HTMLDoc document;
+    int pos;
+    int	lastpos;
+
+    public HTMLParser(HTMLDoc document)
+    { 
+	this.document = document;
+	this.pos = 0;
+    }
+    
+    public void startParse()
+    { 
+	this.pos = 0;
+	this.lastpos = -1;
+	//System.out.println("Starting "+document.urlString());
+    }
+
+    public int atParse()
+    { 
+	return this.pos;
+    }
+
+    public int lastParse()
+    { 
+	return this.lastpos;
+    }
+
+    // do a full text/tag parse
+    public String fullParse()
+    { 
+	String reply = null;
+	int end;
+	int start;
+
+	this.lastpos = this.pos;
+	
+	if (this.pos >= this.document.getContent().length()){
+	    return reply;
+	}
+	
+	if (this.document.getContent().charAt(this.pos) == '<'){
+	    start = this.pos;
+	    
+	    // if we're not at the end of the document,
+	    // read the rest of the tag
+	    if (this.pos == this.document.getContent().length() - 1){
+		this.pos = this.document.getContent().length();
+		return reply;
+	    }
+	    
+	    // if the tag is a comment
+	    if (this.pos < this.document.getContent().length() - 3 &&
+		this.document.getContent().substring(this.pos, this.pos+4).equals("<!--")){
+		end = this.document.getContent().substring(this.pos).indexOf("-->") + 3 + this.pos;
+		reply = this.document.getContent().substring(this.pos, end);
+		this.pos = end;
+	    }
+	    else 
+		{ // read up to the end of the tag
+		    end = this.pos + 1;
+		    while (end < this.document.getContent().length() &&
+			   this.document.getContent().charAt(end) != '>'){
+			end ++;
+		    }
+
+		    // get the whole of the tag into 'reply', and
+		    // set the current pos to immediately after the tag
+		    if (end < this.document.getContent().length()){
+			reply = this.document.getContent().substring(this.pos, end + 1);
+			this.pos = end + 1;
+		    }
+		    // patch the trailing > onto the tag string
+		    else {
+			reply = this.document.getContent().substring(this.pos, end) + ">";
+			this.pos = end;
+		    }
+		}
+	}
+	else
+	    { // hunt for the beginning of the next tag
+		start = this.pos;
+		while ((this.pos < this.document.getContent().length()) &&
+		       (this.document.getContent().charAt(this.pos) != '<')){
+		    this.pos ++;
+		}
+		
+		// return everything up to that tag
+	    reply = this.document.getContent().substring(start, this.pos);
+	    }
+	return reply;
+    }
+
+    // Get the next tag to parse
+    public String nextParse()
+    { 
+	String reply = null;
+	int end;
+
+	this.lastpos = this.pos;
+
+	if (this.document.getContent() == null){
+	    return null;
+	}
+	
+	if (this.pos >= this.document.getContent().length()){
+	    return reply;
+	}
+	
+	// hunt for the beginning of the next tag
+	while ((this.pos < this.document.getContent().length()) &&
+	       (this.document.getContent().charAt(this.pos) != '<')){
+	    this.pos ++;
+	}
+	
+	// if we're not at the end of the document,
+	// read the rest of the tag
+	if (this.pos < this.document.getContent().length()){
+	    if (this.pos == this.document.getContent().length() - 1){
+		this.pos = this.document.getContent().length();
+		return reply;
+	    }
+	    
+	    end = this.pos + 1;
+	    while (end < this.document.getContent().length() &&
+			this.document.getContent().charAt(end) != '>'){
+		end ++;
+	    }
+	    
+	    // get the whole of the tag into 'reply', and
+	    // set the current pos to immediately after the tag
+	    if (end < this.document.getContent().length()){
+		reply = this.document.getContent().substring(this.pos, end + 1);
+		this.pos = end + 1;
+	    }
+	    else{ 
+		this.pos = end;
+	    }
+	}
+	return reply;
+    }
+    
+    
+    /* --
+       -- return next HREF value we come across in the document
+       --
+       -- NB: the checking of the quotes etc is rather lazy - and should be tidied
+       --
+    */
+    public String nextHREF()
+    { 
+	String reply;
+	int start, end;
+	boolean quoted;
+	HTMLTag tag;
+	
+	reply = this.nextParse();
+	while (this.pos < this.document.getContent().length()){
+	    tag = new HTMLTag(reply);
+	    
+	    if (tag.tagName().equals("a")) {
+		start = reply.indexOf("href");
+		if (start == -1) {
+		    start = reply.indexOf("HREF");
+		}
+		
+		if (start >= 0) {
+		    start += 4;
+		    quoted = false;
+		    while (reply.charAt(start) == ' ' ||
+			   reply.charAt(start) == '=' ||
+			   reply.charAt(start) == '"') {
+			if (reply.charAt(start) == '"') {
+			    quoted = true;
+			}
+			start ++;
+		    }
+		    
+		    end = -1;
+		    if (quoted) {
+			end = reply.indexOf('"', start+1);
+		    }
+		    if (end == -1) {
+			end = reply.indexOf(' ', start+1);
+			if (end == -1) {
+			    end = reply.length() - 1;
+			}
+		    }
+		    
+		    reply = reply.substring(start, end);
+		    return reply;
+		}
+	    }
+	    reply = this.nextParse();
+	}
+	return null;
+    }
+
+    // Return the next link in the page
+    public String nextLink(AppletContext ac)
+    { 
+	String reply, reply2;
+	int start, end;
+	boolean quoted;
+	HTMLTag tag;
+	
+	if (this.document.getContent() == null){
+	    return null;
+	}
+	
+	reply = this.nextParse();
+	while (this.pos < this.document.getContent().length()){
+	    tag = new HTMLTag(reply);
+	    if (tag.tagName().equals("a")){
+		reply = tag.idValue("href");
+	    }
+	    else if (tag.tagName().equals("frame")){ // image tag
+		reply = tag.idValue("src");
+	    }
+	    else if (tag.tagName().equals("area")) { // image map area
+		reply = tag.idValue("href");
+	    }
+	    else if (tag.tagName().equals("frame")){
+		reply = tag.idValue("src");
+	    }
+	    else {
+		reply	= null;
+	    }
+	    
+	    if (reply != null){
+		return reply;
+	    }
+ 
+	    reply = this.nextParse();
+	}
+	return null;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTag.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTag.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTag.java	(revision 14150)
@@ -0,0 +1,265 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLTag implements HTMLObject
+{ String tagtext;
+  String name;
+  int	 docpos;
+  int    endpos;
+
+  public static final String HTML_ELEMENT_TYPE = "Element";
+
+  public HTMLTag(String tagtext, int docpos, int endpos)
+  { this.tagtext = tagtext;
+    this.name    = null;
+    this.docpos  = docpos;
+    if (this.docpos < 0)
+    { this.endpos = -1;
+    }
+    else
+    { this.endpos = endpos;
+    }
+  }
+
+  public HTMLTag(String tagtext)
+  { this.tagtext 	= tagtext;
+    this.docpos		= -1;
+    this.name			= null;
+  }
+
+  public String getHTMLType()
+  { return HTML_ELEMENT_TYPE;
+  }
+
+	public HTMLTag endTag()
+	{ HTMLTag end;
+		String	endtext;
+
+		endtext = "</"+this.xtagName()+">";
+
+		end = new HTMLTag(endtext);
+		return end;
+	}
+
+	public boolean tagIsOpening()
+	{ return (this.tagtext.charAt(1) != '/');
+	}
+
+	/**
+	 *  Tells you if a tag is a singleton tag - TODO: complete list
+	 */
+	public boolean tagIsSingleton()
+	{	String tagName = this.tagName();
+		
+		return (tagName.equals("img"));
+	}
+
+	/**
+	 *  @return: the name of a tag including any leading '/'
+	 */
+	public String tagName()
+	{ int offset, start, end;
+		String reply;
+
+		if (this.name != null)
+		{	return this.name;
+		}
+	
+		// NB: a starting '<' and an ending '>' are guaranteed
+		offset = 1; // skip <
+		while (this.tagtext.charAt(offset) <= 32 ||
+					 this.tagtext.charAt(offset) == 127)
+		{ offset ++;
+		}
+		start = offset;
+		while (this.tagtext.charAt(offset) > 32 &&
+					 this.tagtext.charAt(offset) != '>' &&
+					 this.tagtext.charAt(offset) != 127)
+		{ offset ++;
+		}
+		end = offset;
+
+		reply = this.tagtext.substring(start, end);
+		reply = reply.toLowerCase();
+
+		this.name	= reply;
+
+		return reply;
+	}
+
+	// Returns the name of the tag *minus* any leading / for
+	// closing tags
+	public String xtagName()
+	{ String reply;
+
+		reply = this.tagName();
+		if (reply == null || reply.length() == 0)
+		{	return null;
+		}
+
+		if (reply.charAt(0) == '/')
+		{ if (reply.length() > 1)
+			{	reply = reply.substring(1, reply.length());
+			}
+			else
+			{ reply = "";
+			}
+		}
+		return reply;
+	}
+
+	public String endTagName()
+	{	String reply;
+
+		reply = "/" + xtagName();
+		return reply;
+	}
+
+	public boolean isClosing()
+	{ String name;
+
+		name	= this.tagName();
+		if (name.charAt(0) == '/')
+		{	return true;
+		}
+		return false;
+	}
+
+	public int tagLevel()
+	{ String	name;
+		int			level;
+
+		level = 7;
+
+		name = this.xtagName();
+		if (name.charAt(0) == 'h')
+		{ level = Integer.parseInt(name.substring(1, 2));
+		}
+		
+		return level;
+	}
+
+	public int startPos()
+	{	return this.docpos;
+	}
+
+	public int endPos()
+	{	return this.endpos;
+	}
+
+	static public boolean isTag(String text)
+	{ if (text.charAt(0) == '<')
+		{ return true;
+		}
+		return false;
+	}
+
+	public String toString()
+	{	return this.tagtext;
+	}
+
+	//
+	// Returns the value of the identifier given as a string
+	//
+	public String idValue(String idName)
+	{	int offset, start, end;
+		int idstart, idend, valuestart, valueend;
+		boolean isvalue, isquoted, isstopped;
+		char		quotechar;
+		String reply;
+
+		// NB: a starting '<' and an ending '>' are guaranteed
+		offset = 1; // skip <
+		while (this.tagtext.charAt(offset) <= 32 ||
+					 this.tagtext.charAt(offset) == 127)
+		{ offset ++;
+		}
+		start = offset;
+		while (this.tagtext.charAt(offset) > 32 &&
+					 this.tagtext.charAt(offset) != '>' &&
+					 this.tagtext.charAt(offset) != 127)
+		{ offset ++;
+		}
+		end = offset;
+		// End of tag name
+
+		if (this.tagtext.charAt(offset) == '>')
+		{ return null;
+		}
+
+		isstopped = false;
+		while (!isstopped)
+		{	isvalue		= false;
+			isquoted	= false;
+			valuestart = -1;
+			valueend = -1;
+
+			while (this.tagtext.charAt(offset) <= 32 ||
+						this.tagtext.charAt(offset) == 127)
+			{ offset ++;
+			}
+			idstart = offset;
+
+			if (this.tagtext.charAt(offset) == '>')
+			{ break;
+			}
+
+			while (this.tagtext.charAt(offset) > 32 &&
+						 this.tagtext.charAt(offset) != '=' &&
+						 this.tagtext.charAt(offset) != '>' &&
+						 this.tagtext.charAt(offset) != 127)
+			{ offset ++;
+			}
+			idend = offset;
+
+			while (this.tagtext.charAt(offset) <= 32 ||
+						 this.tagtext.charAt(offset) == 127)
+			{ offset ++;
+			}
+
+			if (this.tagtext.charAt(offset) == '=')
+			{ isvalue = true;
+				quotechar = ' ';
+				offset ++;				
+
+				while (this.tagtext.charAt(offset) <= 32 ||
+							 this.tagtext.charAt(offset) == 127)
+				{ offset ++;
+				}
+				if (this.tagtext.charAt(offset) == '"' ||
+						this.tagtext.charAt(offset) == '\'')
+				{ quotechar = this.tagtext.charAt(offset);
+					isquoted = true;
+					offset ++;
+				}
+				valuestart = offset;
+
+				while (offset < this.tagtext.length() &&
+							 (isquoted && this.tagtext.charAt(offset) != quotechar) ||
+							 (!isquoted && this.tagtext.charAt(offset) > 32 &&
+							   this.tagtext.charAt(offset) != '>'))
+				{ offset ++;
+				}
+				valueend = offset;
+			}
+
+			if (offset >= this.tagtext.length() ||
+					this.tagtext.charAt(offset) == '>')
+			{ isstopped = true;
+			}
+			offset ++;
+
+			reply = this.tagtext.substring(idstart, idend);
+			reply = reply.toLowerCase();
+
+			if (reply.equalsIgnoreCase(idName))
+			{ if (isvalue)
+				{ return this.tagtext.substring(valuestart, valueend);
+				}
+				else
+				{ return reply;
+				}
+			}
+		}
+		return null;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java	(revision 14150)
@@ -0,0 +1,60 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLTagVector
+{ Vector vector;
+
+	//
+	// private: creates a Vector of tags from a comma separated list
+	//
+	public HTMLTagVector(String list)
+	{ int pos;
+		String tagname;
+
+		this.vector = new Vector(1);
+
+		// get first index
+		pos = list.indexOf(',');
+
+		// add leading tag to index, if there is > 1 tag currently
+		// in the list, and discard the tag
+		while (pos >= 0)
+		{ tagname = list.substring(0, pos);
+			list		= list.substring(pos+1, list.length());
+			this.vector.addElement((Object) tagname);
+
+			pos			= list.indexOf(',');
+		}
+
+		// add the last tag to the list
+		tagname = list;
+		this.vector.addElement((Object) tagname);
+	}
+
+	public HTMLTagVector(Vector vector)
+	{ this.vector = vector;
+	}
+
+	/**
+	 * Check whether a given tag is in the tagVector.  If <CODE>opening</CODE> is false,
+	 * then the method will return true for both opening and closing versions of the tag.
+	 * If the
+	 */
+	public boolean matches(HTMLTag tag, boolean opening)
+	{ int tagref;
+
+		if (tag.xtagName() == null)
+		{	return false;
+		}
+
+		for (tagref = 0; tagref < this.vector.size(); tagref ++)
+		{ if ((tag.xtagName().equals((String) this.vector.elementAt(tagref))) &&
+		      (tag.tagIsOpening() || !opening))
+			{ return true;
+			}
+		}
+		return false;
+	}
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java	(revision 14150)
@@ -0,0 +1,232 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.File;
+import java.io.FileReader;
+
+import java.net.URL;
+
+import org.w3c.dom.*;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.ext.LexicalHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.doctypes.GMLDocument;
+
+public class HTMLTidy extends DefaultHandler
+{
+    String label;
+    File   file;
+    StringBuffer value;
+    boolean inElement;
+    boolean accumulate;
+    Document document;
+    Element  currentElement;
+    Element root;
+
+    class HTMLCommentHandler implements LexicalHandler
+    {
+	HTMLTidy parent;
+
+	HTMLCommentHandler(HTMLTidy tidier)
+	{ 
+	    this.parent = tidier;
+	}
+
+	public void comment(char ch[], int start, int length)
+	{ 
+	    String comment = new String(ch, start, length);
+	    parent.comment(comment);
+	}
+	
+	public void endCDATA()
+	{
+	}
+
+	public void endDTD()
+	{
+	}
+
+	public void endEntity(String name)
+	{
+	}
+
+	public void startCDATA()
+	{
+	}
+
+	public void startDTD(String name, String publicId, String systemId)
+	{
+	}
+
+	public void startEntity(String name)
+	{
+	}
+    }
+
+    public HTMLTidy(URL url)
+    { 
+	// TODO: complete
+    }
+
+    public HTMLTidy(File file)
+    { 
+	this.file = null;
+	this.label = null;
+	this.value = null;
+	
+	this.document = new org.apache.html.dom.HTMLDocumentImpl();
+	
+	this.currentElement = this.document.getDocumentElement();
+	this.root = this.currentElement;
+	
+	try {
+	    XMLReader handler  = new org.ccil.cowan.tagsoup.Parser();
+	    HTMLCommentHandler commentHandler = new HTMLCommentHandler(this);
+	    handler.setContentHandler(this);
+	    handler.setProperty("http://xml.org/sax/properties/lexical-handler", commentHandler);
+	    handler.parse(file.toURI().toString());
+	}
+	catch (java.io.IOException io)
+	    {
+		System.out.println(io);
+	    }
+	catch (org.xml.sax.SAXException saxEx)
+	    {
+		System.out.println(saxEx);
+	    }
+    }
+    
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { 
+	//System.out.println("<"+localName+">");
+	
+	if (localName.equals("html") == false) {
+	    Element childElement = this.document.createElement(localName);
+	    
+	    this.currentElement.appendChild(childElement);
+	    this.currentElement = childElement;
+	    
+	    for (int a = 0; a < attributes.getLength(); a ++){
+		String name = attributes.getLocalName(a);
+		String value = attributes.getValue(a);
+		
+		childElement.setAttribute(name, value);
+	    }
+	}
+    }
+    
+    public void comment(String text)
+    {
+	Comment comment = document.createComment(text);
+	this.currentElement.appendChild(comment);
+	//System.out.println("<!-- "+text+" -->");
+    }
+    
+    public void endElement(String URI, String localName, String qName)
+    { 
+	// System.out.println("Ending " + localName);
+	// System.out.println("</"+localName+">");
+	
+	if (localName.equals("html")) {
+	    return;
+	}
+	
+	// TODO: check is the same name as current node
+	if (!this.currentElement.getTagName().equals(localName.toUpperCase())) {
+	    // TODO: error handling
+	    System.out.println("Bad end at " + localName + " " + this.currentElement.getTagName());
+	    return;
+	}
+	
+	// Optimisation: is this an empty tag?
+	boolean isEmpty = false;
+	
+	if (this.currentElement.getChildNodes().getLength() == 1){
+	    Node child = this.currentElement.getFirstChild();
+	    
+	    if (child.getNodeType() == Node.TEXT_NODE) {
+		String text = child.getNodeValue();
+		
+		int i = 0;
+		while (i < text.length()){
+		    if (text.charAt(i) > 32 && text.charAt(i) != 160) {
+			break;
+		    }
+		    i++;
+		}
+		if (text.length() == i) {
+		    isEmpty = true;
+		}
+	    }
+	}
+	else if (this.currentElement.getChildNodes().getLength() == 0){
+	    isEmpty = true;
+	}
+	
+	boolean isSingleton = false;
+	String tagName = this.currentElement.getTagName().toLowerCase();
+	
+	if (tagName.equals("img") ||
+	    tagName.equals("br") ||
+	    tagName.equals("meta") ||
+	    tagName.equals("input") ||
+	    tagName.equals("area") ||
+	    tagName.equals("link") ||
+	    tagName.equals("base") ||
+	    //tagName.equals("img") ||
+	    tagName.equals("hr")) {
+	    isSingleton = true;
+	}
+	
+	Element parent =  (Element) this.currentElement.getParentNode();
+
+	if (isEmpty == true && !isSingleton) {
+	    // trim the current child
+	    parent.removeChild(this.currentElement);
+	    //      System.out.println("***Removing child***");
+	}
+	
+	this.currentElement = parent;
+    }
+    
+    public void characters(char c[], int start, int length)
+    { 
+	String string = new String(c, start, length);
+	Node text_node = this.document.createTextNode(string);
+	this.currentElement.appendChild(text_node);
+	//    System.out.println(string);
+    }
+    
+    public void recursePrint(Element element, int indent)
+    {
+	for (int i = 0; i < indent; i++) {
+	    System.out.print(" ");
+	}
+	System.out.println(element.toString());
+	for (int c = 0; c < element.getChildNodes().getLength(); c ++){
+	    Node n = element.getChildNodes().item(c);
+	    
+	    if (n.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+		Element e = (Element) n;
+		
+		this.recursePrint(e, indent+2);
+	    }
+	}
+    }
+    
+    public Document getDocument()
+    { 
+	//    this.recursePrint(this.root, 0);
+	//    System.out.println(this.document.toString());
+	//    System.out.println(this.root);
+	return this.document;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTTPTools.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTTPTools.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTTPTools.java	(revision 14150)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.IOException;
+import java.net.*;
+
+public class HTTPTools
+{
+	public static String getMIMEType(URL url)
+	{ try
+		{	// initiate the connection
+			URLConnection connection = url.openConnection();
+
+			connection.connect(); // actually make the connection
+
+			return connection.getContentType();
+		}
+		catch (IOException exception)
+		{ // TODO: Log error reading url here...
+		}
+		return null;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/MultiMap.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/MultiMap.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/MultiMap.java	(revision 14150)
@@ -0,0 +1,74 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.ArrayList;
+
+public class MultiMap
+{	Map map;
+	
+	public MultiMap()
+	{	this.map = new HashMap();
+	}
+
+	public boolean containsKey(Object key)
+	{	return this.map.containsKey(key);
+	}
+
+	public void setOnly(Object key, Object value)
+	{	List child = new ArrayList();
+		child.add(value);
+		this.map.put(key, child);
+	}
+
+	public void put(Object key, Object value)
+	{	List children;
+
+		children = (List) this.map.get(key);
+		if (children == null)
+		{	children = new ArrayList();
+			this.map.put(key, children);
+		}
+		children.add(value);
+	}
+
+	public Object get(Object key)
+	{	List list = this.getAll(key);
+		if (list == null)
+		{	return null;
+		}
+		return list.get(0);
+	}
+
+	public List getAll(Object key)
+	{	return (List) this.map.get(key);
+	}
+
+	public void remove(Object key)
+	{	this.map.remove(key);
+	}
+
+	public boolean remove(Object key, Object value)
+	{	if (!this.map.containsKey(key))
+		{ return false;
+		}
+
+		List children = (List) this.map.get(key);
+
+		if (children != null)
+		{	for (int i = 0; i < children.size(); i ++)
+			{	if (children.get(i).equals(value))
+				{	children.remove(i);
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+	public Set keySet()
+	{	return this.map.keySet();
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/SQLUtils.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/SQLUtils.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/SQLUtils.java	(revision 14150)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class SQLUtils
+{
+	public void initCollection(String collectionName)
+	{	
+		// Check if the collection already exists
+		// if so, abort
+		// else
+		//   create document table
+		//   create section table
+		//   create metadata table
+		//   create namespace table
+		//   create metadata values table
+		//   create file section table
+		//   create file groups table
+		//   create file table
+		//   create admin table...etc.
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLString.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLString.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLString.java	(revision 14150)
@@ -0,0 +1,619 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.applet.*;
+import java.net.*;
+
+public class URLString
+{ private String url;
+
+  private String urlbase(String url)
+  { int index;
+    String tailString, bodyString;
+    int body;
+
+    index = url.indexOf("//");
+    tailString = url.substring(index+2, url.length());
+    
+    body = tailString.indexOf('/');
+    bodyString = tailString.substring(0, body+1);
+    
+    tailString = url.substring(0, index+2) + bodyString;
+    return tailString;
+  }
+  
+  public URLString(String s)
+  { this.url = s;
+  }
+
+  public URLString(String s, String target)
+  { // reconcile
+    int sref;
+    int cref;
+    int	c;
+
+    // trim fragment identifiers
+    target = noRef(target);
+    if (target == null)
+    { this.url = s;
+      return;
+    }
+
+    // test for absolute
+    cref = target.indexOf(':');
+    if (cref >= 0)
+    { for (c = 0; c < cref; c ++)
+      { if (target.charAt(c) != '+' &&
+	    target.charAt(c) != '-' &&
+	    target.charAt(c) != '.' &&
+	    Character.isLetterOrDigit(target.charAt(c)) == false)
+	{ break;
+	}
+      }
+    }
+    else
+    { c = 0;
+    }
+
+    if (c == cref)
+    { // absolute reference
+      String tmp;
+
+      tmp = target.substring(0, c);
+      tmp = tmp.toLowerCase();
+      if (target.indexOf("//") == cref + 1)
+      { if (target.lastIndexOf('/') == cref + 2)
+	{ target = target + "/";
+	}
+      }
+      /*			if (tmp.equals("http") == false)
+				{ this.url = null;
+				return;
+				}*/
+      this.url = target;
+    }
+    else if (target.indexOf("//") == 0)
+    { // absolute - implicit scheme (ie http).; also check for no trailing '/'
+      if (target.lastIndexOf('/') == 1)
+      { target = target + "/";
+      }
+      target = "http:" + target;
+      this.url = target;
+    }
+    else
+    { // relative - trim silly '/' at end
+      if (target.charAt(0) == '/')	
+      { // if we get a leading '/' then add it to the base of the url
+	s = urlbase(s);
+	target = target.substring(1);
+	this.url = s + target;
+//				System.out.println(this.url+"<"+s+"<"+target);
+	return;
+      }
+
+      while(target.length() >= 2 && target.charAt(0) == '.' && target.charAt(1) == '.')
+      { // move up a level on the left
+	
+	sref = s.lastIndexOf('/');
+	s = s.substring(0, sref);
+
+	target = target.substring(3);
+      }
+      // strip trailing leaf from s
+      sref = s.lastIndexOf('/');
+      s = s.substring(0, sref+1);
+      // append s+target
+      this.url = s + target;
+    }
+    
+    // abandon the url if it is a call to a script
+    if (this.url.indexOf('?') >= 0)
+    { this.url = null;
+    }
+  }
+
+  public static boolean dirParent(String parent, String child)
+  { while (child.length() >= parent.length())
+    { child = child.substring(0, child.lastIndexOf('/'));
+
+      if (parent.equals(child))
+      {	return true;
+      }
+    }
+    return false;
+  }
+
+  public static int dirPos(String from, String to)
+  { String fromdir;
+    String todir;
+
+    fromdir = from.substring(0, from.lastIndexOf('/'));
+    todir   = to.substring(0, to.lastIndexOf('/'));
+
+    if (fromdir.equals(todir))
+    { return 0;
+    }
+    if (fromdir.length() > todir.length())
+    { if (dirParent(todir, fromdir) == true)
+      {	return -1;
+      }
+    }
+    else 
+    { if (dirParent(fromdir, todir) == true)
+      { return 1;
+      }
+    }
+    return -2;
+  }
+
+  public static int commonLevels(String a, String b)
+  { String 	path_a, path_b;
+    int			offset_a, offset_b;
+    String	dir_a, dir_b;
+    int			common;
+
+    path_a	= path(a);
+    path_b	= path(b);
+    common	= 0;
+
+    if (path_a == null || path_b == null)
+    { return 0;
+    }
+
+    do
+    { offset_a = path_a.indexOf('/');
+      offset_b = path_b.indexOf('/');
+
+      if (offset_a > 0)
+      { dir_a		=	path_a.substring(0, offset_a);
+        if (path_a.length() > offset_a + 1)
+	{	path_a	= path_a.substring(offset_a+1);
+	}
+	else
+	{	path_a	= null;
+	}
+      }
+      else
+      {	dir_a		=	path_a;
+        path_a	= null;
+      }
+
+      if (offset_b > 0)
+      {	dir_b		=	path_b.substring(0, offset_b);
+        if (path_b.length() > offset_b + 1)
+	{	path_b	= path_b.substring(offset_b+1);
+	}
+	else
+	{	path_b	= null;
+	}
+      }
+      else
+      {	dir_b		=	path_b;
+        path_b	= null;
+      }
+
+      if (dir_a.length() != dir_b.length() ||
+	  dir_a.equals(dir_b) == false)
+      {	return common;
+      }
+      common ++;
+    } while (path_a != null && path_b != null);
+    
+    return common;
+  }
+
+	public static String dirLevelName(String url, int level)
+	{	String 	lpath;
+		String	dir;
+		int			thislevel = 0;
+		int			offset;
+
+		lpath = path(url);
+		if (lpath == null)
+		{ return null;
+		}
+		do
+		{ offset = lpath.indexOf('/');
+
+			if (offset >= 0)
+			{	dir = lpath.substring(0, offset);
+				lpath = lpath.substring(offset+1);
+			}
+			else
+			{	dir = lpath;
+			}
+
+			thislevel ++;
+
+			if (level == thislevel)
+			{ return dir;
+			}
+		} while (offset >= 0);
+
+		return null;
+	}
+
+	public static int dirLevel(String url)
+	{	String 	lpath;
+		int			level;
+		int			offset;
+
+		lpath = path(url);
+		if (lpath == null)
+		{	return 0;
+		}
+		level = 0;
+		do
+		{ offset = lpath.indexOf('/');
+			if (offset >= 0)
+			{	lpath = lpath.substring(offset+1);
+			}
+			level ++;
+		} while (offset >= 0);
+
+		return level;
+	}
+
+	public static String tidyURL(String urlString, boolean permitScript)
+	{	int cref;
+		int c;
+		int sref;
+		String tmp, s;
+
+		s = urlString;
+		cref = s.indexOf(':');
+		if (cref >= 0)
+		{ for (c = 0; c < cref; c ++)
+			{ if (s.charAt(c) != '+' &&
+						s.charAt(c) != '-' &&
+						s.charAt(c) != '.' &&
+						Character.isLetterOrDigit(s.charAt(c)) == false)
+				{ break;
+				}
+			}
+		}
+		else
+		{ c = 0;
+		}
+
+		if (cref == c)
+		{ tmp = s.substring(0, c).toLowerCase();
+			s = tmp + s.substring(c, s.length());
+			if (s.lastIndexOf('/') == cref + 2)
+			{ s = s + "/";
+			}
+		}
+		else
+		{ sref = s.indexOf("//");
+			if (sref == 0)
+			{ s = "http:" + s;
+			}
+			else
+			{ s = "http://" + s;
+			}
+			if (s.lastIndexOf('/') == 6)
+			{ s = s + "/";
+			}
+		}
+		return s;
+	}
+
+	public static String tidyURL(String urlString)
+	{	return tidyURL(urlString, false);
+	}
+
+	public static URL toURL(String urlString)
+	{ URL reply;
+
+	  try
+		{ reply = new URL(URLString.tidyURL(urlString));
+		}
+		catch (MalformedURLException ex)
+		{ return null;
+		}
+		return reply;
+	}
+
+  public void tidy()
+  { this.url = tidyURL(this.url);
+  }
+
+  public boolean isNull()
+  { if (this.url == null)
+    { return true;
+    } 
+    return false;
+  }
+
+  public static boolean isForeign(String home, URL url)
+  { if (url.toString().length() < home.length())
+    { return true;
+    }
+
+    if (home.equals(url.toString().substring(0, home.length())))
+    { return false;
+    }
+    return true;
+  }
+  
+  /**
+   *	Remove any trailing "#localanchor" local reference from a URL string
+   *
+   *  @param  <code>String</code> the original url
+   *  @return <code>String</code> the cleaned url, sans local references
+   */
+  public static String noRef(String target)
+  { if (target == null)
+    { return null;
+    }
+    if (target.indexOf('#') >= 0)
+    { if (target.indexOf('#') == 0)
+      { return null;
+      }
+      target = target.substring(0, target.indexOf('#'));
+    }
+    return target;
+  }
+
+  /**
+   *	Obtain the trailing local anchor reference from a URL, 
+   *  if it exists.
+   *    e.g. for http://www.test.mdx.ac.uk/example/index.html#credits the
+   *         response would be "credits"
+   *
+   *  @param  <code>String</code> the original url.
+   *  @return <code>String</code> the local reference.  <code>null</code> is
+   *                              returned if no local reference exists.
+   */
+  public static String subRef(String target)
+  { if (target.indexOf('#') >= 0)
+    {	return target.substring(target.indexOf('#')+1);
+    }
+    return null;
+  }
+
+  /**
+   *	Clean a URL of the form GET parameters.
+   *
+   *  @param  <code>String</code> the original url, with GET message parameters
+   *  @return <code>String</code> the cleaned url.
+   */
+  public static String noParameters(String url)
+  { int						c;
+
+    c = url.indexOf("?");
+    if (c < 0)
+    { return url;
+    }
+    return url.substring(0, c);
+  }
+
+	/**
+	 *	Obtain the form parameters from a url
+	 *
+	 *  @param  <code>String</code> the original url, with GET message parameters
+	 *  @return <code>String</code> the form "GET" parameters.  The leading '?' is
+	 *                              preserved in the reply.  If no parameters were
+	 *                              given, the result is <code>null</code>.
+	 */
+	public static String formParameters(String url)
+	{	int						c;
+
+		c = url.indexOf("?");
+		if (c < 0)
+		{	return null;
+		}
+		return url.substring(c);
+	}
+
+	/**
+	 *	Flatten a URL, making the '?' and '&' items into underscores
+	 *
+	 *  @param  <code>String</code> the original url, with GET message parameters
+	 *  @return <code>String</code> the "flattened" url
+	 */
+	public static String flattenedUrl(String oldUrl)
+	{	StringBuffer	reply;
+		int						c;
+		boolean				modified = false;
+
+		// replace ? with _
+		// replate & with _
+		reply = new StringBuffer(oldUrl);
+		for (c = 0; c < reply.length(); c ++)
+		{	if (reply.charAt(c) == '?' ||
+					reply.charAt(c) == '&')
+			{	reply.setCharAt(c, '_');
+				modified = true;
+			}
+		}
+
+		if (modified == true)
+		{	c	= oldUrl.indexOf(".html");
+			if (c >= 0)
+			{	while (c < oldUrl.length() - 5)
+				{	reply.setCharAt(c, reply.charAt(c+5));
+					c ++;
+				}
+				reply.setCharAt(c ++, '.');
+				reply.setCharAt(c ++, 'h');
+				reply.setCharAt(c ++, 't');
+				reply.setCharAt(c ++, 'm');
+				reply.setCharAt(c ++, 'l');
+			}
+		}
+
+		return reply.toString();
+	}
+
+	/**
+	 *	Provide the complete pathname in the given url
+	 *    eg. "example/index.html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> leaf file name
+	 */
+	public static String pathName(String url)
+	{	int offset;
+		String remainder;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset = url.indexOf("//");
+		if (offset < 0)
+		{ return null;
+		}
+
+		remainder = url.substring(offset + 2);
+		offset = remainder.indexOf('/');
+		if (offset < 0)
+		{	return null;
+		}
+		return remainder.substring(offset + 1);
+	}
+
+	/**
+	 *	Provide the leaf filename in the given url
+	 *    eg. "index.html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> leaf file name
+	 */
+	public static String leafName(String url)
+	{	int offset;
+		int dot;
+
+		if (url == null)
+		{	return null;
+		}
+		offset = url.lastIndexOf('/');
+		if (offset < 0)
+		{ return null;
+		}
+		if (offset == url.length() - 1)
+		{ return null;
+		}
+		dot = url.lastIndexOf('/');
+		if (dot < offset)
+		{	return null;
+		}
+		return url.substring(offset+1);
+	}
+
+	/**
+	 *	Provide the hostname in the given url
+	 *    eg. "www.test.mdx.ac.uk" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> host name
+	 */
+	public static String host(String url)
+	{	String scheme, tail;
+		int		 spos;
+
+		spos = url.indexOf("//");
+		if (spos >= 0)
+		{ scheme = url.substring(0, spos + 2);
+			tail = url.substring(spos + 2, url.length());
+			// do sub-areas
+			spos = tail.indexOf('/');
+			if (spos >= 0)
+			{ tail = tail.substring(0, spos+1);
+			}
+			scheme = scheme + tail;
+			return scheme;
+		}
+		return null;
+	}
+
+	/**
+	 *	Return the logical path of the url on the remote server
+	 *    eg. "example" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> logical path
+	 */
+	public static String path(String url)
+	{ String tail;
+		int offset;
+		String leaf;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset 	= url.indexOf("//");
+		if (offset < 0)
+		{	return null;
+		}
+
+		tail		= url.substring(offset+2);
+		offset	= tail.indexOf('/');
+		if (offset < 0)
+		{	return null;
+		}
+
+		tail		= tail.substring(offset+1);
+		if (tail == null || tail.length() == 0)
+		{	return null;
+		}
+
+		offset	= tail.lastIndexOf('/');
+		if (offset < 0)
+		{ return null;
+		}
+		else
+		{	tail = tail.substring(0, offset);
+		}
+		return tail;
+	}
+
+	/**
+	 *	Give the 'dot' extension of a url; 
+	 *    eg. "html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *	@param <code>String</code> url
+	 *  @return <code>String</code> extension.
+	 */
+	public static String extension(String url)
+	{ int offset;
+		String leaf;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset = url.lastIndexOf('/');
+		if (offset >= 0 && offset < url.length() - 1)
+		{ leaf = url.substring(offset+1, url.length());
+			offset = leaf.lastIndexOf('.');
+
+			// if we've got an extension then use it
+			if (offset >= 0)
+			{ int poffset;
+
+				// get parameter offset
+				poffset = leaf.indexOf('?');
+				if (poffset < 0)
+				{	poffset = leaf.length();
+				}
+
+				// give leaf value
+				return leaf.substring(offset, poffset);
+			}
+			else
+			{ return null;
+			}
+		}
+		return null;
+	}
+
+  public static String extension(URL url)
+  { return extension(url.toString());
+  }
+
+  public String toString()
+  { return this.url;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLTools.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLTools.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLTools.java	(revision 14150)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.URL;
+import java.io.File;
+
+public class URLTools
+{
+  public static URL getFileURL(File file)
+  { URL url;
+    String normalisedFileName = file.toString().replace(File.separatorChar, '/');
+
+    try {
+      url = new URL("file", "", normalisedFileName);
+    }
+    catch (java.net.MalformedURLException ex) {
+      return null;
+    }
+    return url;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/XMLTools.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/XMLTools.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/util/XMLTools.java	(revision 14150)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class XMLTools
+{
+    public static String getTag(String namespace, String label, boolean open)
+    {	
+	StringBuffer reply = new StringBuffer();
+
+	reply.append("<");
+	if (!open)
+	    {	
+		reply.append("/");
+	    }
+	if (namespace != null && namespace.length() > 0) {
+	    reply.append(namespace);
+	    reply.append(":");
+	}
+	reply.append(label);
+	reply.append(">");
+	return reply.toString();
+    }
+
+    public static String addAttribute(String tag, String attribute, String value)
+    {	
+	StringBuffer reply = new StringBuffer(tag);
+	reply.insert(reply.length() - 1 , ' ');
+	reply.insert(reply.length() - 1 , attribute);
+	reply.insert(reply.length() - 1 , "=\"");
+	reply.insert(reply.length() - 1 , value);
+	reply.insert(reply.length() - 1 , '\"');
+	return reply.toString();
+    }
+
+    public static String makeSingleton(String tag)
+    {	
+	StringBuffer reply = new StringBuffer(tag);
+	reply.insert(tag.length() - 1, " /");
+	return reply.toString();
+    }
+    
+    public static String getOpenTag(String namespace, String label)
+    {	
+	return getTag(namespace, label, true);
+    }
+    
+    public static String getCloseTag(String namespace, String label)
+    {	
+	return getTag(namespace, label, false);
+    }
+
+    /**
+     *  Clean a string to remove whitespace...
+     *
+     *  @param <code>String</code> the original text
+     *  @return <code>String</code> the 'cleaned' text
+     */
+    public static String cleanString(String original)
+    {	
+	StringBuffer buffer = new StringBuffer(original);
+
+	while (buffer.charAt(0) <= 32) {
+	    buffer.deleteCharAt(0);
+	}
+	
+	while (buffer.charAt(buffer.length() - 1) <= 32){
+	    buffer.deleteCharAt(buffer.length() - 1);
+	}
+	
+	int c = 1;
+	while (c < buffer.length()){
+	    if (buffer.charAt(c) < 32){
+		buffer.setCharAt(c, ' ');
+	    }
+	    if (buffer.charAt(c) == ' ' && buffer.charAt(c-1) == ' '){
+		buffer.deleteCharAt(c);
+		continue;
+	    }
+	    c ++;
+	}
+	return buffer.toString();
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java	(revision 14150)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.validator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public interface ValidatorInterface
+{
+	/**
+	 *  Record the build manager to use for this expander - this is used
+	 *  to pass back any new directories into which expanded files are placed.
+	 */
+	public void configure();
+
+	/**
+	 *  Attempt to expand a file.  The returned result is <code>true</code> if the
+	 *  file was actually expanded.
+	 *
+	 *  @param <code>DocumentID</code> the identity code of the document
+	 *  @param <code>DocumentInterface</code> the document itself
+	 *
+	 *  @return <code>boolean</code> whether the file was validated.
+	 */
+	public boolean validateDocument(DocumentID docID, DocumentInterface document);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java	(revision 14150)
@@ -0,0 +1,59 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+public class IdentifierXPointer extends XPointer
+{ NodeList nodes;
+
+  public IdentifierXPointer(Document document, String xpointer)
+  { super(document, xpointer);
+
+    this.xpointer = XPointer.expandXPath(this.xpointer);
+
+    this.nodes = this.getNodes();
+  }
+
+  private NodeList getNodes()
+  {
+    try {
+      NodeList nodes = XPathAPI.selectNodeList(this.document, this.xpointer, 
+					       this.document.getDocumentElement());
+      return nodes;
+    }
+    catch (Exception ex)
+    { System.out.println(ex.toString());
+    }
+    return null;
+  }
+
+  public Node getStartNode()
+  { return this.nodes.item(0);
+  }
+
+  private void recurseStringify(NodeList nodes, StringBuffer buffer)
+  { 
+    for (int n = 0; n < nodes.getLength(); n ++) {
+      Node node = nodes.item(n);
+
+      // append the actual node itself
+      this.printNode(node, buffer, false);
+
+      // then process its children...
+      if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	NodeList children = node.getChildNodes();
+	this.recurseStringify(children, buffer);
+      }
+
+      // do the close node
+      this.printNode(node, buffer, true);
+    }
+  }
+
+  public String toString()
+  {
+    StringBuffer reply = new StringBuffer();
+    this.recurseStringify(this.nodes, reply);
+    return reply.toString();
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java	(revision 14150)
@@ -0,0 +1,69 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+public class RangeToXPointer extends XPointer
+{
+  public static final String RANGE_TO = "range-to(";
+  NodeList startNodes;
+  NodeList endNodes;
+
+  public RangeToXPointer(Document document, String xpointer)
+  { super(document, xpointer);
+
+    String fromPointer, toPointer;
+
+    int rangeAt = xpointer.indexOf(RANGE_TO);
+
+    fromPointer = xpointer.substring(0, rangeAt-1);
+    toPointer = xpointer.substring(rangeAt+RANGE_TO.length(), xpointer.length()-1);
+
+    fromPointer = XPointer.expandXPath(fromPointer);
+    toPointer   = XPointer.expandXPath(toPointer);
+    //    System.out.println(fromPointer + " " + toPointer);
+    
+    this.startNodes = this.getNodes(fromPointer);
+    this.endNodes = this.getNodes(toPointer);
+    
+    //    System.out.println(this.startNodes.item(0).toString() + " " + this.endNodes.item(0).toString());
+  }
+
+  public Node getStartNode()
+  { return this.startNodes.item(0);
+  }
+
+  private NodeList getNodes(String pointer)
+  {
+    try {
+      NodeList nodes = XPathAPI.selectNodeList(document, pointer, document.getDocumentElement());
+      return nodes;
+    }
+    catch (Exception ex)
+    { System.out.println(ex.toString());
+    }
+    return null;
+  }
+
+  private void recurseStringify(Node at, Node end, StringBuffer reply)
+  {
+    while (at != end && at != null) {
+      this.printNode(at, reply, false);
+      at = this.getNextNode(at, reply);
+      //      System.out.println(at + " " + end);
+    }
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    // skip the first node
+    Node firstNode = this.startNodes.item(0);
+    firstNode = this.getNextNode(firstNode, reply);
+    
+    // actually, we don't, as otherwise the indexed copy of 
+    // the document would be incomplete...
+    this.recurseStringify(this.startNodes.item(0), this.endNodes.item(0), reply);
+    return reply.toString();
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java	(revision 14150)
@@ -0,0 +1,281 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import java.io.File; 
+
+import java.net.URL;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocumentTools;
+import org.greenstone.gsdl3.gs3build.util.HTMLTidy;
+
+public abstract class XPointer
+{
+  Document document;  
+  String   xpointer;
+
+  public XPointer(org.w3c.dom.Document document, String xpointer)
+  { this.document = document;
+    this.xpointer = xpointer;
+    // now process the type of description
+  }
+
+  public static String expandXPath(String xpath)
+  {
+    int          dashAt;
+    String       pathElement;
+    StringBuffer expandedPtr = new StringBuffer();
+
+    if (xpath.startsWith("//")) {
+      xpath = xpath.substring(2);
+      expandedPtr.append("//");
+    }
+    else if (xpath.startsWith("/")) {
+      xpath = xpath.substring(1);
+      expandedPtr.append("/");
+    }
+
+    while (xpath.length() > 0) {
+      dashAt = xpath.indexOf('/');
+      if (dashAt >= 0) {
+	pathElement = xpath.substring(0, dashAt);
+	xpath = xpath.substring(dashAt+1);
+      }
+      else {
+	pathElement = xpath;
+	xpath = "";
+      }
+
+      int c = 0;
+      while (c < pathElement.length()) {
+	if (!Character.isDigit(pathElement.charAt(c))) 
+	  break;
+	c ++;
+      }
+
+      if (c == pathElement.length()) {
+	expandedPtr.append("*[");
+	expandedPtr.append(pathElement);
+	expandedPtr.append("]");
+      }
+      else {
+	expandedPtr.append(pathElement);
+      }
+
+      if (dashAt >= 0) {
+	expandedPtr.append('/');
+      }
+    }
+    xpath = expandedPtr.toString();
+
+    return xpath;
+  }
+
+  public static XPointer processXPointer(Document document, String xpointer)
+  { // do a "cheap and dirty" detection of range-to
+    if (xpointer.indexOf("range-to") >= 0) {
+      return new RangeToXPointer(document, xpointer);
+    } 
+    return new IdentifierXPointer(document, xpointer);
+  }
+
+  public static XPointer processXPointer(Document document, URL url)
+  { String reference = url.getRef();
+
+    reference.trim();
+    if (reference.startsWith("xpointer(")) {
+      reference = reference.substring(9, reference.length() - 1);
+    }
+    else {
+      return null;
+    }
+    //    System.out.println(reference);
+    
+    return processXPointer(document, reference);
+  }
+
+  public static void printNode(Node node, StringBuffer to, boolean close)
+  {
+    if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+      if (!close) {
+	to.append(node.getNodeValue());
+	to.append(" ");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+      if (!close) {
+	to.append("&");
+	to.append(node.getNodeValue());
+	to.append(";");
+	to.append(" ");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      if (!close) {
+	to.append("<!-- ");
+	to.append(node.getNodeValue());
+	to.append(" -->");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      boolean hasChildren = (node.getChildNodes().getLength() > 0);
+
+      if (close && !hasChildren) {
+	return;
+      }
+      to.append("<");
+      if (close) {
+	to.append("/");
+      }
+      to.append(node.getNodeName());
+
+      if (!close) {
+	NamedNodeMap attributes = ((Element) node).getAttributes();
+	for (int a = 0; a < attributes.getLength(); a ++) {
+	  Node attributeNode = attributes.item(a);
+	  to.append(" ");
+	  to.append(attributeNode.getNodeName());
+	  
+	  String value = attributeNode.getNodeValue();
+	  if (value != null && value.length() > 0) {
+	    to.append("=\"");
+	    to.append(value);
+	    to.append("\"");
+	  }
+	}
+      }
+
+      if (!hasChildren) {
+	to.append(" /");
+      }
+      to.append(">");
+      if (close && node.getNodeName()=="Section") {
+	  to.append((char) 3);
+      }
+    }
+  }
+
+    public static Node getNextNode(Node currentNode) {
+	
+	// Try to get the first valid child
+	NodeList children = currentNode.getChildNodes();
+	for (int c = 0; c < children.getLength(); c++) {
+	    Node childNode = children.item(c);
+	    
+	    if (childNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.COMMENT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.TEXT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+		return childNode;
+	    }
+	}	
+	
+	// now try a siblings and parents siblings
+	Node siblingNode = null;
+	do {
+	    siblingNode = currentNode.getNextSibling();
+	    if (siblingNode != null) {
+		// should we be checking type??
+		return siblingNode;
+	    }
+	    
+	    currentNode = currentNode.getParentNode();
+	} while (currentNode != null);
+	
+	return null;
+    }
+
+    
+  public static Node getNextNode(Node currentNode, StringBuffer buffer)
+  { Node reply;
+
+    // Try for children only the once...
+    NodeList children = currentNode.getChildNodes();
+    for (int c = 0; c < children.getLength(); c++) {
+      Node childNode = children.item(c);
+
+      if (childNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.COMMENT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.TEXT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+	return childNode;
+      }
+    }
+    
+    // Repeat looking for a sibling & going up until done...
+    do {
+      reply = currentNode.getNextSibling();
+      if (reply != null) {
+	return reply;
+      }
+      
+      currentNode = currentNode.getParentNode();
+      if (buffer != null && currentNode != null) {
+	printNode(currentNode, buffer, true);
+      }
+    } while (currentNode != null);
+
+    return reply;
+  }
+
+  public abstract Node getStartNode();
+
+
+  public static void main(String args[])
+  {	
+    try
+    { File file = new File(args[0]);
+
+      Document document;
+      if (args[0].endsWith(".html") || args[0].endsWith(".htm")) {
+	HTMLTidy tidier = new HTMLTidy(file);
+	document = tidier.getDocument();
+
+	HTMLDocumentTools docTools = new HTMLDocumentTools(document);
+	docTools.findSections();
+	System.out.println("Sections found");
+      }
+      else {
+	DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	DocumentBuilder builder = factory.newDocumentBuilder();
+	document = builder.parse(file);
+      }
+      
+      XPointer pointer = null;
+      if (args.length < 2) {
+      }
+      else if (args.length == 2) {
+	pointer = XPointer.processXPointer(document, args[1]);
+      }
+      else {
+	pointer = XPointer.processXPointer(document, args[1]+"/range-to("+args[2]+")");
+      }
+      if (pointer != null) {
+	System.out.println(pointer.toString());
+      }
+    }
+    catch (FactoryConfigurationError e) {
+      System.err.println(e);
+    }
+    catch (ParserConfigurationException ex) {
+      System.err.println(ex);
+    }
+    catch (SAXException ex) {
+      System.err.println(ex);
+    }
+    catch (java.io.IOException ex) {
+      System.err.println(ex);
+    }
+  }
+}
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 14150)
@@ -0,0 +1,218 @@
+/*
+ *    AbstractGS3DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Implements the generic retrieval and classifier services for GS3 
+ * collections.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractGS3DocumentRetrieve
+    extends AbstractDocumentRetrieve {
+    protected static final String INDEX_STEM_ELEM = "indexStem";
+
+    protected SQLQuery database = null;
+    protected String index_stem = null;
+
+
+    /** constructor */
+    protected AbstractGS3DocumentRetrieve()
+    {
+	this.database = new SQLQuery();
+	// set up a macro resolver
+    }
+    
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractGS3DocumentRetrieve...");
+
+	// the index stem is either specified in the config file or is "index"
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    System.err.println("GS3MGRetrieve.configure(): indexStem element not found, stem will default to 'index'");
+	    this.index_stem = "index";
+	}
+
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+    
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getTop(node_id);
+    }
+    
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	if (GS3OID.isDocTop(node_id)) { // hack
+	    node_id =  GS3OID.createOID(node_id, "1");
+	}
+	ArrayList documents = database.getDocumentChildren(node_id);
+	if (documents == null || documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+    
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata,
+				      ArrayList metadata_names)   
+	throws GSException {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getDocumentMetadata(node_id, metadata);
+		if (value != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null; // not implemented yet
+    }
+
+    
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+    }
+
+    protected String resolveRelativeLinks(String doc_content, String doc_id) {
+
+	String http_path  = this.site_http_address + "/collect/"+this.cluster_name;
+	doc_content = doc_content.replaceAll("_httpcollection_", http_path);
+	return doc_content;
+	
+    }
+
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id) throws GSException;
+
+}   
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractMGPPSearch.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractMGPPSearch.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractMGPPSearch.java	(revision 14150)
@@ -0,0 +1,779 @@
+/*
+ *    AbstractMGPPSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.util.Iterator;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.Vector;
+import java.io.File;
+
+import org.apache.log4j.*;
+
+/** Partially implements a generic MGPP search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+abstract public class AbstractMGPPSearch
+    extends AbstractSearch
+{
+
+      static Category logger = Category.getInstance(org.greenstone.gsdl3.service.AbstractMGPPSearch.class.getName());
+
+    // extra services offered by mgpp collections
+    private static final String FIELD_QUERY_SERVICE = "FieldQuery";
+    private static final String ADVANCED_FIELD_QUERY_SERVICE = "AdvancedFieldQuery";
+
+    // extra parameters used
+    // private static final String INDEX_FIELD_PARAM = "index";
+    private static final String LEVEL_PARAM = "level";
+    private static final String RANK_PARAM = "sortBy";
+    private static final String RANK_PARAM_RANK = "1";
+    private static final String RANK_PARAM_NONE = "0";
+    private static final String SIMPLE_FIELD_PARAM = "simpleField";
+    private static final String ADVANCED_FIELD_PARAM = "complexField";
+
+    // more params for field query
+    private static final String FIELD_QUERY_PARAM = "fqv";
+    private static final String FIELD_STEM_PARAM = "fqs";
+    private static final String FIELD_CASE_PARAM = "fqc";
+    private static final String FIELD_ACCENT_PARAM="fqa";
+    private static final String FIELD_FIELD_PARAM = "fqf";
+    private static final String FIELD_COMBINE_PARAM = "fqk";
+    private static final String FIELD_COMBINE_PARAM_AND = "0";
+    private static final String FIELD_COMBINE_PARAM_OR = "1";
+    private static final String FIELD_COMBINE_PARAM_NOT = "2";
+   
+    private static final String ACCENT_PARAM="accent";
+
+    // some stuff for config files
+    private static final String SEARCH_TYPE_ELEM = "searchType";
+    private static final String SEARCH_TYPE_PLAIN = "plain";
+    private static final String SEARCH_TYPE_FORM = "form";
+    private static final String SEARCH_TYPE_FORM_SIMPLE = "simple";
+    private static final String SEARCH_TYPE_FORM_ADVANCED = "advanced";
+    protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    protected static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+    protected static final String LEVEL_ELEM = "level";
+
+    protected static final String STEMINDEX_OPTION = "stemIndexes";
+    protected static final String MAXNUMERIC_OPTION = "maxnumeric";
+
+    protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+    protected static final String STEM_ATT = "stem";
+    protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+    protected static final String FREQ_ATT = "freq";
+
+    private static final int TEXT_QUERY = 0;
+    private static final int SIMPLE_QUERY = 1;
+    private static final int ADVANCED_QUERY = 2;
+
+    protected static final String FIELD_ATT = "field";
+    private MGPPWrapper mgpp_src=null;
+
+   // the default level for retrieval - and we'll use it for searching too
+    private String default_level=null;
+    // the default field for searching
+    private String default_field = null;
+    // which search services will we offer??
+    private boolean plain_search = false;
+    private boolean simple_form_search = false;
+    private boolean advanced_form_search = false;
+
+    // stem indexes available
+    private boolean does_case=true;
+    private boolean does_stem=true;
+    private boolean does_accent=false;
+    
+    // maxnumeric - not used yet. needs to be passed to MGPPWrapper.
+    private int maxnumeric = 4;
+
+    /** the stem used for the index files */
+    protected String index_stem = null;
+
+    public AbstractMGPPSearch()
+    {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (super.configure(info, extra_info) == false)
+	    return false;
+	
+	// Do specific configuration
+	logger.info("Configuring AbstractMGPPSearch...");
+
+	// do we support any of the extended features?
+	does_chunking = true;
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	} // otherwise its "" and will be the first one in the list
+	
+	// the index stem is either specified in the config file or is "index"
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    logger.warn("AbstractMGPPSearch.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+
+	// get index options
+	Element index_option_list = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_OPTION_ELEM + GSXML.LIST_MODIFIER);
+	if (index_option_list != null) {
+	    NodeList options = index_option_list.getElementsByTagName(GSXML.INDEX_OPTION_ELEM);
+	    for (int i=0; i<options.getLength(); i++) {
+		Element opt = (Element)options.item(i);
+		String name = opt.getAttribute(GSXML.NAME_ATT);
+		String value = opt.getAttribute(GSXML.VALUE_ATT);
+		if (name.equals(MAXNUMERIC_OPTION)) {
+		    int maxnum = Integer.parseInt(value);
+		    if (4 <= maxnum && maxnum < 512) {
+			maxnumeric = maxnum;
+		    }
+		}
+		else if (name.equals(STEMINDEX_OPTION)) {
+		    int stemindex = Integer.parseInt(value);
+		    // stem and case are true by default, accent folding false by defualt
+		    if ((stemindex & 1) == 0) {
+			does_case = false;
+		    }
+		    if ((stemindex & 2) == 0) {
+			does_stem = false;
+		    }
+		    if ((stemindex & 4) != 0) {
+			does_accent = true;
+		    }
+		}
+	    }
+	}
+
+	// the generic config has set up the text query service, but we may not want it
+	Element search_type_list = (Element) GSXML.getChildByTagName(info, SEARCH_TYPE_ELEM + GSXML.LIST_MODIFIER);
+	if (search_type_list == null) {
+	    // assume form and plain
+	    this.plain_search = true;
+	    this.simple_form_search = true;
+	    this.advanced_form_search = true;
+	} else {
+	    NodeList types = search_type_list.getElementsByTagName(SEARCH_TYPE_ELEM);
+	    for (int i=0; i<types.getLength(); i++) {
+		Element t = (Element)types.item(i);
+		String type_name = t.getAttribute(GSXML.NAME_ATT);
+		if (type_name.equals(SEARCH_TYPE_PLAIN)) {
+		    this.plain_search = true;
+		} else if (type_name.equals(SEARCH_TYPE_FORM)) {
+		    String type_type = t.getAttribute(GSXML.TYPE_ATT);
+		    if (type_type.equals("")) {
+			this.simple_form_search = true;
+			this.advanced_form_search = true;
+		    } else if (type_type.equals(SEARCH_TYPE_FORM_SIMPLE)) {
+			this.simple_form_search = true;
+		    } else if (type_type.equals(SEARCH_TYPE_FORM_ADVANCED)) {
+			this.advanced_form_search = true;
+			
+		    }
+		}
+	    }
+	}
+
+	if (!this.plain_search) {
+	    // need to remove the TextQuery service
+	    Element tq_service = GSXML.getNamedElement(short_service_info, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	    short_service_info.removeChild(tq_service);
+	    
+	}
+
+       
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    logger.error("default level not specified!");
+	    return false;
+	}
+
+	// the default level is also the level which gdbm is expecting
+	// this must not be overwritten
+	this.mgpp_src.setReturnLevel(this.default_level);
+	// return term info
+	this.mgpp_src.setReturnTerms(true);
+	// set the default - this may be overwritten by query params
+	this.mgpp_src.setQueryLevel(this.default_level);
+
+	// set up the extra services which are available for this collection
+	// check the config info - if there is no field list, then there is no fielded searching
+	
+	Element field_list = (Element) GSXML.getChildByTagName(info, GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	if (field_list==null) {
+	    // nothing more to do
+	    return true;
+	}
+
+	// the format info is the same for all services
+	Element format_info = (Element)format_info_map.get(TEXT_QUERY_SERVICE);
+
+	// find the default field - use the first one
+	Element first_field = (Element)GSXML.getChildByTagName(field_list, GSXML.FIELD_ELEM);
+	default_field = first_field.getAttribute(GSXML.SHORTNAME_ATT);
+	// else set up the fielded query services
+	
+	if (this.simple_form_search) {
+	    // set up short_service_info_ - for now just has id and type - name will be added in on teh fly
+	    Element fq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    fq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    fq_service.setAttribute(GSXML.NAME_ATT, FIELD_QUERY_SERVICE);
+	    this.short_service_info.appendChild(fq_service);
+
+	    if (format_info != null) {
+		this.format_info_map.put(FIELD_QUERY_SERVICE, format_info);
+	    }
+	}
+	
+	if (this.advanced_form_search) {
+	    Element afq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    afq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    afq_service.setAttribute(GSXML.NAME_ATT, ADVANCED_FIELD_QUERY_SERVICE);
+	    this.short_service_info.appendChild(afq_service);
+	    
+	    if (format_info != null) {
+		this.format_info_map.put(ADVANCED_FIELD_QUERY_SERVICE, format_info);
+	    }
+	}
+	return true;
+
+    }
+
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+	// should we check that the service is actually on offer? presumably we wont get asked for services that we haven't advertised previously.
+	
+	if (!service_id.equals(FIELD_QUERY_SERVICE) && !service_id.equals(ADVANCED_FIELD_QUERY_SERVICE)) {
+	    return super.getServiceDescription(service_id, lang, subset);
+	}
+	
+	
+	Element service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	service.setAttribute(GSXML.NAME_ATT, service_id);
+	if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service_id+".name", lang)));
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service_id+".submit", lang)));
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service_id+".description", lang)));
+	
+	}
+	if (subset == null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);	
+	    service.appendChild(param_list);
+	    if (service_id.equals(FIELD_QUERY_SERVICE)) {
+		
+		addCustomQueryParams(param_list, lang);
+		createParameter(MAXDOCS_PARAM, param_list, lang);
+		// create a multi param for the fields etc
+		// text box, field
+		Element multiparam = null;
+		Element param=null;
+		multiparam = GSXML.createParameterDescription(this.doc, SIMPLE_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+		multiparam.setAttribute("occurs", "4");
+		param_list.appendChild(multiparam);
+		
+		// the components
+		createParameter(FIELD_QUERY_PARAM, multiparam,  lang);
+		createParameter(FIELD_FIELD_PARAM, multiparam,  lang);
+		
+	    } else { 
+		createParameter(LEVEL_PARAM, param_list,  lang);
+		createParameter(RANK_PARAM, param_list, lang);
+		createParameter(MAXDOCS_PARAM, param_list,  lang);
+		
+		
+		// create a multi param for the fields etc
+		// text box, stem, case, field
+		
+		Element multiparam = null;
+		Element param=null;
+		
+		multiparam = GSXML.createParameterDescription(this.doc, ADVANCED_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+		multiparam.setAttribute("occurs", "4");
+		param_list.appendChild(multiparam);
+		
+		createParameter(FIELD_COMBINE_PARAM, multiparam, lang);
+		createParameter(FIELD_QUERY_PARAM, multiparam, lang);
+		if (this.does_case) {
+		    createParameter(FIELD_CASE_PARAM, multiparam, lang);
+		}
+		if (this.does_stem) {
+		    createParameter(FIELD_STEM_PARAM, multiparam, lang);
+		}
+		if (this.does_accent) {
+		    createParameter(FIELD_ACCENT_PARAM, multiparam, lang);
+		}
+		createParameter(FIELD_FIELD_PARAM, multiparam, lang);    
+		
+	    } 
+	}
+	return service;
+	
+    }
+
+    /** add in the mgpp specific params to TextQuery */
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	createParameter(LEVEL_PARAM, param_list, lang);
+	if (this.does_case){
+	    createParameter(CASE_PARAM, param_list, lang);
+	}
+	if (this.does_stem){
+	    createParameter(STEM_PARAM, param_list, lang);
+	}
+	if (this.does_accent){
+	    createParameter(ACCENT_PARAM, param_list, lang);
+	}
+	createParameter(MATCH_PARAM, param_list, lang);
+	createParameter(RANK_PARAM, param_list, lang);
+    }
+
+    /** create a param and add to the list */
+    protected void createParameter(String name, Element param_list, String lang) 
+    {
+	Element param = null;
+	if (name.equals(LEVEL_PARAM)) {
+	    ArrayList level_ids = new ArrayList();
+	    ArrayList level_names = new ArrayList();
+	    getLevelData(level_ids, level_names, lang);
+	    if (level_ids.size()>1) {
+		// the first one is the default
+		param = GSXML.createParameterDescription2(this.doc, LEVEL_PARAM, getTextString("param."+LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)level_ids.get(0), level_ids, level_names);
+	    }
+	    
+	} else if (name.equals(RANK_PARAM)) {
+	    String [] vals1 = {RANK_PARAM_RANK, RANK_PARAM_NONE };
+	    String [] vals1_texts = { getTextString("param."+RANK_PARAM+"."+RANK_PARAM_RANK, lang), getTextString("param."+RANK_PARAM+"."+RANK_PARAM_NONE, lang)};
+	    
+	    param = GSXML.createParameterDescription(this.doc, RANK_PARAM, getTextString("param."+RANK_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, RANK_PARAM_RANK, vals1, vals1_texts );
+	    
+	} else if (name.equals(FIELD_QUERY_PARAM)) {
+	    param = GSXML.createParameterDescription(this.doc, FIELD_QUERY_PARAM, getTextString("param."+FIELD_QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+	    
+	    
+	} else if (name.equals(FIELD_CASE_PARAM) || name.equals(FIELD_STEM_PARAM) || name.equals(FIELD_ACCENT_PARAM)) {
+	    String[] bool_ops = {"0", "1"};
+	    String[] bool_texts = {getTextString("param.boolean.off", lang, "AbstractSearch"),getTextString("param.boolean.on", lang, "AbstractSearch")}; 
+	    param = GSXML.createParameterDescription(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_BOOLEAN, BOOLEAN_PARAM_ON, bool_ops, bool_texts);
+	    
+	} else if (name.equals(FIELD_FIELD_PARAM)) {
+	    ArrayList fields = new ArrayList();
+	    ArrayList field_names = new ArrayList();
+	    getIndexData(fields, field_names, lang);
+	    // the field list -  read from config file
+	    param = GSXML.createParameterDescription2(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)fields.get(0), fields, field_names );
+	    
+	} else if (name.equals(FIELD_COMBINE_PARAM)) {
+	    
+	    String []vals = {FIELD_COMBINE_PARAM_AND, FIELD_COMBINE_PARAM_OR, FIELD_COMBINE_PARAM_NOT};
+	    String []val_texts = {getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_AND, lang), getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_OR, lang), getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_NOT, lang)}; 
+		
+	    
+	    param = GSXML.createParameterDescription(this.doc, FIELD_COMBINE_PARAM, "", GSXML.PARAM_TYPE_ENUM_SINGLE, FIELD_COMBINE_PARAM_AND, vals, val_texts);
+	    param.setAttribute(GSXML.PARAM_IGNORE_POS_ATT, "0");
+	}
+
+	if (param != null) {
+	    param_list.appendChild(param);
+	} else {
+	    super.createParameter(name, param_list, lang);
+	}
+    }
+    
+    // should cache some of this
+    protected void getLevelData(ArrayList level_ids, ArrayList level_names, String lang) 
+    {
+	Element level_list = (Element)GSXML.getChildByTagName(this.config_info, LEVEL_ELEM+GSXML.LIST_MODIFIER);
+	NodeList levels = level_list.getElementsByTagName(LEVEL_ELEM);
+	for (int i=0; i<levels.getLength(); i++) {
+	    level_ids.add(((Element)levels.item(i)).getAttribute(GSXML.NAME_ATT));
+	    level_names.add(getTextString("level."+level_ids.get(i), lang));
+	}
+	
+    }
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang){
+	// we currently only have one index, and use the field data as the index
+		    
+	// the field list -  read from config file
+	Element field_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	NodeList fields = field_list.getElementsByTagName(GSXML.FIELD_ELEM);
+	for (int i=0; i< fields.getLength();i++) {
+	    String shortname = ((Element)fields.item(i)).getAttribute(GSXML.SHORTNAME_ATT);
+	    String name = ((Element)fields.item(i)).getAttribute(GSXML.NAME_ATT);
+	    if (name.equals("")) {
+		// no name, ignore
+		continue;
+	    }
+	    // TODO change field so that name is the id, and full metadata name is somthing else
+	    if (shortname.equals("")) {
+		shortname = name;
+	    }
+	    index_ids.add(shortname);
+	    // should these be changed to a text element based on lang?
+	    // or is the name of a metadata element eg dc:Title its 
+	    // name in all langs
+	    index_names.add(name);
+	}
+    }
+
+    // the following three functions are needed so the base class can 
+    // call the process+SERVICE_NAME methods
+    /** process a text query */
+    protected Element processTextQuery(Element request) {
+	return processAnyQuery(request, TEXT_QUERY);
+    }
+
+    /** process a field query */    
+    protected Element processFieldQuery(Element request) {
+	return processAnyQuery(request, SIMPLE_QUERY);
+    }
+
+    /** process an advanced field query */
+    protected Element processAdvancedFieldQuery(Element request) {
+	return processAnyQuery(request, ADVANCED_QUERY);
+    }
+    
+    /** process a  query */
+    protected Element processAnyQuery(Element request, int query_type)
+    {
+
+	String service_name=null;
+	String empty_query_test_param=null;
+	// set up the type specific bits
+	switch (query_type) {
+	case TEXT_QUERY:
+	    service_name = TEXT_QUERY_SERVICE; 
+	    empty_query_test_param = QUERY_PARAM;
+	    break;
+	case SIMPLE_QUERY:
+	    service_name = FIELD_QUERY_SERVICE;
+	    empty_query_test_param = FIELD_QUERY_PARAM;
+	    break;
+	case ADVANCED_QUERY:
+	    service_name = ADVANCED_FIELD_QUERY_SERVICE;
+	    empty_query_test_param = FIELD_QUERY_PARAM;
+	    break;
+	default:
+	    // should never get here
+	    logger.error("wrong query type!!");
+	    return null;
+	}
+	
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, service_name);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(empty_query_test_param);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If a field hasn't been specified, use the default - for textQuery
+	String field = (String) params.get(INDEX_PARAM); 
+	if (field == null) {
+	    field = default_field;
+	}
+	
+	// set up mgpp_src
+	String indexdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mgpp_src.loadIndexData(indexdir);
+	setStandardQueryParams(params);
+	
+	// if field search, create the query string
+	switch (query_type) {
+	case TEXT_QUERY:
+	    query = addFieldInfo(query, field);
+	    break;
+	case SIMPLE_QUERY:
+	    query = parseFieldQueryParams(params);
+	    break;
+	case ADVANCED_QUERY:
+	    query = parseAdvancedFieldQueryParams(params);
+	    break;
+	}
+
+	// run the query
+       	this.mgpp_src.runQuery(query);
+	MGPPQueryResult mqr= this.mgpp_src.getQueryResult();
+
+	// build up the response
+	
+	// Create a metadata list to store information about the query results
+	// should we be using metadataList? or something else?
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	long totalDocs = mqr.getTotalDocs();
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+totalDocs);
+
+	// Create a document list to store the matching documents, and add them
+	Vector docs = mqr.getDocs();
+
+	// add a metadata item to specify docs returned
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+docs.size());
+	// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+	GSXML.addMetadata(this.doc, metadata_list, "query", query);
+	if (docs.size()>0) {
+	    Element document_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    result.appendChild(document_list);
+	    for (int d = 0; d < docs.size(); d++) {
+		long docnum = ((MGPPDocInfo) docs.elementAt(d)).num_;
+		float rank = ((MGPPDocInfo) docs.elementAt(d)).rank_;
+		String doc_id = MGPPNum2OID(docnum);
+		Element doc_node = createDocNode(doc_id, Float.toString(rank));
+		//doc_node.setAttribute("rank", Float.toString(rank));
+		document_list.appendChild(doc_node);
+	    }
+	}
+	// Create a term list to store the term information, and add it
+	String query_level = (String)params.get(LEVEL_PARAM); // the current query level
+	Element term_list = this.doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);
+	Vector terms = mqr.getTerms();
+	for (int t = 0; t < terms.size(); t++) {
+	    MGPPTermInfo term_info = (MGPPTermInfo) terms.get(t);
+	    
+	    Element term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+	    term_elem.setAttribute(GSXML.NAME_ATT, term_info.term_);
+	    term_elem.setAttribute(STEM_ATT, "" + term_info.stem_method_);
+	    term_elem.setAttribute(FREQ_ATT, "" + term_info.term_freq_);
+	    term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + term_info.match_docs_);
+	    field = term_info.tag_;
+	    if (field.equals(query_level)) {
+		// ignore
+		field = "";
+	    }
+	    term_elem.setAttribute(FIELD_ATT, field);
+	    
+	    Vector equiv_terms = term_info.equiv_terms_;
+	    Element equiv_term_list = this.doc.createElement(EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+	    term_elem.appendChild(equiv_term_list);
+	    
+	    for (int et = 0; et < equiv_terms.size(); et++) {
+		String equiv_term = (String) equiv_terms.get(et);
+
+		Element equiv_term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+		equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term);
+		equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "");
+		equiv_term_elem.setAttribute(FREQ_ATT, "");
+		equiv_term_list.appendChild(equiv_term_elem);
+	    }
+
+	    term_list.appendChild(term_elem);
+	}
+
+	return result;
+    }
+
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap params) {
+
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+	   
+	    if (name.equals(CASE_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON)?true:false);
+		this.mgpp_src.setCase(val);		
+	    } else if (name.equals(STEM_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON)?true:false);
+		this.mgpp_src.setStem(val);
+	    } else if (name.equals(MAXDOCS_PARAM)&& !value.equals("")) {
+		int docs = Integer.parseInt(value);
+		this.mgpp_src.setMaxDocs(docs);
+	    } else if (name.equals(LEVEL_PARAM)) {
+		this.mgpp_src.setQueryLevel(value);
+	    } else if (name.equals(MATCH_PARAM)) {
+		int mode;
+		if (value.equals(MATCH_PARAM_ALL)) mode=1;
+		else mode=0;
+		this.mgpp_src.setMatchMode(mode);
+	    } else if (name.equals(RANK_PARAM)) {
+		if (value.equals(RANK_PARAM_RANK)) {
+		    this.mgpp_src.setSortByRank(true);
+		} else if (value.equals(RANK_PARAM_NONE)) {
+		    this.mgpp_src.setSortByRank(false);
+		}
+	    } // ignore any others
+	}
+	return true;
+    }
+
+    protected String addFieldInfo(String query, String field) {
+	if (field.equals("") || field.equals("ZZ")) {
+	    return query;
+	}
+	return "["+query+"]:"+field;
+    }
+    /** combines all the field params into a single query
+     * - for simple field query */
+    protected String parseFieldQueryParams(HashMap params) {
+
+	StringBuffer final_query = new StringBuffer(256);
+	String text_line = (String)params.get(FIELD_QUERY_PARAM);
+	String[] texts = text_line.split(",", -1);
+	String field_line = (String)params.get(FIELD_FIELD_PARAM);
+	String[] fields = field_line.split(",", -1);
+	String combine="&";
+	String match = (String)params.get(MATCH_PARAM);
+	if (match.equals(MATCH_PARAM_SOME)) {
+	    combine = "|";
+	}
+	
+	for (int i=0; i<texts.length; i++) {
+
+	    String q = texts[i].trim();
+	    if (!q.equals("")) {
+		addQueryElem(final_query, q, fields[i], combine);
+	    }
+	}
+
+	return final_query.toString();
+    }
+
+    /** combines all the field params into a single query
+     * - for advanced field query */   
+    protected String parseAdvancedFieldQueryParams(HashMap params) {
+
+	StringBuffer final_query = new StringBuffer(256);
+	String text_line = (String)params.get(FIELD_QUERY_PARAM);
+	String[] texts = text_line.split(",", -1);
+	String field_line = (String)params.get(FIELD_FIELD_PARAM);
+	String[] fields = field_line.split(",", -1);
+	String case_line = (String)params.get(FIELD_CASE_PARAM);
+	String[] cases = case_line.split(",", -1);
+	String stem_line = (String)params.get(FIELD_STEM_PARAM);
+	String[] stems = stem_line.split(",", -1);
+	String  combine_line = (String)params.get(FIELD_COMBINE_PARAM);
+	String [] combines = combine_line.split(",", -1);
+	String combine = "&";
+	for (int i=0; i<texts.length; i++) {
+	    if (i==0) {// assume first one is blank
+		combine = "";
+	    } else {
+		String x = combines[i]; 
+		if (x.equals(FIELD_COMBINE_PARAM_AND)) {
+		    combine = "&";
+		} else if (x.equals(FIELD_COMBINE_PARAM_OR)) {
+		    combine = "|";
+		} else if (x.equals(FIELD_COMBINE_PARAM_NOT)) {
+		    combine = "!";
+		} 
+		
+	    }
+
+	    String q = texts[i].trim();
+	    if (!q.equals("")) {
+		q = addStemAndCase(q, stems[i], cases[i]);
+		addQueryElem(final_query, q, fields[i], combine);
+	    }
+	}
+
+	return final_query.toString();
+    }
+
+    protected void addQueryElem(StringBuffer s, String q, String f, String c) {
+
+	String combine="";
+	if (s.length()>0) {
+	    combine = " "+c+" ";
+	}
+	if (f.equals("")||f.equals("ZZ")) {
+	    s.append(combine+q);
+	} else {
+	    s.append(combine+"["+q+"]:"+f);
+	}
+    }
+    
+    protected String addStemAndCase(String q, String s, String c) {
+	String mods = "#";
+	if (c.equals("1")) {
+	    mods += "i";
+	} else {
+	    mods += "c";
+	}
+	if (s.equals("1")) {
+	    mods += "s";
+	} else {
+	    mods+= "u";
+	}
+	StringBuffer temp = new StringBuffer();
+	String [] terms = q.split(" ");
+	for (int i=0; i<terms.length; i++) {
+	    String t = terms[i].trim();
+	    
+	    if (!t.equals("") && !t.equals("TX")) {
+		temp.append(" "+t+mods);
+	    }
+	}
+	return temp.toString();
+    }
+
+    /** convert MGPP internal id to Greenstone oid */
+    abstract protected String MGPPNum2OID(long docnum);
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractMGSearch.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractMGSearch.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/AbstractMGSearch.java	(revision 14150)
@@ -0,0 +1,311 @@
+/*
+ *    AbstractMGSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java
+import java.util.Vector;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.Iterator;
+import java.io.File;
+
+import org.apache.log4j.*;
+
+/** Partially implements a generic MG search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+abstract public class AbstractMGSearch
+    extends AbstractSearch
+{
+
+    static Category logger = Category.getInstance(org.greenstone.gsdl3.service.AbstractMGSearch.class.getName());
+    protected static final String CASE_PARAM = "case";
+    protected static final String STEM_PARAM = "stem";
+    protected static final String MATCH_PARAM = "matchMode";
+    protected static final String MATCH_PARAM_ALL = "all";
+    protected static final String MATCH_PARAM_SOME = "some";
+
+    protected static final String BOOLEAN_PARAM_ON = "1";
+    protected static final String BOOLEAN_PARAM_OFF = "0";
+
+    protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+    protected static final String STEM_ATT = "stem";
+    protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+    protected static final String FREQ_ATT = "freq";
+
+    // Elements used in the config file that are specific to this class
+    protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    protected static final String INDEX_STEM_ELEM = "indexStem";
+    protected static final String INDEX_ELEM = "index";
+
+    /** the stem used for the index files */
+    protected String index_stem = null;
+    protected MGWrapper mg_src = null;
+
+    
+    public AbstractMGSearch()
+    {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	// do we support any of the extended features?
+	does_chunking = true;
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	} // otherwise will be "", and the first one will be the default
+
+	// the index stem is either the collection name or is specified in the config file
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    logger.error("AbstractMGSearch.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+	
+	// get display info from extra info
+	if (extra_info !=null) {
+	    Document owner = info.getOwnerDocument();
+	    // so far we have index specific display elements, and global format elements 
+	    NodeList indexes = info.getElementsByTagName(GSXML.INDEX_ELEM);
+	    Element config_search = (Element)GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+	    
+	    for (int i=0; i<indexes.getLength();i++) {
+		Element ind = (Element)indexes.item(i);
+		String name = ind.getAttribute(GSXML.NAME_ATT);
+		Element node_extra = GSXML.getNamedElement(config_search,
+							   GSXML.INDEX_ELEM,
+							   GSXML.NAME_ATT,
+							   name);
+		if (node_extra == null) {
+		    logger.error("haven't found extra info for index named "+name);
+		    continue;
+		}
+		
+		// get the display elements if any - displayName
+		NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		if (display_names !=null) {
+		    for (int j=0; j<display_names.getLength(); j++) {
+			Element e = (Element)display_names.item(j);
+			ind.appendChild(owner.importNode(e, true));
+		    }
+		}
+	    } // for each index
+	}
+	return true;
+    }
+
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	createParameter(CASE_PARAM, param_list, lang);
+	createParameter(STEM_PARAM, param_list, lang);
+	createParameter(MATCH_PARAM, param_list, lang);	
+    }
+
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang) {
+	// the index info -
+	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, INDEX_ELEM+GSXML.LIST_MODIFIER);
+	NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+	int len = indexes.getLength();
+	// now add even if there is only one
+	for (int i=0; i<len; i++) {
+	    Element index = (Element)indexes.item(i);
+	    index_ids.add(index.getAttribute(GSXML.NAME_ATT));
+	    index_names.add(GSXML.getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+		
+	}
+
+    }
+    /** do the actual query */
+    protected Element processTextQuery(Element request) 
+    {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If an index hasn't been specified, use the default
+	String index = (String) params.get(INDEX_PARAM); 
+	if (index == null) {
+	    index = this.default_index;
+	}
+
+	// The location of the MG index and text files
+	String basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +  File.separatorChar;  // Needed for MG
+	String textdir = GSFile.collectionTextPath(this.index_stem);
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, index);
+	this.mg_src.setIndex(indexpath);
+	
+	// set the mg query parameters to the values the user has specified
+	setStandardQueryParams(params);
+	this.mg_src.runQuery(basedir, textdir, query);
+	MGQueryResult mqr = this.mg_src.getQueryResult();
+	if (mqr.isClear()) {
+	    // something has gone wrong
+	    GSXML.addError(this.doc, result, "Couldn't query the mg database", GSXML.ERROR_TYPE_SYSTEM);
+	    return result;
+	}
+	long totalDocs = mqr.getTotalDocs();
+
+	// Get the docnums out, and convert to HASH ids
+	Vector docs = mqr.getDocs();
+	if (docs.size() == 0) {
+	    logger.error("No results found...\n");
+	}
+
+	// Create a metadata list to store information about the query results
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+
+	// Add a metadata element specifying the number of matching documents
+	// because teh total number is just the number returned, use numDocsReturned, not numDocsMatched
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+totalDocs);
+	// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+	GSXML.addMetadata(this.doc, metadata_list, "query", query);
+
+	if (docs.size() > 0) {
+	    // Create a document list to store the matching documents, and add them
+	    Element document_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    result.appendChild(document_list);
+	    for (int d = 0; d < docs.size(); d++) {
+		long docnum = ((MGDocInfo) docs.elementAt(d)).num_;
+		float rank = ((MGDocInfo) docs.elementAt(d)).rank_;
+		String doc_id = MGNum2OID(docnum);
+		Element doc_node = createDocNode(doc_id, Float.toString(rank));
+		document_list.appendChild(doc_node);
+	    }
+	}
+	
+	// Create a term list to store the term information, and add it
+	Element term_list = this.doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);
+	Vector terms = mqr.getTerms();
+	for (int t = 0; t < terms.size(); t++) {
+	    MGTermInfo term_info = (MGTermInfo) terms.get(t);
+
+	    String term = term_info.term_;
+	    int stem_method = term_info.stem_method_;
+	    Vector equiv_terms = term_info.equiv_terms_;
+
+	    Element term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+	    term_elem.setAttribute(GSXML.NAME_ATT, term);
+	    term_elem.setAttribute(STEM_ATT, "" + stem_method);
+
+	    Element equiv_term_list = this.doc.createElement(EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+	    term_elem.appendChild(equiv_term_list);
+
+	    long total_term_freq = 0;
+	    for (int et = 0; et < equiv_terms.size(); et++) {
+		MGEquivTermInfo equiv_term_info = (MGEquivTermInfo) equiv_terms.get(et);
+
+		Element equiv_term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+		equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term_info.term_);
+		equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + equiv_term_info.match_docs_);
+		equiv_term_elem.setAttribute(FREQ_ATT, "" + equiv_term_info.term_freq_);
+		equiv_term_list.appendChild(equiv_term_elem);
+
+		total_term_freq += equiv_term_info.term_freq_;
+	    }
+
+	    term_elem.setAttribute(FREQ_ATT, "" + total_term_freq);
+	    term_list.appendChild(term_elem);
+	}
+	return result;
+    }
+
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap params)
+    {
+	// set the default ones
+	this.mg_src.setReturnTerms(true);
+	this.mg_src.setCase(true); // turn casefolding on by default
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+
+	    if (name.equals(CASE_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setCase(val);
+	    }
+	    else if (name.equals(STEM_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setStem(val);
+	    }
+	    else if (name.equals(MATCH_PARAM)) {
+		int mode = (value.equals(MATCH_PARAM_ALL) ? 1 : 0);
+		this.mg_src.setMatchMode(mode);
+	    }
+	    else if (name.equals(MAXDOCS_PARAM)) {
+		int docs = Integer.parseInt(value);
+		this.mg_src.setMaxDocs(docs);
+	    } // ignore any others
+	}
+	return true;
+    }
+
+    /** convert MG internal id to Greenstone oid */
+    abstract protected String MGNum2OID(long docnum);
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3Browse.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3Browse.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3Browse.java	(revision 14150)
@@ -0,0 +1,193 @@
+/*
+ *    GS3Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+import java.io.File;
+
+/** Greenstone 3 collection classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+public class GS3Browse
+    extends AbstractBrowse
+{
+
+    protected SQLQuery database = null;
+
+    /** constructor */
+    public GS3Browse()
+    {
+	this.database = new SQLQuery();
+
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+    
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.err.println("Configuring GS3Browse...");
+	
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+	
+	return true;
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+
+        /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+   }
+    
+    /** returns the id of the root node of the document containing node 
+     * node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getDocID(node_id);
+    }
+    
+   /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	ArrayList documents = database.getClassifierDocChildren(node_id);
+	if (documents == null) {
+	    documents = new ArrayList();
+	}
+	ArrayList children = database.getClassifierChildren(node_id);
+	if (children != null) {
+	    documents.addAll(children);
+	}
+	if (documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+    
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata, ArrayList metadata_names) {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getClassifierMetadata(node_id, metadata);
+		if (value != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null;
+    }
+
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    protected boolean isDocumentId(String node_id){
+	if (node_id.startsWith("CL")) {
+	    return false;
+	}
+	return true;
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 14150)
@@ -0,0 +1,121 @@
+/*
+ *    GS3MGPPRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGPPRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+
+    private MGPPWrapper mgpp_src = null;
+    
+    private String default_level = null;
+    private String mgpp_textdir = null;
+	
+    public GS3MGPPRetrieve() {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS3MGPPRetrieve...");
+
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    System.err.println("Error: default level not specified!");
+	    return false;
+	}
+	// System.out.println("Default level: " + default_level_);
+	
+	// The location of the MGPP text files
+	mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionTextPath(this.index_stem);
+
+	return true;
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mgpp_src.getDocument(this.mgpp_textdir, 
+						    this.default_level,
+						    doc_int);
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+	} catch (Exception e) {
+	    System.out.println("exception happended with mgpp_src.getDocument()" + e);
+	    doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_int+"\n";
+	}
+
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPSearch.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPSearch.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPSearch.java	(revision 14150)
@@ -0,0 +1,141 @@
+/*
+ *    GS3MGPPSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.io.File;
+
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public class GS3MGPPSearch 
+    extends AbstractMGPPSearch 
+{
+    protected SQLQuery database = null;
+    /** the base index prefix */
+    protected String base_index_prefix = null;
+
+
+    /** constructor */
+    public GS3MGPPSearch()
+    {
+	this.database = new SQLQuery();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// get the base prefix
+	Element def = (Element) GSXML.getChildByTagName(info, "baseIndexPrefix");
+	if (def != null) {
+	    this.base_index_prefix = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.base_index_prefix == null || this.base_index_prefix.equals("")) {
+	    System.err.println("Error: base index prefix not specified!");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Search Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Search Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	return database.documentHasChildren(node_id);
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MGPP internal id to Greenstone oid */
+    protected String MGPPNum2OID(long docnum)
+    {
+	String mg_id = this.base_index_prefix+"."+docnum;
+	String doc_id = database.MGNum2OID(mg_id);
+	// hack !!!
+	if (doc_id.endsWith("-All")) {
+	    // get rid of the All bit
+	    doc_id = doc_id.substring(0,doc_id.length()-4);
+	    //doc_id=doc_id.replaceAll("All","1");
+	}
+	return doc_id;
+	
+    }
+
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGRetrieve.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 14150)
@@ -0,0 +1,133 @@
+/*
+ *    GS3MGRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+
+    private MGWrapper mg_src = null;
+    private String mg_basedir = null;
+    private String mg_textdir = null;
+    private String default_index = null;
+    
+    public GS3MGRetrieve() {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS3MGRetrieve...");
+	// System.out.println("info:\n" + converter_.getString(info));
+	// System.out.println("extra_info:\n" + converter_.getString(extra_info));
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_index == null || this.default_index.equals("")) {
+	    System.err.println("Error: default index not specified!");
+	    return false;
+	}
+
+	// System.out.println("Default index: " + this.default_index);
+
+	
+	// The location of the MG index and text files
+	mg_basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+	mg_textdir = GSFile.collectionTextPath(this.index_stem);
+	// index is only needed to start up MG, not used so just use the default index
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mg_src.setIndex(indexpath);
+	return true;
+    }
+    
+    /** returns the content of a node.
+     * node_id should already have been translated if necessary
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mg_src.getDocument(this.mg_basedir, 
+						  this.mg_textdir, 
+						  doc_int);
+	    // remove any ctrl-c or ctrl-b 
+	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+
+	} catch (Exception e) {
+	    System.out.println("exception happended with mg_src.getDocument()");
+	    throw new GSException("Couldn't get document content for id: "+ doc_int+"\n"+e.getMessage());
+	    //doc_content = "this is the content for section hash id "+ doc_id+", mg doc num "+doc_int+"\n";
+	}
+	
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGSearch.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGSearch.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/service/GS3MGSearch.java	(revision 14150)
@@ -0,0 +1,141 @@
+/*
+ *    GS3MGSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.io.File;
+
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public class GS3MGSearch 
+    extends AbstractMGSearch 
+{
+    protected SQLQuery database = null;
+    /** the base index prefix */
+    protected String base_index_prefix = null;
+
+
+    /** constructor */
+    public GS3MGSearch()
+    {
+	this.database = new SQLQuery();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// get the base prefix
+	Element def = (Element) GSXML.getChildByTagName(info, "baseIndexPrefix");
+	if (def != null) {
+	    this.base_index_prefix = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.base_index_prefix == null || this.base_index_prefix.equals("")) {
+	    System.err.println("Error: base index prefix not specified!");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Search Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Search Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	return database.documentHasChildren(node_id);
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MG internal id to Greenstone oid */
+    protected String MGNum2OID(long docnum)
+    {
+	String mg_id = this.base_index_prefix+"."+docnum;
+	String doc_id = database.MGNum2OID(mg_id);
+	// hack !!!
+	if (doc_id.endsWith("-All")) {
+	    // get rid of the All bit
+	    doc_id = doc_id.substring(0,doc_id.length()-4);
+	    //doc_id=doc_id.replaceAll("All","1");
+	}
+	return doc_id;
+	
+    }
+
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/GS3OID.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/GS3OID.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/GS3OID.java	(revision 14150)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.util;
+
+public class GS3OID extends OID {
+
+    /**  public static boolean isTop(String oid) {
+	if (oid.equals("")) {
+	    return false;
+	}
+	int pos = oid.indexOf('.');
+	if (pos == -1) { // its the top document, not the top structure
+	    return false;
+	}
+	pos = pod.indexOf('.', pos+1);
+	if (pos == -1) {
+	    return true; // there is only one dot
+	}
+	return false; // there is more than one dot
+		
+	}*/
+
+    public static boolean isDocTop(String oid) {
+	return oid.indexOf('-')==-1;
+    }
+
+    public static String createOID(String doc_id, String section_label) {
+	return doc_id + "-"+section_label;
+    }
+
+    public static String getDocID(String oid) {
+	if (isDocTop(oid)) {
+	    return oid;
+	}
+	return oid.substring(0, oid.indexOf('-'));
+    }
+    public static String getSectionLabel(String oid) {
+
+	if (isDocTop(oid)) {
+	    return "";
+	}
+	return oid.substring(oid.indexOf('-')+1);
+
+    }
+
+    // put this here for now - may need to go with SQL stuff later
+    public static String translateOID(String oid) {
+	
+	int p = oid.lastIndexOf('.');
+	if (p != oid.length()-3) {
+	    System.out.println("translateoid error: '.' is not the third to last char!!");
+	    return oid;
+	}
+	
+	String top = oid.substring(0, p);
+	String suff = oid.substring(p+1);
+	if (suff.equals("pr")) {
+	    return getParent(top);
+	} else if (suff.equals("rt")) {
+	    return getTop(top);
+	} 
+	return oid;
+
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/GSSQL.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/GSSQL.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/GSSQL.java	(revision 14150)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.util;
+
+public class GSSQL {
+
+    // table names
+    public static final String DOCUMENT_TABLE = "document";
+    public static final String STRUCTURE_TABLE = "structure";
+    public static final String DIVISION_TABLE = "divisions";
+    public static final String DIVISION_FILE_TABLE = "divisionfilerefs";
+    public static final String DIVISION_METADATA_TABLE = "divisionmetarefs";
+    public static final String FILE_SECTION_TABLE = "filesection";
+    public static final String FILE_GROUP_TABLE = "filegroups";
+    public static final String FILE_TABLE = "files";
+    public static final String METADATA_TABLE = "metadata"; 
+    public static final String NAMESPACE_TABLE = "namespaces";
+    public static final String METADATA_VALUE_TABLE = "mdvalues";
+    public static final String CLASSIFIER_TABLE = "classifiers";
+    public static final String CLASS_DATA_TABLE = "classdata";
+    public static final String CLASS_DOCUMENT_TABLE = "classdocuments";
+
+    // unique key names - references to the tables
+    public static final String METADATA_REF = "MetadataRef";
+    public static final String NAMESPACE_REF = "NamespaceRef";
+    public static final String STRUCTURE_REF = "StructureRef";
+    public static final String DIVISION_REF = "DivisionRef";
+    public static final String FILE_SECTION_REF = "FileSectionRef";
+    public static final String FILE_GROUP_REF = "FileGroupRef";
+    public static final String CLASSIFIER_REF = "ClassifyRef";
+    
+    // combined refs
+    public static final String PARENT_REF = "ParentRef";
+    public static final String PARENT_TYPE = "ParentType";
+    
+    // id fields - these are unique ids per document - ie combined with docid
+    public static final String DOCUMENT_ID = "DocID";
+    public static final String METADATA_ID = "MetaID";
+    public static final String FILE_ID = "FileID";
+    public static final String STRUCTURE_ID = "StructureID";
+    public static final String SECTION_ID = "SectionID";
+    public static final String FILE_GROUP_ID = "FileID";
+    public static final String FILE_SECTION_ID = "FileID";
+    public static final String CLASSIFIER_ID = "ClassifyID";
+    public static final String PARENT_ID = "ParentID";
+    
+    // type fields
+    public static final String DOC_TYPE = "DocType";
+    public static final String NAMESPACE_TYPE = "NamespaceType";
+    public static final String STRUCTURE_TYPE = "StructureType";
+    public static final String DIVISION_TYPE = "DivisionType";
+    public static final String FILE_LOCATION_TYPE = "FileLocType";
+    public static final String MIME_TYPE = "MIMEType";
+
+    // other stuff
+    public static final String LABEL = "Label";
+    public static final String VALUE = "Value";
+    public static final String LABEL_ORDER = "LabelOrder";
+    public static final String SHORT_LABEL = "ShortLabel";
+    public static final String USER_LABEL = "UserLabel";
+    public static final String NAME = "Name";
+    public static final String DESCRIPTION = "Description";
+    public static final String FILE_LOCATION = "FileLocation";
+    public static final String CREATOR = "Creator";
+    public static final String DOCUMENT_ORDER = "DocOrder";
+    public static final String NUM_LEAF_DOCUMENTS = "NumLeafDocs";
+
+    public static final String COMMA = ",";
+    public static final String EQUALS = "=";
+    public static final String QUOTE = "'";
+    public static final String EQUALS_QUOTE = "='";
+    public static final String DOT = ".";
+    public static final String END = ";";
+    public static final String NOT="!";
+
+    // some values in the fields
+    public static final String STRUCTURE = "Structure";
+    public static final String DIVISION = "Division";
+    public static final String ALL = "All";
+    public static final String MG_NUM = "mgseqno";
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/SQLQuery.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/SQLQuery.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gs3build/src/org/greenstone/gsdl3/util/SQLQuery.java	(revision 14150)
@@ -0,0 +1,416 @@
+package org.greenstone.gsdl3.util;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnectionFactory;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import java.util.ArrayList;
+
+public class SQLQuery {
+    GS3SQLConnection connection = null;
+    
+    public SQLQuery() {
+
+    }
+
+    public boolean setDatabase(String db_name) {
+	connection = GS3SQLConnectionFactory.getGS3SQLConnection(db_name);
+	try {
+
+	    // test the connection
+	    String query = "select * from "+GSSQL.DOCUMENT_TABLE+GSSQL.END;
+	    Statement s = connection.createStatement();
+	    s.execute(query);
+	    s.close();
+	    return true;
+	} catch (Exception e) {
+	    System.err.println("SQLQuery.setDatabase():"+e);
+	}
+	closeConnection();
+	return false;
+    }
+
+    public void closeConnection() {
+	if (connection != null) {
+	    connection.close();
+	    connection=null;
+	}
+    }
+
+    public String MGNum2OID(String mg_num) {
+	// get doc id and meta id
+	String query = "select "+ 
+	    GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+
+	    GSSQL.COMMA+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+
+	    " from "+GSSQL.METADATA_VALUE_TABLE+GSSQL.COMMA+GSSQL.NAMESPACE_TABLE+GSSQL.COMMA+GSSQL.METADATA_TABLE+
+	    " where "+
+	    GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.LABEL+GSSQL.EQUALS_QUOTE+GSSQL.MG_NUM+GSSQL.QUOTE+
+	    " and "+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.VALUE+GSSQL.EQUALS_QUOTE+mg_num+GSSQL.QUOTE+
+	    " and "+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+GSSQL.EQUALS+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+
+	    " and "+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+GSSQL.EQUALS+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+
+	    GSSQL.END;
+	
+	Statement statement = null;
+	ResultSet results = null;
+	String doc_id = null;
+	String meta_id = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		doc_id = results.getString(GSSQL.DOCUMENT_ID);
+		meta_id = results.getString(GSSQL.METADATA_ID);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+	    System.err.println("SQLQuery.MGNum2OID(): "+e);
+	    return null;
+	}
+
+	// now get division label
+	query = "select "+
+	    GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.SHORT_LABEL+
+	    " from "+GSSQL.DIVISION_TABLE+GSSQL.COMMA+GSSQL.DIVISION_METADATA_TABLE+
+	    " where "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+GSSQL.EQUALS_QUOTE+meta_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+GSSQL.EQUALS+GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+
+	    GSSQL.END;
+
+	
+	String short_label = null;
+	try {
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		short_label = results.getString(GSSQL.SHORT_LABEL);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	    statement.close();
+	} catch (SQLException e) {
+
+	    System.err.println("SQLQuery.MGNum2OID() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return GS3OID.createOID(doc_id, short_label);
+    }
+
+    public String OID2MGNum(String oid) {
+	String id = getDocumentMetadata(oid, "gsdl3.mgseqno");
+	return id;
+
+    }
+
+    // does the document have sections, or is it just a single page document?
+    public boolean isHierarchicalDocument(String oid) {
+	// first just check the OID - it may already have section info in it
+	if (!GS3OID.isDocTop(oid)) {
+	    return true;
+	}
+
+	// now we need to check the database
+	String query = "select * "+ 
+	    " from "+GSSQL.STRUCTURE_TABLE +
+	    " where "+ GSSQL.DOCUMENT_ID +GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and "+ GSSQL.STRUCTURE_ID +GSSQL.EQUALS_QUOTE + "Section" + GSSQL.QUOTE +
+	    GSSQL.END;
+	boolean is_hierarchical = false;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    is_hierarchical = results.first();
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    return false;
+	}
+	
+	return is_hierarchical;
+    }
+    
+    // classifier metadata is not actually metadata
+    public String getClassifierMetadata(String oid, String full_meta_name) {
+
+	if (!full_meta_name.equals("Title") && !full_meta_name.equals("numleafdocs")) {
+	    return null;
+	}
+	// get the description
+	String field_name="";
+	if (full_meta_name.equals("Title")) {
+	    field_name = GSSQL.DESCRIPTION;
+	} else {
+	    field_name = GSSQL.NUM_LEAF_DOCUMENTS;
+	}
+	
+	String query = "select "+ field_name+
+	    " from " + GSSQL.CLASSIFIER_TABLE +
+	    " where " + GSSQL.CLASSIFIER_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    GSSQL.END;
+	
+	String value = null;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    if (results.first()) {
+		value = results.getString(field_name);
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getClassifierMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	return value;
+	
+    }
+    
+    public ArrayList getDocumentChildren(String oid) {
+
+	ArrayList children = new ArrayList();
+	String doc_id = oid;
+	String section_id = "All"; // if we are just passed a doc id
+	if (!GS3OID.isDocTop(oid)) {
+	    doc_id = GS3OID.getDocID(oid);
+	    section_id = GS3OID.getSectionLabel(oid);
+	}
+//  	int sep_index = oid.indexOf(".");
+//  	if (sep_index != -1) {
+//  	    doc_id = oid.substring(0, sep_index);
+//  	    section_id = oid.substring(sep_index+1);
+//  	}
+
+	// get the division ref for this section
+	String query = "select "+GSSQL.DIVISION_REF +
+	    " from "+GSSQL.DIVISION_TABLE+
+	    " where "+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.SHORT_LABEL+GSSQL.EQUALS_QUOTE+section_id+GSSQL.QUOTE+
+	    GSSQL.END;
+
+	ResultSet results = null;
+	Statement statement = null;
+	String div_ref = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		div_ref = results.getString(GSSQL.DIVISION_REF);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getDocumentChildren() Error1: "+e.getMessage());
+	    return null;
+	}
+
+	// now get the list of children
+	query = "select "+GSSQL.SHORT_LABEL +
+	    " from "+GSSQL.DIVISION_TABLE+
+	    " where "+ GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.PARENT_TYPE+GSSQL.EQUALS_QUOTE+GSSQL.DIVISION+GSSQL.QUOTE+
+	    " and "+GSSQL.PARENT_REF+GSSQL.EQUALS_QUOTE+div_ref+GSSQL.QUOTE+
+	    GSSQL.END;
+	try {
+	    results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.SHORT_LABEL);
+		children.add(GS3OID.createOID(doc_id, id));
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getDocumentChildren() Error2: "+e.getMessage());
+	    return null;
+	}
+
+	return children;
+	
+    }
+    
+    public boolean documentHasChildren(String oid) {
+	// for now, use getChildren, but can we do this a better way??
+	ArrayList children = getDocumentChildren(oid);
+	if (children==null || children.size()==0) {
+	    return false;
+	}
+	return true;
+	
+    }
+    public ArrayList getClassifierChildren(String oid) {
+
+	ArrayList children = new ArrayList();
+
+	String query = "select "+ GSSQL.CLASSIFIER_ID +
+	    " from " + GSSQL.CLASSIFIER_TABLE +
+	    " where " + GSSQL.PARENT_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and " + GSSQL.NUM_LEAF_DOCUMENTS + GSSQL.NOT+GSSQL.EQUALS_QUOTE+ "0"+GSSQL.QUOTE +
+	    GSSQL.END; 
+	
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.CLASSIFIER_ID);
+		children.add(id);
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getClassifierChildren Error: "+e.getMessage());
+	    return null;
+	}
+	return children;
+    }
+
+    public ArrayList getClassifierDocChildren(String oid) {
+	ArrayList children = new ArrayList();
+
+	String query = "select "+ GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+ GSSQL.DOCUMENT_ID+
+	    " from " + GSSQL.CLASSIFIER_TABLE + GSSQL.COMMA+GSSQL.CLASS_DOCUMENT_TABLE+
+	    " where " + GSSQL.CLASSIFIER_TABLE +GSSQL.DOT+ GSSQL.CLASSIFIER_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and " + GSSQL.CLASSIFIER_TABLE +GSSQL.DOT+ GSSQL.CLASSIFIER_REF+GSSQL.EQUALS+GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+ GSSQL.CLASSIFIER_REF+
+	    " order by "+GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ORDER+
+	    GSSQL.END;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.DOCUMENT_ID);
+		children.add(id);
+	    }
+	} catch (java.sql.SQLException e) {
+		
+	    System.err.println("SQLQuery.getClassifierDocChildren() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return children;
+    }
+    
+    public String getDocumentMetadata(String oid, String full_meta_name) {
+
+	String doc_id = oid;
+	String section_id = "All"; // if we are just passed a doc id
+	if (!GS3OID.isDocTop(oid)) {
+	    doc_id = GS3OID.getDocID(oid);
+	    section_id = GS3OID.getSectionLabel(oid);
+	}
+//  	int sep_index = oid.indexOf(".");
+//  	if (sep_index != -1) {
+//  	    doc_id = oid.substring(0, sep_index);
+//  	    section_id = oid.substring(sep_index+1);
+//  	}
+
+
+	// get the metadata block id
+	String query = "select "+ 
+	    GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID +
+	    " from " + GSSQL.DIVISION_METADATA_TABLE+GSSQL.COMMA+GSSQL.DIVISION_TABLE+
+	    " where " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+GSSQL.EQUALS+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+
+	    " and " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.SHORT_LABEL+GSSQL.EQUALS_QUOTE+section_id+GSSQL.QUOTE+
+	    " and " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+ 
+	    GSSQL.END;
+	
+	Statement statement = null;
+	ResultSet results = null;
+	String meta_id = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		meta_id = results.getString(GSSQL.METADATA_ID);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+
+	    System.err.println("SQLQuery.getDocumentMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	// now get the list of namespace refs for the right namespace
+	int sep_index = full_meta_name.indexOf('.');
+	String meta_ns = "gsdl3";
+	String meta_name = full_meta_name;
+	if (sep_index != -1) {
+
+	    meta_ns = full_meta_name.substring(0, sep_index);
+	    meta_name = full_meta_name.substring(sep_index+1);
+	}
+
+	query = "select * "+ 
+	    " from "+GSSQL.NAMESPACE_TABLE+GSSQL.COMMA+GSSQL.METADATA_TABLE+GSSQL.COMMA+GSSQL.METADATA_VALUE_TABLE+
+	    " where "+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+GSSQL.EQUALS_QUOTE+meta_id+GSSQL.QUOTE+
+	    " and "+ GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and " + GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+GSSQL.EQUALS+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+
+	    " and "+ GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_TYPE+GSSQL.EQUALS_QUOTE+meta_ns+GSSQL.QUOTE+
+	    " and "+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+GSSQL.EQUALS+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+
+	    GSSQL.END;
+	
+	String meta_value = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    while (results.next()) {
+		String m_name = results.getString(GSSQL.LABEL);
+		if (meta_name.equals(m_name)) {
+		    meta_value = results.getString(GSSQL.VALUE);
+		    break;
+		}
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+
+	    System.err.println("SQLQuery.getDocumentMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return meta_value;
+    }
+    public static void main (String [] args) {
+
+	SQLQuery self = new SQLQuery();
+	self.setDatabase("gs3test");
+	System.out.println("getting oid for mg num 3");
+	String oid = self.MGNum2OID("stx.3");
+	System.out.println("oid was "+oid);
+	
+	System.out.println("title metadata for mg num 3:");
+	String meta = self.getDocumentMetadata(oid, "gsdl3.title");
+	if (meta != null) {System.out.println(meta);}
+	System.out.println("subject metadata for mg num 3:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.Subject");
+	if (meta != null) {System.out.println(meta);}
+
+	System.out.println("converting back to mg, oid "+oid);
+	String id = self.OID2MGNum(oid);
+	System.out.println(""+id);
+
+	System.out.println("getting oid for mg num 8");
+	oid = self.MGNum2OID("stx.8");
+	System.out.println("oid was "+oid);
+
+	System.out.println("title metadata for mg num 8:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.title");
+	if (meta != null) {System.out.println(meta);}
+	System.out.println("subject metadata for mg num 8:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.Subject");
+	if (meta != null) {System.out.println(meta);}
+
+	System.out.println("converting back to mg, oid "+oid);
+	id = self.OID2MGNum(oid);
+	System.out.println(""+id);
+    }
+
+
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/.cvsignore
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/.cvsignore	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/.cvsignore	(revision 14150)
@@ -0,0 +1,3 @@
+build
+dist
+test-classes
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/COPYING
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/COPYING	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/COPYING	(revision 14150)
@@ -0,0 +1,353 @@
+The files in this directory and all its subdirectories belong to the
+Greenstone Alerting Service. These files, with the exemption of the JAR files
+in the lib subdirectory, are Copyright (C) 2004, 2005 by Andrea Schweer
+(schweer@cs.waikato.ac.nz) and are licensed under the GNU General Public
+License Version 2 (see below). For the JAR files in the lib subdirectory, see
+the license information inside each JAR file or in the lib subdirectory.
+
+===============================================================================
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/build.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/build.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/build.xml	(revision 14150)
@@ -0,0 +1,517 @@
+<!--
+     General purpose build script for web applications and web services,
+     including enhanced support for deploying directly to a Tomcat 4
+     based server.
+
+     This build script assumes that the source code of your web application
+     is organized into the following subdirectories underneath the source
+     code directory from which you execute the build script:
+
+        docs                 Static documentation files to be copied to
+                             the "docs" subdirectory of your distribution.
+
+        src                  Java source code (and associated resource files)
+                             to be compiled to the "WEB-INF/classes"
+                             subdirectory of your web applicaiton.
+
+        web                  Static HTML, JSP, and other content (such as
+                             image files), including the WEB-INF subdirectory
+                             and its configuration file contents.
+
+     $Id$
+-->
+
+
+<!-- A "project" describes a set of targets that may be requested
+     when Ant is executed.  The "default" attribute defines the
+     target which is executed if no specific target is requested,
+     and the "basedir" attribute defines the current working directory
+     from which Ant executes the requested task.  This is normally
+     set to the current working directory.
+-->
+
+<project name="GSDL Alerting Service" default="compile" basedir=".">
+
+
+
+<!-- ===================== Property Definitions =========================== -->
+
+
+<!--
+
+  Each of the following properties are used in the build script.
+  Values for these properties are set by the first place they are
+  defined, from the following list:
+
+  * Definitions on the "ant" command line (ant -Dfoo=bar compile).
+
+  * Definitions from a "build.properties" file in the top level
+    source directory of this application.
+
+  * Definitions from a "build.properties" file in the developer's
+    home directory.
+
+  * Default definitions in this build.xml file.
+
+  You will note below that property values can be composed based on the
+  contents of previously defined properties.  This is a powerful technique
+  that helps you minimize the number of changes required when your development
+  environment is modified.  Note that property composition is allowed within
+  "build.properties" files as well as in the "build.xml" script.
+
+-->
+
+  <property file="build.properties"/>
+  <property file="${user.home}/build.properties"/>
+
+
+<!-- ==================== File and Directory Names ======================== -->
+
+
+<!--
+
+  These properties generally define file and directory names (or paths) that
+  affect where the build process stores its outputs.
+
+  app.name             Base name of this application, used to
+                       construct filenames and directories.
+                       Defaults to "myapp".
+
+  app.path             Context path to which this application should be
+                       deployed (defaults to "/" plus the value of the
+                       "app.name" property).
+
+  app.version          Version number of this iteration of the application.
+
+  build.home           The directory into which the "prepare" and
+                       "compile" targets will generate their output.
+                       Defaults to "build".
+
+  catalina.home        The directory in which you have installed
+                       a binary distribution of Tomcat 4.  This will
+                       be used by the "deploy" target.
+
+  dist.home            The name of the base directory in which
+                       distribution files are created.
+                       Defaults to "dist".
+
+  manager.password     The login password of a user that is assigned the
+                       "manager" role (so that he or she can execute
+                       commands via the "/manager" web application)
+
+  manager.url          The URL of the "/manager" web application on the
+                       Tomcat installation to which we will deploy web
+                       applications and web services.
+
+  manager.username     The login username of a user that is assigned the
+                       "manager" role (so that he or she can execute
+                       commands via the "/manager" web application)
+
+-->
+
+  <property name="app.name"      value="gsdl-as"/>
+  <property name="app.path"      value="/alerting"/>
+  <property name="app.version"   value="0.1-dev"/>
+  <property name="build.home"    value="${basedir}/build"/>
+  <property name="catalina.home" value="../../comms/jakarta/tomcat/"/>
+  <property name="dist.home"     value="${basedir}/dist"/>
+  <property name="docs.home"     value="${basedir}/docs"/>
+  <property name="docs.link"     value="http://java.sun.com/j2se/1.4.2/docs/api/"/>
+  <property name="manager.url"   value="http://localhost:8080/manager"/>
+  <property name="manager.username" value="admin"/>
+  <property name="manager.password" value="admin"/>
+  <property name="src.home"      value="${basedir}/src"/>
+  <property name="web.home"      value="${basedir}/web"/>
+
+
+<!-- ================== Custom Ant Task Definitions ======================= -->
+
+
+<!--
+
+  These properties define custom tasks for the Ant build tool that interact
+  with the "/manager" web application installed with Tomcat 4.  Before they
+  can be successfully utilized, you must perform the following steps:
+
+  - Copy the file "server/lib/catalina-ant.jar" from your Tomcat 4
+    installation into the "lib" directory of your Ant installation.
+
+  - Create a "build.properties" file in your application's top-level
+    source directory (or your user login home directory) that defines
+    appropriate values for the "manager.password", "manager.url", and
+    "manager.username" properties described above.
+
+  For more information about the Manager web application, and the functionality
+  of these tasks, see <http://localhost:8080/tomcat-docs/manager-howto.html>.
+
+-->
+
+  <taskdef name="install" classname="org.apache.catalina.ant.InstallTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="list"    classname="org.apache.catalina.ant.ListTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="reload"  classname="org.apache.catalina.ant.ReloadTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="remove"  classname="org.apache.catalina.ant.RemoveTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="false"/>
+  <property name="compile.optimize"    value="true"/>
+
+
+
+<!-- ==================== External Dependencies =========================== -->
+
+
+<!--
+
+  Use property values to define the locations of external JAR files on which
+  your application will depend.  In general, these values will be used for
+  two purposes:
+  * Inclusion on the classpath that is passed to the Javac compiler
+  * Being copied into the "/WEB-INF/lib" directory during execution
+    of the "deploy" target.
+
+  Because we will automatically include all of the Java classes that Tomcat 4
+  exposes to web applications, we will not need to explicitly list any of those
+  dependencies.  You only need to worry about external dependencies for JAR
+  files that you are going to include inside your "/WEB-INF/lib" directory.
+
+-->
+
+<!-- Dummy external dependency -->
+<!--
+  <property name="foo.jar"
+           value="/path/to/foo.jar"/>
+-->
+
+
+<!-- ==================== Compilation Classpath =========================== -->
+
+<!--
+
+  Rather than relying on the CLASSPATH environment variable, Ant includes
+  features that makes it easy to dynamically construct the classpath you
+  need for each compilation.  The example below constructs the compile
+  classpath to include the servlet.jar file, as well as the other components
+  that Tomcat makes available to web applications automatically, plus anything
+  that you explicitly added.
+
+-->
+
+  <path id="compile.classpath">
+
+    <!-- Include all JAR files that will be included in /WEB-INF/lib -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+    <fileset dir="lib">
+    	<include name="*.jar"/>
+    </fileset>
+    <!-- include greenstone files -->
+    <fileset dir="../../web/WEB-INF/lib/">
+      <include name="*.jar"/>
+    </fileset>
+    <!-- Include all elements that Tomcat exposes to applications -->
+    <pathelement location="${catalina.home}/common/classes"/>
+    <fileset dir="${catalina.home}/common/endorsed">
+      <include name="*.jar"/>
+    </fileset>
+    <fileset dir="${catalina.home}/common/lib">
+      <include name="*.jar"/>
+    </fileset>
+    <pathelement location="${catalina.home}/shared/classes"/>
+    <fileset dir="${catalina.home}/shared/lib">
+      <include name="*.jar"/>
+    </fileset>
+
+  </path>
+
+
+
+<!-- ==================== All Target ====================================== -->
+
+<!--
+
+  The "all" target is a shortcut for running the "clean" target followed
+  by the "compile" target, to force a complete recompile.
+
+-->
+
+  <target name="all" depends="clean,compile"
+   description="Clean build and dist directories, then compile"/>
+
+
+
+<!-- ==================== Clean Target ==================================== -->
+
+<!--
+
+  The "clean" target deletes any previous "build" and "dist" directory,
+  so that you can be ensured the application can be built from scratch.
+
+-->
+
+  <target name="clean"
+   description="Delete old build and dist directories">
+    <delete dir="${build.home}"/>
+    <delete dir="${dist.home}"/>
+  </target>
+
+
+
+<!-- ==================== Compile Target ================================== -->
+
+<!--
+
+  The "compile" target transforms source files (from your "src" directory)
+  into object files in the appropriate location in the build directory.
+  This example assumes that you will be including your classes in an
+  unpacked directory hierarchy under "/WEB-INF/classes".
+
+-->
+
+  <target name="compile" depends="prepare"
+   description="Compile Java sources">
+
+    <!-- Compile Java classes as necessary -->
+    <mkdir    dir="${build.home}/WEB-INF/classes"/>
+    <javac srcdir="${src.home}"
+          destdir="${build.home}/WEB-INF/classes"
+            debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+         optimize="${compile.optimize}">
+        <classpath refid="compile.classpath"/>
+    </javac>
+
+    <!-- Copy application resources -->
+    <copy  todir="${build.home}/WEB-INF/classes">
+      <fileset dir="${src.home}" excludes="**/*.java"/>
+    </copy>
+
+  </target>
+
+
+
+<!-- ==================== Dist Target ===================================== -->
+
+
+<!--
+
+  The "dist" target creates a binary distribution of your application
+  in a directory structure ready to be archived in a tar.gz or zip file.
+  Note that this target depends on two others:
+
+  * "compile" so that the entire web application (including external
+    dependencies) will have been assembled
+
+  * "javadoc" so that the application Javadocs will have been created
+
+-->
+
+  <target name="dist" depends="compile,javadoc"
+   description="Create binary distribution">
+
+    <!-- Copy documentation subdirectories -->
+    <mkdir   dir="${dist.home}/docs"/>
+    <copy    todir="${dist.home}/docs">
+      <fileset dir="${docs.home}"/>
+    </copy>
+
+    <!-- Create application JAR file -->
+    <jar jarfile="${dist.home}/${app.name}-${app.version}.war"
+         basedir="${build.home}"/>
+
+    <!-- Copy additional files to ${dist.home} as necessary -->
+
+  </target>
+
+
+
+<!-- ==================== Install Target ================================== -->
+
+<!--
+
+  The "install" target tells the specified Tomcat 4 installation to dynamically
+  install this web application and make it available for execution.  It does
+  *not* cause the existence of this web application to be remembered across
+  Tomcat restarts; if you restart the server, you will need to re-install all
+  this web application.
+
+  If you have already installed this application, and simply want Tomcat to
+  recognize that you have updated Java classes (or the web.xml file), use the
+  "reload" target instead.
+
+  NOTE:  This target will only succeed if it is run from the same server that
+  Tomcat is running on.
+
+  NOTE:  This is the logical opposite of the "remove" target.
+
+-->
+
+  <target name="install" depends="compile"
+   description="Install application to servlet container">
+
+    <install url="${manager.url}"
+        username="${manager.username}"
+        password="${manager.password}"
+            path="${app.path}"
+             war="file://${build.home}"/>
+
+  </target>
+
+
+<!-- ==================== Javadoc Target ================================== -->
+
+<!--
+
+  The "javadoc" target creates Javadoc API documentation for the Java
+  classes included in your application.  Normally, this is only required
+  when preparing a distribution release, but is available as a separate
+  target in case the developer wants to create Javadocs independently.
+
+-->
+
+  <target name="javadoc" depends="compile"
+   description="Create Javadoc API documentation">
+
+    <mkdir          dir="${dist.home}/docs/api"/>
+    <javadoc sourcepath="${src.home}"
+                destdir="${dist.home}/docs/api"
+    			   link="${docs.link}"
+    	        private="true"
+           packagenames="*">
+      <classpath refid="compile.classpath"/>
+    </javadoc>
+
+  </target>
+
+
+
+<!-- ====================== List Target =================================== -->
+
+<!--
+
+  The "list" target asks the specified Tomcat 4 installation to list the
+  currently running web applications, either loaded at startup time or
+  installed dynamically.  It is useful to determine whether or not the
+  application you are currently developing has been installed.
+
+-->
+
+  <target name="list"
+   description="List installed applications on servlet container">
+
+    <list    url="${manager.url}"
+        username="${manager.username}"
+        password="${manager.password}"/>
+
+  </target>
+
+
+<!-- ==================== Prepare Target ================================== -->
+
+<!--
+
+  The "prepare" target is used to create the "build" destination directory,
+  and copy the static contents of your web application to it.  If you need
+  to copy static files from external dependencies, you can customize the
+  contents of this task.
+
+  Normally, this task is executed indirectly when needed.
+
+-->
+
+  <target name="prepare">
+
+    <!-- Create build directories as needed -->
+    <mkdir  dir="${build.home}"/>
+    <mkdir  dir="${build.home}/WEB-INF"/>
+    <mkdir  dir="${build.home}/WEB-INF/classes"/>
+
+
+    <!-- Copy static content of this web application -->
+    <copy todir="${build.home}">
+      <fileset dir="${web.home}"/>
+    </copy>
+
+    <jar destfile="lib/templates.jar" basedir="lib/templates" includes="*.vm"/>
+
+    <!-- Copy external dependencies as required -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+    <mkdir  dir="${build.home}/WEB-INF/lib"/>
+    <copy todir="${build.home}/WEB-INF/lib">
+        <fileset dir="lib" includes="*.jar"/>
+    </copy>
+
+    <!-- Copy static files from external dependencies as needed -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+
+  </target>
+
+
+<!-- ==================== Reload Target =================================== -->
+
+<!--
+
+  The "reload" signals the specified application Tomcat 4 to shut itself down
+  and reload. This can be useful when the web application context is not
+  reloadable and you have updated classes or property files in the
+  /WEB-INF/classes directory or when you have added or updated jar files in the
+  /WEB-INF/lib directory.
+
+  NOTE: The /WEB-INF/web.xml web application configuration file is not reread
+  on a reload. If you have made changes to your web.xml file you must stop
+  then start the web application. 
+
+-->
+
+  <target name="reload" depends="compile"
+   description="Reload application on servlet container">
+
+    <reload url="${manager.url}"
+       username="${manager.username}"
+       password="${manager.password}"
+           path="${app.path}"/>
+
+  </target>
+
+
+<!-- ==================== Remove Target =================================== -->
+
+<!--
+
+  The "remove" target tells the specified Tomcat 4 installation to dynamically
+  remove this web application from service.
+
+  NOTE:  This is the logical opposite of the "install" target.
+
+-->
+
+  <target name="remove"
+   description="Remove application on servlet container">
+
+    <remove url="${manager.url}"
+       username="${manager.username}"
+       password="${manager.password}"
+           path="${app.path}"/>
+
+  </target>
+
+
+</project>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/create-tables.sql
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/create-tables.sql	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/create-tables.sql	(revision 14150)
@@ -0,0 +1,44 @@
+create table users (
+	username varchar(128) primary key, 
+	password text not null
+);
+
+create table subscriptions (
+	id int primary key auto_increment, 
+	name varchar(255), 
+	email text, 
+	rss bool not null default 0, 
+	page bool not null default 0, 
+	user varchar(128) not null references users.username, 
+	
+	unique(user,name)
+);
+
+create table predicates (
+	id int primary key auto_increment, 
+	field varchar(128) not null, 
+	value varchar(255), 
+	type enum("org.greenstone.gsdlas.profiles.EqualsPredicate", "org.greenstone.gsdlas.profiles.SubstringMatchPredicate", "org.greenstone.gsdlas.profiles.QueryPredicate") not null,
+	
+	unique(field,type,value)
+);
+
+create table subs_to_predicates (
+	subscription int not null references subscriptions.id, 
+	predicate int not null references predicates.id, 
+	
+	unique (subscription,predicate)
+);
+
+create table events (
+	id int primary key auto_increment,
+	timestamp datetime not null,
+	content text not null
+);
+
+create table events_to_subs (
+	event int not null references events.id,
+	subscription int not null references subscriptions.id,
+
+	unique (event,subscription)
+);
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/descriptor.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/descriptor.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/descriptor.xml	(revision 14150)
@@ -0,0 +1,7 @@
+<isd:service
+  xmlns:isd="http://xml.apache.org/xml-soap/deployment"
+  id="alerting" checkMustUnderstands="false">
+  <isd:provider type="java" scope="Session" methods="receiveEvent">
+    <isd:java class="org.greenstone.gsdlas.AlertingService" static="false"/>
+  </isd:provider>
+</isd:service>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/.cvsignore
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/.cvsignore	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/.cvsignore	(revision 14150)
@@ -0,0 +1,1 @@
+templates.jar
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/error.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/error.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/error.vm	(revision 14150)
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Greenstone Digital Library Software</title>
+		<link href="default.css" rel="stylesheet" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>$title</p>
+			<p class="error">$message</p>
+			<p class="details">$details</p>
+			<a href="/alerting">back</a> to the alerting service
+		</div>
+
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/events.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/events.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/events.vm	(revision 14150)
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="default.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Events for subscription $subscription.name</p>
+<p class="description">$subscription</p>
+<p><a href="/alerting/service?action=showFeed&amp;subscriptionID=$subscription.id">show as feed</a></p>
+#foreach ($event in $list)
+<p>$event</p>
+#end
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/feed.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/feed.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/feed.vm	(revision 14150)
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<!--<?xml-stylesheet type="text/xml" href="/alerting/rss2html.xml"?>-->
+<rss version="2.0">
+   <channel>
+      <title>Events for Subscription $subscription.name</title>
+      <link>/alerting/service?action=showFeed&amp;subscriptionID=$subscription.id</link>
+      <description>TODO</description>
+      <language>en</language>
+      <lastBuildDate>$lastBuildDate</lastBuildDate>
+      <generator>Alerting Service for Greenstone</generator>
+      <webMaster>$webmaster</webMaster>
+      
+      #foreach($event in $list)
+      <item>
+         <title>$event.document_title</title>
+         <link>$event.document_url</link>
+         <description>some description</description>
+         <pubDate>$event.timestamp</pubDate>
+         <guid>$event.document_url</guid>
+         <category>$event.type</category>
+      </item>
+      #end
+   </channel>
+</rss>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/general.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/general.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/general.vm	(revision 14150)
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="default.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>$title</p>
+<p class="message">$message</p>
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/list.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/list.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/list.vm	(revision 14150)
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="list.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>$title</p>
+#foreach($sub in $list)
+	<div class="subscription">
+		<p><span class="sub_id">Subscription $sub.Id</span><br/>
+		<a href="service?action=showEvents&subscriptionID=$sub.Id">show matching events</a> | <a href="service?action=editSubscription&subscriptionID=$sub.Id">edit subscription</a> | <a href="service?action=deleteSubscription&subscriptionID=$sub.Id">delete subscription</a></p>
+<!--		<table class="subscription">
+			<thead>
+				<tr><th>Field</th><th>Predicate</th></tr>
+			</thead>
+			<tfoot><tr><th/><th/></tr></tfoot>
+			<tbody>
+				#foreach($pred in $sub.Predicates)
+					<tr>
+						<td>$pred.Field</td>
+						<td>$pred.Value</td>
+					</tr>
+				#end
+			</tbody>
+		</table> // -->
+		<pre>
+		$sub
+		</pre>
+		<br/>
+	</div>
+#end
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/login.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/login.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/login.vm	(revision 14150)
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="/alerting/form.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Login</p>
+#if ($error) 
+<div class="error">
+An error occured:<br/>
+$errormessage
+</div>
+#end
+<form action="/alerting/service?action=login" method="post"/>
+<table>
+<tr>
+<th><label for="username">username</label></th><td><input type="text" name="username" id="username"/></td>
+</tr>
+<tr>
+<th><label for="password">password</label></th><td><input type="password" name="password" id="password"/></td>
+</tr>
+</table>
+<br/>
+<input type="submit" value="login"/>
+</form>
+<div class="description">Don't have an account yet? <a href="/alerting/service?action=showRegistrationForm">Register</a> now!</a></div>
+</div>
+
+</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/register.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/register.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/register.vm	(revision 14150)
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="/alerting/form.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Register</p>
+#if ($error)
+<div class="error">
+An error occured:<br/>
+$errormessage
+</div>
+#end
+<form action="/alerting/service?action=register" method="post"/>
+<table>
+<tr>
+<th><label for="username">username</label></th><td><input type="text" name="username" id="username"/></td>
+</tr>
+<tr>
+<th><label for="password">password</label></th><td><input type="password" name="password" id="password"/></td>
+</tr>
+<tr>
+<th><label for="password2">repeat password</label></th><td><input type="password" name="password2" id="password2"/></td>
+</tr>
+</table>
+<br/>
+<input type="submit" value="create account"/>
+</form>
+</div>
+
+</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_collection.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_collection.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_collection.vm	(revision 14150)
@@ -0,0 +1,76 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (3/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td class="completed">host</td><td class="current">collection</td><td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the collection(s) to monitor</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				</div>
+				<div>
+					<h4>Which collection(s) to monitor</h4>
+					<div class="explanation">
+					You have two ways of choosing the collection(s) to monitor: 
+					<ol>
+					<li>You can choose one or more collections from the list below. Only events from the collections(s) you have chosen will be sent to you<br/>
+					<select multiple="multiple" name="hostCollectionID" size="7" style="margin:5px 0px;">
+						<option value="">any collection on any host</option>
+						#foreach($host in $hostnames)
+						<optgroup label="$host">
+							#foreach($collection in $collectionnames.get($host))
+							<option>$host|$collection</option>
+							#end
+							<option selected="selected" value="$host">any collection on host $host</option>
+						</optgroup>
+					#end
+					</select></li>
+					<li>You can enter a query in the field below. Only events coming from a collection whose name matches this query will be sent to you.<br/>
+					<table style="margin:5px 0px;">
+						<tr class="odd">
+							<th>collection name</th>
+							<td width="25%">contains <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+							<td>
+								<input type="text" name="collection_query" style="width:80%;" value="$!prefill.collection_query"/>
+							</td>
+						</tr>
+						</table>
+						If you enter a query, the selection made in the list above will be ignored.
+						<!-- collection description? creator? language? -->
+						</li>
+					</ol>
+					If you just click &quot;continue&quot;, events coming from <em>any collection</em> will be sent to you.
+					</div>
+				</div>
+				<p class="explanation">
+				In the last step, you will be able to configure the the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="collection"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_host.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_host.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_host.vm	(revision 14150)
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (2/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td  class="current">host</td><td>collection</td><td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the host(s) to monitor</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				</div>
+				<div>
+					<h4>Which host(s) to monitor</h4>
+					<div class="explanation">
+					You have two ways of choosing the host(s) to monitor: 
+					<ol>
+					<li>You can choose one or more hosts from the list below. Only events from the host(s) you have chosen will be sent to you.<br/>
+					<select multiple="multiple" name="hostID" size="3" style="margin:5px 0px;">
+					#set($firstoption = true);
+					#foreach($host in $hostnames)
+					<option #if($firstoption) selected="selected" #set($firstoption = false) #end>$host</option>
+					#end
+					<option value="">any host</option>
+					</select></li>
+					<li>You can enter a query in the field below. Only events coming from a host whose name matches this query will be sent to you.<br/>
+					<table style="margin:5px 0px;">
+						<tr class="odd">
+							<th>host</th>
+							<td width="25%">contains <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+							<td>
+								<input type="text" name="host_query" style="width:80%;" value="$!prefill.host_query"/>
+							</td>
+						</tr>
+						</table>
+						If you enter a query, the selection made in the list above will be ignored.
+						</li>
+					</ol>
+					If you just click &quot;continue&quot;, only events coming from the <em>current host</em> will be sent to you.
+					</div>
+				</div>
+				<p class="explanation">
+				In the next steps, you will be able to choose the collection(s) to monitor, and you will be able to configure the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="host"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_notification.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_notification.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_notification.vm	(revision 14150)
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (4/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td class="completed">host</td><td class="completed">collection</td><td class="current">notification</td>
+			</tr>
+			</table>
+			<p>Configure the way you will be notified of events </p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				<p class="explanation">You can <a href="preview.html" target="_new">preview</a> which documents currently match your criteria (following the link will open a new window).</p>
+				</div>
+				<div>
+					<h4>How to be notified of events matching your subscription</h4>
+					<div class="explanation">
+					You will be able to see the subscription in the list of all your subscriptions after logging in to the alerting service. <br/>
+					You can assign a name to your subscription that makes it easier for you to recognise it: 
+					<input name="subscription_name" type="text" value="$!prefill.subscription_name"/>
+					</div>
+					<br/>
+					<table>
+						<tr class="odd">
+							<td><input name="way" type="checkbox" value="page" checked="checked" id="chck-page"/></td>
+							<td colspan="2">
+							<label for="chck-page">on my &quot;new since my last login&quot; page</label>
+							</td>
+						</tr>
+						<tr class="even">
+							<td><input name="way" type="checkbox" value="email" id="chck-email"/></td>
+							<td><label for="chck-email">via e-mail to</label>
+							</td>
+							<td>
+								<input type="text" name="email" value="$!prefill.email"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<td><input name="way" type="checkbox" value="rss" id="chck-rss"/></td>
+							<td colspan="2"><label for="chck-rss">with an <acronym title="Really Simple Syndication">RSS</acronym> feed</label> <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+						</tr>
+						</table>
+				</div>
+				<br/>
+				
+				<input type="hidden" name="current_page" value="notification"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next" disabled="disabled"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_type-details.vm
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_type-details.vm	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/lib/templates/sub_type-details.vm	(revision 14150)
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Create a subscription</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (1/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="current">type/details</td>
+			<td>host</td>
+			<td>collection</td>
+			<td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the event type and how to identify the document(s) to watch</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div>
+					<h4>Event type</h4>
+					<input name="type" type="checkbox" value="new_document"  checked="checked" id="chck-new-document"/> <label for="chck-new-document">new document(s)</label><br/>
+					<input name="type" type="checkbox" value="document_changed"  id="chck-doc-changed"/> <label for="chck-doc-changed">document(s) changed</label><br/>
+					<input name="type" type="checkbox" value="document_deleted"  id="chck-doc-deleted"/> <label for="chck-doc-deleted">document(s) deleted</label><br/>
+				</div>
+				<div>
+					<h4>How to identify the document(s)</h4>
+					<table>
+						<tr class="even">
+							<th>title</th>
+							<td width="25%">matches <img class="help" alt="help" src="images/help.png" longdesc="normal Greenstone search" title="normal Greenstone search"/>
+							</td>
+							<td>
+								<input type="text" name="document_title" style="width:80%;" value="$!prefill.document_title"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<th>content</th>
+							<td>matches <img class="help" alt="help" src="images/help.png" longdesc="normal Greenstone search" title="normal Greenstone search"/>
+							</td>
+							<td>
+								<input type="text" name="document_content" style="width:80%;" value="$!prefill.document_content"/>
+							</td>
+						</tr>
+						<tr class="even">
+							<th>metadata</th>
+							<td>has field <img class="help" alt="help" src="images/help.png" longdesc="metadata field exists" title="metadata field exists"/>
+							</td>
+							<td>
+								<input type="text" name="metadata_has_field" value="$!prefill.metadata_has_field" style="width:80%;"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<th>metadata</th>
+							<td>has value <img class="help" alt="help" src="images/help.png" longdesc="metadata field exists and has value" title="metadata field exists and has value"/>
+							</td>
+							<td>
+								<input type="text" name="metadata_field" value="$!prefill.metatada_field" style="width:40%;"/> is
+								<input type="text" name="metadata_value" value="$!prefill.metadata_value" style="width:40%;"/>
+							</td>
+						</tr>
+					</table>
+				</div>
+				<p class="explanation">
+				In the next steps, you will be able to select the hosts and collections that you want to monitor for events. You will also be able to configure the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="type-details"/>
+				<input disabled="disabled" type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input disabled="disabled" type="submit" name="next_page" value="finish"/>
+				<br/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java	(revision 14150)
@@ -0,0 +1,547 @@
+/*
+ * Created on Nov 22, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.*;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.*;
+
+import org.apache.velocity.Template;
+import org.apache.velocity.context.Context;
+import org.apache.velocity.servlet.VelocityServlet;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.profiles.Predicate;
+import org.greenstone.gsdlas.profiles.Subscription;
+import org.greenstone.gsdlas.users.UserManagementException;
+import org.greenstone.gsdlas.users.UserManager;
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class AlertingService extends VelocityServlet {
+    
+    /* (non-Javadoc)
+     * @see javax.servlet.Servlet#init(javax.servlet.ServletConfig)
+     */
+    public void init(ServletConfig config) throws ServletException {
+        super.init(config);
+        System.out.println("reloading subs");
+        ProfileStore.getInstance().restoreFromDatabase();
+        try {
+            Set subs = ProfileStore.getInstance().getAllSubscriptionsFor("andrea");
+            System.out.println("reloaded " + subs.size() + " subs for andrea: ");
+            for (Iterator iter = subs.iterator(); iter.hasNext();) {
+                System.out.println(iter.next());
+            }
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+    public static final String[] actions = new String[] {
+            "createSubscription",
+            "deleteSubscription",
+            "editSubscription",
+            "showEvents",
+            "showFeed",
+            "listSubscriptions",
+            "login",
+            "register",
+            "logout",
+            "showLoginForm",
+            "showRegistrationForm"
+    };
+    
+    
+    protected Template handleRequest(HttpServletRequest req,
+            HttpServletResponse res, Context context) {
+        
+        String action = req.getParameter(Constants.ACTION_PARAM);
+
+        Map args = req.getParameterMap();
+        
+        if (action != null && action.equals("receiveEvent")) {
+            receiveEvent(args);
+            return null;
+        }
+        
+        args = normalise(args);
+        
+        if (action == null || !ArrayHelper.contains(actions, action)) {
+            String title = "Unknown action";
+            String message = "I don't know how to " + action;
+            String details = "The only actions I know are " + ArrayHelper.toString(actions);
+            return showError(context, message, details);
+        }
+                
+        String templateString = "";
+        
+        try {
+            Method method = AlertingService.class.getDeclaredMethod(action, new Class[] {Map.class, Context.class});
+            templateString = (String) method.invoke(this, new Object[] {args, context});
+        } catch (Exception e) {
+            String message = "An error has occured, I couldn't do what you told me to do.";
+            String details = e.getMessage() + " (" + e.getClass().getName() + "); "
+            	+ e.getCause() 
+            	+ " ; action is " + action;
+            return showError(context, message, details);
+        }
+        
+        Template template = null;
+        try {
+            template = getTemplate(templateString);
+        } catch (Exception e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+            try {
+                error((HttpServletRequest)context.get(REQUEST), 
+                        (HttpServletResponse)context.get(RESPONSE),
+                        e);
+            } catch (Exception e2) {
+                // TODO Auto-generated catch block
+                e2.printStackTrace();
+            }
+        }
+        return template;
+    }
+    
+    /**
+     * @param arguments
+     * @param context
+     * @return the Velocity template to use
+     * @throws Exception
+     */
+    public String createSubscription(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "createSubscription");
+            return showLoginForm(arguments, context);
+        }
+        if (arguments.containsKey("next_page") && arguments.get("next_page").equals("finish")) {
+            arguments.putAll(getPageArgsFromSession(session));
+            arguments.put("username", session.getAttribute("username"));
+            ProfileStore.getInstance().createSubscription(arguments);
+            return listSubscriptions(arguments, context);
+        } else {
+            return showSubscriptionWizardPage(arguments, context, true);
+        }
+    }
+
+    public String deleteSubscription(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "deleteSubscription");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        try {
+            ProfileStore.getInstance().deleteSubscription(subscriptionID);
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (SQLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        return listSubscriptions(arguments, context);
+    }
+    
+    public String editSubscription(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "editSubscription");
+            return showLoginForm(arguments, context);
+        }
+        if (arguments.containsKey("next_page") && arguments.get("next_page").equals("finish")) {
+            ProfileStore.getInstance().changeSubscription(arguments, session);
+            return listSubscriptions(arguments, context);
+        } else {
+            return showSubscriptionWizardPage(arguments, context, false);
+        }
+    }
+    
+    public String showFeed(Map arguments, Context context) throws NumberFormatException, DatabaseException, SQLException {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "showFeed");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        Integer subID = new Integer(subscriptionID);
+        Set events = EventStore.getInstance().getEvents(subID);
+        context.put("list", events);
+        context.put("subscription", ProfileStore.getInstance().getSubscription(subID.intValue()));
+        HttpServletResponse res = (HttpServletResponse) context.get(RESPONSE);
+        res.setContentType("text/xml");
+        return "feed.vm";
+    }
+    
+    public String showEvents(Map arguments, Context context) throws NumberFormatException, DatabaseException, SQLException {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "showEvents");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        Integer subID = new Integer(subscriptionID);
+        Set events = EventStore.getInstance().getEvents(subID);
+        context.put("list", events);
+        context.put("subscription", ProfileStore.getInstance().getSubscription(subID.intValue()));
+        return "events.vm";
+    }
+    
+    public String showLoginForm(Map arguments, Context context) {
+        return "login.vm";
+    }
+    
+    public String showRegistrationForm(Map arguments, Context context) {
+        return "register.vm";
+    }
+
+    public String listSubscriptions(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "listSubscriptions");
+            return showLoginForm(arguments, context);
+        }
+        String username = (String) session.getAttribute("username");
+        context.put("title", "List of Subscriptions for " + username);
+        try {
+            Collection subscriptions = ProfileStore.getInstance().getAllSubscriptionsFor(username); 
+            context.put("list", subscriptions);
+            return "list.vm";
+        } catch (DatabaseException de) {
+            context.put("message", "couldn't get list of subscriptions for " + username);
+            context.put("details", de.getMessage());
+            return "error.vm";
+        }
+    }
+    
+    public String login(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        try {
+            UserManager.getInstance().loginUser(arguments, session);
+        } catch (UserManagementException e) {
+            context.put("error", Boolean.TRUE);
+            context.put("errormessage", e.getMessage());
+            return showLoginForm(arguments, context); 
+        }
+        if (session.getAttribute("next_action") != null) {
+            String nextAction = (String) session.getAttribute("next_action");
+            Method method = AlertingService.class.getDeclaredMethod(nextAction, new Class[] {Map.class, Context.class});
+            return (String) method.invoke(this, new Object[] {arguments, context});
+        }
+        return listSubscriptions(arguments, context);
+    }
+    
+    public String register(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        try {
+            UserManager.getInstance().createUser(arguments, session);
+        } catch (UserManagementException e) {
+            context.put("error", Boolean.TRUE);
+            context.put("errormessage", e.getMessage());
+            return showRegistrationForm(arguments, context);
+        }
+        return login(arguments, context);
+    }
+    
+    public String logout(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        Enumeration atts = session.getAttributeNames();
+        while (atts.hasMoreElements()) {
+            session.removeAttribute((String) atts.nextElement());
+        }
+        session.invalidate();
+        return "general.vm";
+    }
+    
+    public void receiveEvent(Map rawEvent) {
+        Map event = new TreeMap();
+        for (Iterator iter = rawEvent.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (key.equals("action")) continue; // we don't want this
+            String[] value = (String[]) rawEvent.get(key);
+            event.put(key, value[0]);
+        }
+        System.out.println("receiving event " + event);
+        
+        GreenstoneCommunicator gsComm = null;
+        try {
+            String hostID = (String) event.get(Constants.HOST_ID_FIELD);
+            gsComm = new GreenstoneCommunicator(new URL(hostID));
+        } catch (Exception e) {
+            System.err.println("Can't communicate to Greenstone: " + e.getMessage());
+            e.printStackTrace();
+        }
+        
+        Set matchedSubscriptions = ProfileStore.getInstance().filter(event, gsComm);
+        
+        try {
+            EventStore.getInstance().add(event, matchedSubscriptions);
+            Notifier.getInstance().sendNotifications(event, matchedSubscriptions);
+        } catch (Exception e) {
+            System.err.println("Couldn't save events: " + e.getMessage());
+            e.printStackTrace();
+        }
+        System.out.println(matchedSubscriptions.size() + " matching subscriptions: " + matchedSubscriptions);
+    }
+
+    /**
+     * @param args
+     */
+    private Map normalise(Map args) {
+        Map result = new TreeMap();
+        for (Iterator iter = args.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (Predicate.isMultiValued(key) || key.equals("way")) {
+                // multi-valued attributes
+                String[] values = ((String[]) args.get(key));
+                result.put(key, Arrays.asList(values));
+            } else {
+                String firstValue = ((String[])args.get(key))[0];
+                result.put(key, firstValue);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * @param context
+     * @param message
+     * @param details
+     * @return
+     */
+    private Template showError(Context context, String message, String details) {
+        context.put("title", "Error");
+        context.put("message", message);
+        context.put("details", details);
+        try {
+            return getTemplate("error.vm");
+        } catch (Exception e) {
+            try {
+                super.error((HttpServletRequest)context.get("req"), 
+                        (HttpServletResponse)context.get("res"), 
+                        e);
+            } catch (Exception e2) {
+                e2.printStackTrace();
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @param arguments
+     * @param context
+     * @param create
+     * @return
+     * @throws Exception
+     */
+    private String showSubscriptionWizardPage(Map arguments, Context context, boolean create) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (create) {
+            context.put(Constants.ACTION_PARAM, "createSubscription");
+        } else {
+            // prefill from existing subscription
+            context.put(Constants.ACTION_PARAM, "editSubscription");
+            String subscriptionID = (String) arguments.get("subscriptionID");
+            int subID = Integer.parseInt(subscriptionID);
+            prefillFromSubscription(session, subID);
+        }
+        if (!arguments.containsKey("current_page")) {
+            return "sub_type-details.vm";
+        }
+        
+        String currentPage = (String) arguments.get("current_page");
+        String direction = (String) arguments.get("next_page");
+        
+        if (arguments.containsKey(Constants.HOST_QUERY_FIELD)) {
+            String hostQuery = (String) arguments.get(Constants.HOST_QUERY_FIELD);
+            if (hostQuery != null && hostQuery.length() != 0)
+                arguments.remove(Constants.HOST_ID_FIELD);
+        }
+        if (arguments.containsKey(Constants.COLLECTION_QUERY_FIELD)) {
+            String collQuery = (String) arguments.get(Constants.COLLECTION_QUERY_FIELD);
+            if (collQuery != null && collQuery.length() != 0)
+                arguments.remove(Constants.COLLECTION_ID_FIELD);
+        }
+        
+        // save page arguments
+        savePageArgsToSession(currentPage, arguments, session);
+        
+        String nextPage = getNextPage(currentPage, direction);
+
+        // fill prefill
+        context.put("prefill", getPageArgsFromSession(nextPage, session));
+        
+        // fill preview
+        context.put("preview", getPagePreview(nextPage, session));
+        
+        // get page-specific stuff
+        if (nextPage.equals("host")) {
+            String[] hostNames;
+            try {
+                GreenstoneCommunicator gsComm = new GreenstoneCommunicator();
+                hostNames = gsComm.getHostNames();
+            } catch (Exception e) {
+                hostNames = new String[] { "localhost" };
+            }
+            context.put("hostnames", hostNames);
+            session.setAttribute("hostnames", hostNames);
+        } else if (nextPage.equals("collection")) {
+            List hostNames = (List) arguments.get(Constants.HOST_ID_FIELD);
+            if (hostNames == null || hostNames.isEmpty()) {
+                hostNames = new Vector();
+                // no host names -> use host query
+                String[] hostsFromSession = (String[]) session.getAttribute("hostnames");
+                if (hostsFromSession == null || hostsFromSession.length == 0) {
+                    try {
+                        GreenstoneCommunicator gsComm = new GreenstoneCommunicator();
+                        hostsFromSession = gsComm.getHostNames();
+                    } catch (Exception e) {
+                        hostsFromSession = new String[] { "localhost" };
+                    }
+                }
+                String hostQuery = (String) arguments.get(Constants.HOST_QUERY_FIELD);
+                for (int i = 0; i < hostsFromSession.length; i++) {
+                    if (hostsFromSession[i] != null && hostsFromSession[i].indexOf(hostQuery) >= 0) {
+                        hostNames.add(hostsFromSession[i]);
+                    }
+                }
+            }
+            
+            Map collNames = new TreeMap();
+            for (Iterator iter = hostNames.iterator(); iter.hasNext();) {
+                String host = (String) iter.next();
+                Set collNamesForHost = new TreeSet();   
+                try {
+                    URL url = new URL("http://" + host + ":8080/soap/servlet/rpcrouter");
+                    GreenstoneCommunicator gsComm = new GreenstoneCommunicator(url);
+                    collNamesForHost.addAll(Arrays.asList(gsComm.getCollectionNames()));
+                } catch (Exception e) {
+                    // TODO Auto-generated catch block
+                    e.printStackTrace();
+                }	
+                collNames.put(host, collNamesForHost);
+            }
+            context.put("collectionnames", collNames);
+            context.put("hostnames", hostNames);
+        }
+        
+        return "sub_" + nextPage + ".vm";
+    }
+
+    /**
+     * @param session
+     * @param subID
+     */
+    private void prefillFromSubscription(HttpSession session, int subID) {
+        Subscription sub = ProfileStore.getInstance().getSubscription(subID);
+        Map typeArgs = new HashMap();
+        // TODO really fill stuff
+        savePageArgsToSession("type-details", typeArgs, session);
+        Map hostArgs = new HashMap();
+        savePageArgsToSession("host", hostArgs, session);
+        Map collArgs = new HashMap();
+        savePageArgsToSession("collection", collArgs, session);
+        Map notificationArgs = new HashMap();
+        savePageArgsToSession("notification", notificationArgs, session);
+    }
+
+    /**
+     * @param nextPage
+     * @param session
+     * @return
+     */
+    private Map getPagePreview(String nextPage, HttpSession session) {
+        Map preview = new TreeMap();
+        if (nextPage.equals("type-details")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("type-details", session));
+        if (nextPage.equals("host")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("host", session));
+        if (nextPage.equals("collection")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("collection", session));
+        return preview;
+    }
+
+    /**
+     * @param currentPage
+     * @param direction
+     * @return
+     * @throws Exception
+     */
+    private String getNextPage(String currentPage, String direction) throws Exception {
+        String nextPage;
+        if (currentPage.equals("host") && direction.equals("back")) {
+            nextPage = "type-details";
+        } else if (currentPage.equals("type-details") || (currentPage.equals("collection") && direction.equals("back"))) {
+            nextPage = "host";
+        } else if (currentPage.equals("host") || (currentPage.equals("notification") && direction.equals("back"))) {
+            nextPage = "collection";
+        } else if (currentPage.equals("collection")) {
+            nextPage = "notification";
+        } else {
+            throw new Exception("unknown combination of currentPage=" + currentPage + " and nextPage=" + direction);
+        }
+        return nextPage;
+    }
+
+    /**
+     * @param page
+     * @param session
+     * @return 
+     */
+    private Map getPageArgsFromSession(String page, HttpSession session) {
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs == null || !pageArgs.containsKey(page))
+            return new TreeMap();
+        return (Map) pageArgs.get(page);
+    }
+
+    private Map getPageArgsFromSession(HttpSession session) {
+        Map result = new TreeMap();
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs != null) {
+            for (Iterator iter = pageArgs.values().iterator(); iter.hasNext();) {
+                Map args = (Map) iter.next();
+                result.putAll(args);
+            }
+        }
+        return result;
+    }
+    
+    /**
+     * @param page
+     * @param arguments
+     * @param session
+     */
+    private void savePageArgsToSession(String page, Map arguments, HttpSession session) {
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs == null) {
+            pageArgs = new TreeMap();
+        }
+        pageArgs.put(page, arguments);
+        session.setAttribute("page_args", pageArgs);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/Constants.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/Constants.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/Constants.java	(revision 14150)
@@ -0,0 +1,31 @@
+/*
+ * Created on Dec 21, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public abstract class Constants {
+    public static final String ACTION_PARAM = "action";
+    
+    public static final String HOST_QUERY_FIELD = "host_query";
+    public static final String COLLECTION_QUERY_FIELD = "collection_query";
+    public static final String HOST_ID_FIELD = "hostID";
+    public static final String COLLECTION_ID_FIELD = "collectionID";
+    public static final String HOST_COLLECTION_ID_FIELD = "hostCollectionID";
+    public static final String DOCUMENT_ID_FIELD = "documentID";
+    public static final String DOCUMENT_CONTENT_FIELD = "document_content";
+    public static final String DOCUMENT_TITLE_FIELD = "document_title";
+    public static final String TYPE_FIELD = "type";
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/EventStore.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/EventStore.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/EventStore.java	(revision 14150)
@@ -0,0 +1,169 @@
+/*
+ * Created on Nov 23, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.sql.*;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.Date;
+
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.profiles.Subscription;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class EventStore {
+
+    private static EventStore instance;
+    
+    private EventStore() {
+    }
+    
+    /**
+     * @return
+     */
+    public static EventStore getInstance() {
+        if (instance == null) instance = new EventStore();
+        return instance;
+    }
+    
+    public void add(Map event, Set subscriptions) throws DatabaseException, SQLException, ParseException {
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+
+        int eventID = saveEventToDatabase(event);
+        
+        for (Iterator iter = subscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            int subID = sub.getId();
+            String sqlString = "INSERT INTO events_to_subs (event,subscription) " +
+            		"VALUES (" + eventID + "," + subID + ");";
+            statement.executeUpdate(sqlString);
+        }
+    }
+
+    /**
+     * @param event
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     * @throws ParseException
+     */
+    private int saveEventToDatabase(Map event) throws DatabaseException, SQLException, ParseException {
+        String eventString = mapAsString(event);
+        String timestamp = (String) event.get("timestamp");
+        timestamp = convertToSQLDatetime(timestamp);
+        
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String query = "SELECT id FROM events WHERE timestamp = " + timestamp + 
+        	" AND content = '" + eventString + "';";
+        ResultSet results = statement.executeQuery(query);
+        if (results.next()) {
+            return results.getInt("id");
+        }
+        
+        String insert = "INSERT INTO events (timestamp,content) " +
+        		"VALUES (" + timestamp + ",'" + eventString + "');"; 
+        statement.executeUpdate(insert);
+        results = statement.executeQuery(query);
+        if (results.next()) {
+            return results.getInt("id");
+        }
+        throw new DatabaseException("couldn't save event to database");
+    }
+
+    /**
+     * converts a timestamp from an event to a format recognised by MySQL
+     * @param timestamp
+     * @return
+     * @throws ParseException
+     */
+    private String convertToSQLDatetime(String timestamp) throws ParseException {
+        Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z").parse(timestamp);
+        return new SimpleDateFormat("yyyyMMddHHmmss").format(date);
+    }
+
+    /**
+     * @param event
+     */
+    private String encodeSpecialChars(String string) {
+        String encodedString = string.replaceAll("&", "&amp;");
+        encodedString = encodedString.replaceAll("<", "&lt;");
+        encodedString = encodedString.replaceAll(">", "&gt;");
+        encodedString = encodedString.replaceAll("\'", "&apos;");
+        encodedString = encodedString.replaceAll("\"", "&quot;");
+        return encodedString;
+    }
+
+    /**
+     * @param subscriptionID
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public Set getEvents(Integer subscriptionID) throws DatabaseException, SQLException {
+        Set result = new HashSet(); // can't use TreeSet, because we put Maps in there, and Map isn't instanceof Comparable
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String query = "SELECT content " +
+        		"FROM events e JOIN events_to_subs ets ON (e.id = ets.event) " +
+        		"WHERE ets.subscription = " + subscriptionID.intValue() + ";";
+        ResultSet results = statement.executeQuery(query);
+        while (results.next()) {
+            Map event = stringAsMap(results.getString("content"));
+            result.add(event);
+        }
+        return result;
+    }
+
+    private String mapAsString(Map map) {
+        StringBuffer buffer = new StringBuffer();
+        for (Iterator iter = map.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) map.get(key);
+            String encodedValue = encodeSpecialChars(value);
+            buffer.append(key);
+            buffer.append(">"); // > has been encoded to &gt; in the original String
+            buffer.append(encodedValue);
+            if (iter.hasNext()) {
+                buffer.append(">>");
+            }
+        }
+        return buffer.toString();
+    }
+    
+    private Map stringAsMap(String string) {
+        Map map = new HashMap();
+        String[] lines = string.split(">>");
+        for (int i = 0; i < lines.length; i++) {
+            String line = lines[i];
+            String[] columns = line.split(">");
+            String key = columns[0];
+            String value = null;
+            if (columns.length > 1) {
+                value = columns[1];
+            }
+            map.put(key, value);
+        }
+        return map;
+    }
+    
+    public void cleanUp() {
+        // TODO implement method 
+        // delete all events that are too old (?)
+        // delete all events that don't match any subscriptions
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java	(revision 14150)
@@ -0,0 +1,215 @@
+/*
+ * Created on Nov 17, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.*;
+
+import org.apache.soap.Constants;
+import org.apache.soap.Fault;
+import org.apache.soap.rpc.*;
+import org.w3c.dom.*;
+
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class GreenstoneCommunicator {
+
+    private URL host;
+    private DocumentBuilder builder;
+
+    public GreenstoneCommunicator(URL host) throws Exception {
+        this.host = host;
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        try {
+            builder = factory.newDocumentBuilder();
+        } catch (ParserConfigurationException e) {
+            throw new Exception(e);
+        }
+
+    }
+
+    /**
+     * 
+     */
+    public GreenstoneCommunicator() {
+        
+        // TODO Auto-generated constructor stub
+    }
+
+    public String[] getHostNames() {
+        // TODO method implementation
+        return new String[] {"localhost"};
+    }
+    
+    public String[] getCollectionNames() throws Exception {
+        Document document = createSubsetDescriptionRequest("", "collectionList");
+
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList nodeList = response.getElementsByTagName("collection");
+        String[] result = new String[nodeList.getLength()];
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            Node name = node.getAttributes().getNamedItem("name");
+            result[i] = name.getNodeValue();
+        }
+        
+	System.out.println(ArrayHelper.toString(result));
+        
+        return result;
+    }
+    
+    public boolean supportsTextSearch(String collection) throws Exception {
+        Document document = createSubsetDescriptionRequest(collection, "serviceList");
+        
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList nodeList = response.getElementsByTagName("service");
+        
+        boolean hasTextQuery = false;
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String name = node.getAttributes().getNamedItem("name").getNodeValue(); 
+            if (name.equals("TextQuery")) {
+                hasTextQuery = true;
+                break;
+            }
+        }
+        
+        if (!hasTextQuery) return false; // doesn't support text queries at all
+        
+        document = createSubsetDescriptionRequest(collection + "/TextQuery", "paramList");
+        
+        response = sendToGreenstone(document.getDocumentElement());
+        
+        nodeList = response.getElementsByTagName("param");
+        
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String name = node.getAttributes().getNamedItem("name").getNodeValue();
+            if (name == null) break;
+            String value = node.getAttributes().getNamedItem("type").getNodeValue();
+            if (value == null) break;
+            if (name.equals("query") && value.equals("string")) {
+                return true;
+            }
+        }
+        
+        return false;
+    }
+    
+    public Set fullTextSearch(String collection, String query) throws Exception {
+        Document document = builder.newDocument();
+        Element root = document.createElement("message");
+        document.appendChild(root);
+    
+        Element requestElement = document.createElement("request");
+        requestElement.setAttribute("lang", "en");
+        requestElement.setAttribute("type", "process");
+        requestElement.setAttribute("to", collection + "/TextQuery");
+        root.appendChild(requestElement);
+    
+        Element paramListElement = document.createElement("paramList");
+        requestElement.appendChild(paramListElement);
+    
+        Element paramElement = document.createElement("param");
+        paramElement.setAttribute("name", "query");
+        paramElement.setAttribute("value", query);
+        // BTS 3 what if numReturnedDocs < numMatchedDocs
+        paramListElement.appendChild(paramElement);
+        
+        
+        Set result = new TreeSet();
+        //DOM2Writer.serializeAsXML(root, new OutputStreamWriter(System.out));
+        
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList documentNodes = response.getElementsByTagName("documentNode");
+        
+        for(int i = 0; i < documentNodes.getLength(); i++) {
+            Node node = documentNodes.item(i);
+            String docID = node.getAttributes().getNamedItem("nodeID").getNodeValue();
+            Matcher matcher = Pattern.compile("(\\d{8}:\\d+).*").matcher(docID);
+            if (matcher.matches()) {
+                docID = matcher.group(1);
+            }
+            result.add(docID);
+        }
+        
+        return result;
+    }
+    
+    private Element sendToGreenstone(Element message) throws Exception {
+        Call call = new Call();
+        // TODO get the uri from a configuration file
+        call.setTargetObjectURI("localsite");
+        call.setMethodName("process");
+        // set Encoding Style to use literal XML
+        call.setEncodingStyleURI(Constants.NS_URI_LITERAL_XML);
+        // Set Method Parameters - use literal xml
+        Parameter param = new Parameter("message", 
+                org.w3c.dom.Element.class,
+                message, 
+                Constants.NS_URI_LITERAL_XML);
+
+        Vector paramList = new Vector();
+        paramList.addElement(param);
+        call.setParams(paramList);
+
+        // Invoke the Service
+        Response response = call.invoke(host, "");
+
+        // Check for Faults
+        if (!response.generatedFault()) { // no error
+            // Extract Return value
+            Parameter result = response.getReturnValue();
+            return (Element) result.getValue();
+        } else {
+            //  Extract Fault Code and String
+            Fault f = response.getFault();
+            String faultCode = f.getFaultCode();
+            String faultString = f.getFaultString();
+            throw new Exception(faultCode + ": Fault occured: " + faultString);
+        }
+    }
+
+    /**
+     * @return
+     */
+    private Document createSubsetDescriptionRequest(String object, String subsetType) {
+        Document document = builder.newDocument();
+        Element root = document.createElement("message");
+        document.appendChild(root);
+    
+        Element requestElement = document.createElement("request");
+        requestElement.setAttribute("lang", "en");
+        requestElement.setAttribute("type", "describe");
+        requestElement.setAttribute("to", object);
+        root.appendChild(requestElement);
+    
+        Element paramListElement = document.createElement("paramList");
+        requestElement.appendChild(paramListElement);
+    
+        Element paramElement = document.createElement("param");
+        paramElement.setAttribute("name", "subset");
+        paramElement.setAttribute("value", subsetType);
+        paramListElement.appendChild(paramElement);
+        return document;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java	(revision 14150)
@@ -0,0 +1,77 @@
+/*
+ * Created on Nov 23, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.util.*;
+
+import org.greenstone.gsdlas.profiles.Subscription;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class NotificationStore {
+
+    private static NotificationStore instance;
+    private Map eventsForSubscriptionID = new TreeMap();
+    
+    private NotificationStore() {
+    }
+    
+    /**
+     * @return
+     */
+    public static NotificationStore getInstance() {
+        if (instance == null) instance = new NotificationStore();
+        return instance;
+    }
+    
+    public void add(Map event, Set subscriptions) {
+        encodeValues(event);
+        for (Iterator iter = subscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            Integer subscriptionID = new Integer(sub.getId()); 
+            if (!eventsForSubscriptionID.containsKey(subscriptionID)) {
+                eventsForSubscriptionID.put(subscriptionID, new HashSet());
+            }
+            ((Set) eventsForSubscriptionID.get(subscriptionID)).add(event);
+        }
+    }
+
+    /**
+     * @param event
+     */
+    private void encodeValues(Map event) {
+        for (Iterator iter = event.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) event.get(key);
+            String encodedValue = value.replaceAll("&", "&amp;");
+            encodedValue = encodedValue.replaceAll("<", "&lt;");
+            encodedValue = encodedValue.replaceAll(">", "&gt;");
+            encodedValue = encodedValue.replaceAll("\'", "&apos;");
+            encodedValue = encodedValue.replaceAll("\"", "&quot;");
+            
+            event.put(key, encodedValue);
+        }
+    }
+
+    /**
+     * @param subscriptionID
+     * @return
+     */
+    public Set getEvents(Integer subscriptionID) {
+        if (eventsForSubscriptionID.containsKey(subscriptionID)) {
+            return Collections.unmodifiableSet((Set) eventsForSubscriptionID.get(subscriptionID));
+        }
+        return new TreeSet();
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/Notifier.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/Notifier.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/Notifier.java	(revision 14150)
@@ -0,0 +1,96 @@
+/*
+ * Created on Jan 1, 2005
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.util.*;
+import java.util.Map;
+import java.util.Set;
+
+import javax.mail.MessagingException;
+
+import org.greenstone.gsdlas.profiles.Subscription;
+import org.greenstone.gsdlas.util.Mailer;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ * 
+ * Singleton
+ */
+public class Notifier {
+
+    private static Notifier instance; 
+    
+    private Notifier () {
+        // hide default constructor
+    }
+    
+    /**
+     * @return
+     */
+    public static Notifier getInstance() {
+        if (instance == null) instance = new Notifier();
+        return instance;
+    }
+
+    /**
+     * @param event
+     * @param matchedSubscriptions
+     */
+    public void sendNotifications(Map event, Set matchedSubscriptions) {
+        // TODO read smtp host and sender from configuration file
+        Mailer mailer = new Mailer(System.getProperty("user.name") + "@localhost", "localhost");
+        String eventText = eventToEmail(event);
+        String subject = "Event for your subscription";
+        
+        for (Iterator iter = matchedSubscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            if (!sub.wantsEMailNotification()) continue;
+
+            StringBuffer message = new StringBuffer();
+            // TODO formatting of notification email via velocity?
+            message.append("You've created a subscription at the Greenstone Alerting Service running on " +
+            		"localhost:\n\n");
+            message.append(sub.toString());
+            message.append("\n");
+            message.append("An event has occurred that matches this subscription:\n\n");
+            message.append(eventText);
+            message.append("\n");
+            message.append("Go to http://localhost:8080/alerting/service?action=listSubscriptions to " +
+            		"change your subscriptions, or to http://localhost:8080/alerting/service?action=showEvents&subscriptionID=" + 
+            		sub.getId() + " to see all events that match the subscription.\n");
+            
+            String to = sub.getMailAddress();
+            
+            try {
+                mailer.sendMail(to, subject, message.toString());
+            } catch (MessagingException e) {
+                System.err.println("Couldn't notify for subscription " + sub.getId() + ": " + e.getMessage());
+                e.printStackTrace();
+            }
+        }
+    }
+
+    /**
+     * @param event
+     * @return
+     */
+    private String eventToEmail(Map event) {
+        StringBuffer message = new StringBuffer();
+        for (Iterator iter = event.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) event.get(key);
+            message.append(key + ": " + value + "\n");
+        }
+        return message.toString();
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java	(revision 14150)
@@ -0,0 +1,416 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.*;
+
+import javax.servlet.http.HttpSession;
+
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.profiles.*;
+
+/**
+ * Storage for all Profiles. Singleton.
+ * 
+ * @author schweer
+ *  
+ */
+public class ProfileStore {
+
+    private static ProfileStore instance;
+
+    private Set noEqualsSubscriptions = new TreeSet();
+    private Map dontCareSubscriptions = new TreeMap();
+    private Map subscriptions = new TreeMap();
+    
+
+    private ProfileStore() {
+        // hide default constructor
+    }
+
+    /**
+     * @return the one existing instance of this class.
+     */
+    public static ProfileStore getInstance() {
+        if (instance == null) {
+            instance = new ProfileStore();
+        }
+        return instance;
+    }
+
+    /**
+     * @param subscription
+     *            the subscription to add to the profile store.
+     */
+    private void addSubscription(Subscription subscription) {
+        List documentIdPredicate = subscription.getPredicateList(Constants.DOCUMENT_ID_FIELD);
+        List collectionIdPredicate = subscription.getPredicateList(Constants.COLLECTION_ID_FIELD);
+        List hostIdPredicate = subscription.getPredicateList(Constants.HOST_ID_FIELD);
+        List typePredicate = subscription.getPredicateList(Constants.TYPE_FIELD);
+        
+        Integer subId = new Integer(subscription.getId());
+        
+        if (documentIdPredicate.isEmpty()) {
+            if (!dontCareSubscriptions.containsKey(Constants.DOCUMENT_ID_FIELD)) {
+                dontCareSubscriptions.put(Constants.DOCUMENT_ID_FIELD, new TreeSet());
+            }
+            Set list = (Set) dontCareSubscriptions.get(Constants.DOCUMENT_ID_FIELD);
+            list.add(subId);
+        }
+        if (hostIdPredicate.isEmpty()) {
+            if (collectionIdPredicate.isEmpty()) {
+                if (!dontCareSubscriptions.containsKey(Constants.COLLECTION_ID_FIELD)) {
+                    dontCareSubscriptions.put(Constants.COLLECTION_ID_FIELD, new TreeSet());
+                }
+                Set list = (Set) dontCareSubscriptions.get(Constants.COLLECTION_ID_FIELD);
+                list.add(subId);
+            }
+        }
+        if (typePredicate.isEmpty()) {
+            if (!dontCareSubscriptions.containsKey(Constants.TYPE_FIELD)) {
+                dontCareSubscriptions.put(Constants.TYPE_FIELD, new TreeSet());
+            }
+            Set list = (Set) dontCareSubscriptions.get(Constants.TYPE_FIELD);
+            list.add(subId);
+        }
+        
+        if (documentIdPredicate.isEmpty() && collectionIdPredicate.isEmpty() 
+                && hostIdPredicate.isEmpty() && typePredicate.isEmpty()) {
+            noEqualsSubscriptions.add(subId);
+        }
+        subscriptions.put(subId, subscription);
+    }
+
+    /**
+     * @return
+     */
+    public Set getSubscriptionsWithoutEqualsPredicates() {
+        return Collections.unmodifiableSet(noEqualsSubscriptions);
+    }
+
+    /**
+     * Computes the list of equality subscriptions satisfied by the event.
+     * @param event
+     * @return
+     */
+    private Set getPartiallyMatchedSubscriptions(Map event) {
+        Set result = new TreeSet();
+        
+        EqualsPredicate predicate;
+        
+        // all subscriptions with matching type...
+        predicate = PredicateFactory.getEqualsPredicate(Constants.TYPE_FIELD, 
+                (String)event.get(Constants.TYPE_FIELD));
+        if (predicate != null)
+            result.addAll(predicate.getSubscriptionIDs());
+        
+        // ...but only those subscriptions where the doc id matches...
+        Set docIDSubs = new TreeSet();
+        predicate = PredicateFactory.getEqualsPredicate(Constants.DOCUMENT_ID_FIELD, 
+                (String)event.get(Constants.DOCUMENT_ID_FIELD));
+        if (predicate != null)
+            docIDSubs.addAll(predicate.getSubscriptionIDs());
+        docIDSubs.addAll(getDontCareSubscriptions(Constants.DOCUMENT_ID_FIELD));
+        result.retainAll(docIDSubs);
+        
+        // ...and only those subscriptions where the host/collection id matches:
+        Set hostCollIDSubs = new TreeSet();
+        //   i.e.: host|coll is "don't care"...
+        hostCollIDSubs.addAll(getDontCareSubscriptions(Constants.COLLECTION_ID_FIELD));
+        //   ...or host matches (-> coll is "don't care" for this host)...
+        predicate = PredicateFactory.getEqualsPredicate(Constants.HOST_ID_FIELD, 
+                (String)event.get(Constants.HOST_ID_FIELD));
+        if (predicate != null)
+            hostCollIDSubs.addAll(predicate.getSubscriptionIDs());
+        //   ...or host|coll matches
+        predicate = PredicateFactory.getEqualsPredicate(Constants.COLLECTION_ID_FIELD, 
+                (String)event.get(Constants.COLLECTION_ID_FIELD));
+        if (predicate != null)
+            hostCollIDSubs.addAll(predicate.getSubscriptionIDs());
+        
+        result.retainAll(hostCollIDSubs);
+        
+        return result;
+    }
+
+    /**
+     * 
+     * @param gsComm 
+     * @return
+     * @throws UnsupportedFieldException
+     */
+    private Set findMatchedQueryPredicates(Map event, GreenstoneCommunicator gsComm) {
+        Set matchedPreds = new TreeSet();
+        
+        // iterate through all document content predicates
+        // execute the query stored in the predicate
+        // predicate is matched iff the docID occurs in the query result
+        
+        for (Iterator iter = PredicateFactory.getQueryPredicates(Constants.DOCUMENT_CONTENT_FIELD).iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            String collection = (String) event.get(Constants.COLLECTION_ID_FIELD);
+            String query = predicate.getValue();
+            Set results;
+            try {
+                results = gsComm.fullTextSearch(collection, query);
+                if (results.contains(event.get(Constants.DOCUMENT_ID_FIELD))) {
+                    matchedPreds.add(predicate);
+                }
+            } catch (Exception e) {
+            }
+        }
+        
+        return matchedPreds;
+    }
+
+    /**
+     * Computes the set of SubstringMatchPredicates satisfied by the event.
+     * @param event
+     * 
+     * @return
+     */
+    private Set findMatchedSubstringMatchPredicates(Map event) {
+        Set result = new TreeSet();
+        
+        for (Iterator iter = PredicateFactory.getAllSubstringMatchPredicates().iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            if (predicate.isSatisfied(event)) {
+                result.add(predicate);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Filters the event against the profiles. It uses the equality-preferred
+     * algorithm as described in Fabret, Llirbat, Pereira and Shasha:
+     * <em>Efficient Matching for Content-based Publish/Subscribe 
+     * Systems</em>.
+     * @param event
+     * @param gsComm 
+     */
+    public Set filter(Map event, GreenstoneCommunicator gsComm) {
+        Set matchedSubscriptions = new TreeSet();
+    
+        Set partiallyMatchedSubscriptions 
+        	= getPartiallyMatchedSubscriptions(event);
+    
+        Set matchedPredicates = findMatchedSubstringMatchPredicates(event);
+        matchedPredicates.addAll(findMatchedQueryPredicates(event, gsComm));
+    
+        Map numMatchedPredicates = new TreeMap();
+        for (Iterator iter = matchedPredicates.iterator(); iter.hasNext();) {
+            Predicate pred = (Predicate) iter.next();
+            for (Iterator iterator = pred.getSubscriptionIDs().iterator(); iterator
+                    .hasNext();) {
+                Integer subId = (Integer) iterator.next();
+                Subscription sub = (Subscription) subscriptions.get(subId);
+                Integer count = (Integer) numMatchedPredicates.get(subId);
+                int newCountValue = (count == null ? 1 : count.intValue() + 1);
+                numMatchedPredicates.put(subId,
+                        new Integer(newCountValue));
+            }
+        }
+    
+        Set notUnmatchedSubscriptions = partiallyMatchedSubscriptions;
+        notUnmatchedSubscriptions.addAll(noEqualsSubscriptions);
+        for (Iterator iter = notUnmatchedSubscriptions.iterator(); iter
+                .hasNext();) {
+            Integer subId = (Integer) iter.next();
+            Subscription sub = (Subscription) subscriptions.get(subId);
+            int matchedPredicatesCount = 0;
+            if (numMatchedPredicates.get(subId) != null) {
+                matchedPredicatesCount = ((Integer) numMatchedPredicates.get(subId)).intValue();
+            }
+            if (matchedPredicatesCount == sub.getNumOfNonEqualsPredicates()) {
+                matchedSubscriptions.add(sub);
+            }
+        }
+        return matchedSubscriptions;
+    }
+
+
+    /**
+     * @param arguments
+     * @throws Exception
+     */
+    public void createSubscription(Map arguments) throws Exception {
+        Subscription sub = new Subscription(arguments);
+        addSubscription(sub);
+    }
+
+    
+    /**
+     * @param subscriptionID
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public void deleteSubscription(String subscriptionID) throws DatabaseException, SQLException {
+        Integer subID = new Integer(subscriptionID);
+        System.out.println("deleting subscription " + subscriptionID);
+        if (!subscriptions.containsKey(subID)) {
+            return;
+        }
+        Subscription sub = (Subscription) subscriptions.get(subID);
+        for (Iterator iter = dontCareSubscriptions.values().iterator(); iter.hasNext();) {
+            Set entry = (Set) iter.next();
+            if (entry.contains(subID))
+                entry.remove(subID);
+        }
+        if (noEqualsSubscriptions.contains(subID))
+            noEqualsSubscriptions.remove(subID);
+        for (Iterator iter = sub.getPredicates().iterator(); iter.hasNext();) {
+            Predicate pred = (Predicate) iter.next();
+            pred.removeSubscription(subID);
+        }
+
+        subscriptions.remove(subID);
+        
+        
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        
+        Statement statement = conn.createStatement();
+        
+        String sqlString = "DELETE FROM subscriptions WHERE id = " + subscriptionID;
+        statement.executeUpdate(sqlString);
+        
+        sqlString = "SELECT predicate FROM subs_to_predicates " +
+        		"WHERE subscription = " + subscriptionID;
+        ResultSet predicates = statement.executeQuery(sqlString);
+        while (predicates.next()) {
+            // if there aren't any other subscriptions using this predicate, delete predicate
+            int predicateID = predicates.getInt("predicate");
+            
+        }
+        
+        
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     */
+    public void changeSubscription(Map arguments, HttpSession session) {
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        // BTS 14 really edit the subscription
+        
+    }
+
+    /**
+     * @param username
+     * @return
+     * @throws DatabaseException
+     */
+    public Set getAllSubscriptionsFor(String username) throws DatabaseException {
+        Set result = new TreeSet();
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            String sqlString = "SELECT id FROM subscriptions " +
+            		"WHERE user like '" + username + "';";
+            ResultSet idList = statement.executeQuery(sqlString);
+            while (idList.next()) {
+                int id = idList.getInt("id");
+                result.add(subscriptions.get(new Integer(id)));
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        
+        return result;
+    }
+
+    /**
+     * @param string
+     * @return
+     */
+    private Set getDontCareSubscriptions(String field) {
+        Set result = new TreeSet();
+        if (dontCareSubscriptions.containsKey(field)) {
+            result = (Set) dontCareSubscriptions.get(field);
+        }
+        return result;
+    }
+
+    /**
+     * 
+     */
+    public void restoreFromDatabase() {
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            String sqlString = "SELECT * FROM subscriptions";
+            ResultSet subs = statement.executeQuery(sqlString);
+            while (subs.next()) {
+                // construct map for all predicates
+                int id = subs.getInt("id");
+                Map map = new TreeMap();
+                map.put("username", subs.getString("user"));
+                map.put("email", subs.getString("email"));
+                map.put("subscription_name", subs.getString("name"));
+                Vector ways = new Vector();
+                if (subs.getInt("rss") != 0) {
+                    ways.add("rss");
+                }
+                if (subs.getInt("page") != 0) {
+                    ways.add("page");
+                }
+                map.put("way", ways);
+                Statement stmnt = conn.createStatement();
+                sqlString = "SELECT p.type, p.field, p.value from subs_to_predicates stp join predicates p on(stp.predicate = p.id) where stp.subscription = " + id + ";";
+                ResultSet predicates = stmnt.executeQuery(sqlString);
+                while (predicates.next()) {
+                    String type = predicates.getString(Constants.TYPE_FIELD);
+                    String field = predicates.getString("field");
+                    if (type.equals(SubstringMatchPredicate.class.getName())) {
+                        if (field.equals(Constants.COLLECTION_ID_FIELD)) {
+                            field = Constants.COLLECTION_QUERY_FIELD;
+                        } else if (field.equals(Constants.HOST_ID_FIELD)) {
+                            field = Constants.HOST_QUERY_FIELD;
+                        }
+                    }
+                    String value = predicates.getString("value");
+                    if (type.equals(EqualsPredicate.class.getName())) {
+                        if (!map.containsKey(field)) {
+                            map.put(field, new Vector());
+                        }
+                        ((List) map.get(field)).add(value);
+                    } else {
+                        map.put(field, value);
+                    }
+                }
+                try {
+                    createSubscription(map);
+                } catch (Exception e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                }
+            }
+        } catch (SQLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * @param i
+     * @return
+     */
+    public Subscription getSubscription(int id) {
+        return (Subscription) subscriptions.get(new Integer(id));
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java	(revision 14150)
@@ -0,0 +1,21 @@
+/*
+ * Created on Nov 8, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UnsupportedFieldException extends Exception {
+    public UnsupportedFieldException(String msg) {
+        super(msg);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java	(revision 14150)
@@ -0,0 +1,29 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.database;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class DatabaseException extends Exception {
+    
+    public DatabaseException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * @param string
+     */
+    public DatabaseException(String message) {
+        super(message);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java	(revision 14150)
@@ -0,0 +1,56 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.database;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class DatabaseManager {
+
+    private static DatabaseManager instance;
+    private Connection database;
+
+    private DatabaseManager() {
+        // hide constructor
+    }
+    
+    public static DatabaseManager getInstance() {
+        if (instance == null) {
+            instance = new DatabaseManager();
+        }
+        return instance;
+    }
+    
+    public Connection getDatabaseConnection() throws DatabaseException {
+        if(database == null) {
+            String connectionString = "jdbc:mysql://localhost/alerting?user=gsdl3admin";
+            try {
+                Class.forName("com.mysql.jdbc.Driver").newInstance();
+                database = DriverManager.getConnection(connectionString);
+            } catch (Exception e) {
+                if (e instanceof SQLException) {
+                    SQLException sqle = (SQLException) e;
+                    System.err.println("SQLState: " + sqle.getSQLState());
+                }
+                throw new DatabaseException("Could not establish connection to database (" + connectionString + ")", e);
+            }
+        }
+        return database;
+    }
+    
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java	(revision 14150)
@@ -0,0 +1,35 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.Map;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class EqualsPredicate extends Predicate {
+
+    public EqualsPredicate(String field, String id) {
+        super.field = field;
+        super.value = id;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#isSatisfied(java.util.Map)
+     */
+    public boolean isSatisfied(Map event) {
+        String value = (String) event.get(field);
+        return value != null && value.equals(value);
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java	(revision 14150)
@@ -0,0 +1,26 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+/**
+ * Exception thrown when a Predicate cannot be created due to a syntax error.
+ * @author schweer
+ * 
+ */
+public class ParseException extends Exception {
+
+    /**
+     * @param message the exception message.
+     * @see Exception#Exception(String)
+     */
+    public ParseException(String message) {
+        super(message);
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java	(revision 14150)
@@ -0,0 +1,175 @@
+/*
+ * Created on Nov 2, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.*;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.*;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public abstract class Predicate implements Comparable {
+
+    protected Set subscriptions = new TreeSet();
+    protected String field;
+    protected String value;
+    protected int id;
+    
+
+    public static final String[] singleValueFields = new String[] {
+            Constants.DOCUMENT_TITLE_FIELD,
+            Constants.DOCUMENT_CONTENT_FIELD,
+            "metadata_has_field",
+            Constants.HOST_QUERY_FIELD,
+            Constants.COLLECTION_QUERY_FIELD };
+    public static final String[] multiValueFields = new String[] {
+            Constants.TYPE_FIELD,
+            Constants.HOST_ID_FIELD,
+            Constants.COLLECTION_ID_FIELD,
+            Constants.HOST_COLLECTION_ID_FIELD,
+            Constants.DOCUMENT_ID_FIELD };
+    
+    public void addSubscription(int subscriptionID) {
+        subscriptions.add(new Integer(subscriptionID));
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getSubscriptions()
+     */
+    public Set getSubscriptionIDs() {
+        return Collections.unmodifiableSet(subscriptions);
+    }
+
+    public int getID() {
+        return id;
+    }
+    
+    public void setID(int i) {
+        this.id = i;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getKey()
+     */
+    public String getField() {
+        return field;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getValue()
+     */
+    public String getValue() {
+        return value;
+    }
+
+    public String toString() {
+        return field + "=" + value;
+    }
+    
+    public boolean equals(Object other) {
+        if (other == null || !(other instanceof Predicate)) return false;
+        return id == ((Predicate)other).getID();
+    }
+    
+    public int compareTo(Object other) {
+        Predicate otherPredicate = (Predicate) other;
+        return new Integer(id).compareTo(new Integer(otherPredicate.getID()));
+    }
+    
+    public abstract boolean isSatisfied(Map event);
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isSingleValued(String key) {
+        return ArrayHelper.contains(singleValueFields, key);
+    }
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isMultiValued(String key) {
+        return ArrayHelper.contains(multiValueFields, key);
+    }
+
+    /**
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    int saveToDatabase() throws SQLException, DatabaseException {
+        String query = "SELECT id FROM predicates " +
+        		"WHERE field = '" + field + 
+        		"' AND value = '" + value + 
+        		"' AND type = '" + this.getClass().getName() + "';";
+        Statement statement = DatabaseManager.getInstance().getDatabaseConnection().createStatement();
+        System.out.println(query);
+        ResultSet result = statement.executeQuery(query);
+        if (result.next()) { // predicate already exists in database
+            int id = result.getInt("id"); 
+            System.out.println("predicate already exists with id " + id);
+            return id;
+        }
+        // predicate has been newly created
+        String insert = "INSERT INTO predicates (field, type, value) " +
+        "VALUES ('" + field + "', '" + this.getClass().getName() + "', '" + value + "');";
+        System.out.println(insert);
+        statement.execute(insert);
+        result = statement.executeQuery(query);
+        if (result.next()) {
+            return result.getInt("id");
+        } else {
+            throw new DatabaseException("could not save predicate");
+        }
+    }
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isFieldName(String key) {
+        return ArrayHelper.contains(singleValueFields, key) || ArrayHelper.contains(multiValueFields, key);
+    }
+
+    /**
+     * @param subID
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public void removeSubscription(Integer subID) throws SQLException, DatabaseException {
+        subscriptions.remove(subID);
+
+        Statement statement = DatabaseManager.getInstance().getDatabaseConnection().createStatement();
+        String sqlString = "DELETE FROM subs_to_predicates " +
+        	"WHERE subscription = " + subID + 
+        	" AND predicate = " + id;
+        statement.executeUpdate(sqlString);
+        
+        if (subscriptions.isEmpty()) {
+            PredicateFactory.deletePredicate(this);
+            sqlString = "DELETE FROM predicates WHERE id = " + id;
+            statement.executeUpdate(sqlString);
+        }
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java	(revision 14150)
@@ -0,0 +1,245 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.SQLException;
+import java.util.*;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public abstract class PredicateFactory {
+
+    private static int id = 0; 
+    
+    private static Map equalsPredicates = new TreeMap();
+    
+    // substring for collection_name -> SubstringMatchPredicates referring to it
+    private static Map collectionNameMatchPredicates = new TreeMap();
+    private static Map hostNameMatchPredicates = new TreeMap();
+    
+    // query -> QueryPredicates referring to it
+    private static Map documentTitleQueryPredicates = new TreeMap();
+    private static Map documentContentQueryPredicates = new TreeMap();
+    
+    
+    /**
+     * @param key
+     * @param value
+     * @return
+     * @throws ParseException
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public static Predicate createPredicate(String key, String value)
+            throws SQLException, DatabaseException {
+        System.out.println("creating predicate for key " + key + " and value " + value);
+        if (!Predicate.isSingleValued(key)) {
+            return null;
+        }
+        Predicate result = null;
+        if (value == null || value.length() == 0) {
+            return null;
+        }
+        
+        if (Predicate.isMultiValued(key)) {
+            List list = new ArrayList();
+            list.add(value);
+            return (Predicate) createPredicates(key, list).get(0);
+        } else if (key.startsWith("document")) {
+            result = createQueryPredicate(key, value);
+        } else if (key.endsWith("_query")) {
+            result = createSubstringMatchPredicate(key, value);
+        } else {
+        	return null;
+        }
+        result.setID(result.saveToDatabase());
+        return result;
+    }
+    
+    public static List createPredicates(String key, List values) throws SQLException, DatabaseException {
+        if (!Predicate.isMultiValued(key)) {
+            return null;
+        }
+        List results = createEqualsPredicates(key, values);
+        for (Iterator iter = results.iterator(); iter.hasNext();) {
+            EqualsPredicate pred = (EqualsPredicate) iter.next();
+
+            pred.setID(pred.saveToDatabase());
+        }
+        return results;
+    }
+
+    /**
+     * @param key
+     * @param value
+     * @return
+     */
+    private static SubstringMatchPredicate createSubstringMatchPredicate(String key, String value) {
+        SubstringMatchPredicate predicate = null;
+        if (key.equals(Constants.COLLECTION_QUERY_FIELD)) {
+            if (collectionNameMatchPredicates.containsKey(value)) {
+                predicate = (SubstringMatchPredicate) collectionNameMatchPredicates.get(value);
+            } else {
+                predicate = new SubstringMatchPredicate(Constants.COLLECTION_ID_FIELD, value);
+                collectionNameMatchPredicates.put(value, predicate);
+            }
+        } else if (key.equals(Constants.HOST_QUERY_FIELD)) {
+            if (hostNameMatchPredicates.containsKey(value)) {
+                predicate = (SubstringMatchPredicate) hostNameMatchPredicates.get(value);
+            } else {
+                predicate = new SubstringMatchPredicate(Constants.HOST_ID_FIELD, value);
+                hostNameMatchPredicates.put(value, predicate);
+            }
+        }
+        // TODO other fields?
+        return predicate;
+    }
+
+    /**
+     * @param key
+     * @param values
+     * @return
+     */
+    private static List createEqualsPredicates(String key, List values) {
+        List result = new Vector();
+        
+        Map keyToPredicates;
+        if (!equalsPredicates.containsKey(key)) {
+            keyToPredicates = new HashMap();
+            equalsPredicates.put(key, keyToPredicates);
+        }
+        keyToPredicates = (Map) equalsPredicates.get(key);
+        
+        for (Iterator iter = values.iterator(); iter.hasNext();) {
+            String value = (String) iter.next();
+            if (value == null || value.length() == 0) {
+                continue;
+            }
+
+            if (key.equals(Constants.HOST_ID_FIELD) && !(value.startsWith("http://"))) {
+                value = "http://" + value + ":8080/soap/servlet/rpcrouter";
+            }
+            
+            EqualsPredicate predicate = null;
+            if (keyToPredicates.containsKey(value)) {
+                predicate = (EqualsPredicate) keyToPredicates.get(value);
+            } else {
+                predicate = new EqualsPredicate(key, value);
+                keyToPredicates.put(value, predicate);
+            }
+            result.add(predicate);
+        }
+        return result;
+    }
+    
+    /**
+     * @param field
+     * @param query
+     * @return
+     */
+    private static QueryPredicate createQueryPredicate(String field,
+            String query) {
+        QueryPredicate predicate = null;
+        if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+            if (!documentContentQueryPredicates.containsKey(query)) {
+                predicate = new QueryPredicate(field, query);
+                documentContentQueryPredicates.put(query, predicate);
+            }
+            return (QueryPredicate) documentContentQueryPredicates.get(query);
+        } else if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+            if (!documentTitleQueryPredicates.containsKey(query)) {
+                predicate = new QueryPredicate(field, query);
+                documentTitleQueryPredicates.put(query, predicate);
+            }
+            return (QueryPredicate) documentTitleQueryPredicates.get(query);
+        }
+        return predicate;
+    }
+
+    public static Collection getAllQueryPredicates() {
+        Set result = new TreeSet();
+        result.addAll(documentTitleQueryPredicates.values());
+        result.addAll(documentContentQueryPredicates.values());
+        return Collections.unmodifiableSet(result);
+    }
+    
+    public static Collection getQueryPredicates(String field) {
+        Set result = new TreeSet();
+        if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+            result.addAll(documentTitleQueryPredicates.values());
+        } else if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+            result.addAll(documentContentQueryPredicates.values());
+        }
+        return Collections.unmodifiableSet(result);
+    }
+    
+    public static Collection getAllSubstringMatchPredicates() {
+        Collection result = new Vector();
+        result.addAll(hostNameMatchPredicates.values());
+        result.addAll(collectionNameMatchPredicates.values());
+        return Collections.unmodifiableCollection(result);
+    }
+
+    /**
+     * @param string
+     * @param string2
+     * @return
+     */
+    public static EqualsPredicate getEqualsPredicate(String key, String value) {
+        EqualsPredicate result = null; 
+        if (equalsPredicates.containsKey(key)) {
+            result = (EqualsPredicate) ((Map)equalsPredicates.get(key)).get(value);
+        }
+        return result;
+    }
+
+    /**
+     * @param predicate
+     */
+    public static void deletePredicate(Predicate predicate) {
+        String field = predicate.getField();
+        String value = predicate.getValue();
+        
+        if (predicate instanceof EqualsPredicate) {
+            Map map = (Map) equalsPredicates.get(field);
+            if (map.containsKey(value)) {
+                map.remove(value);
+            }
+        } else if (predicate instanceof QueryPredicate) {
+            if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+                documentTitleQueryPredicates.remove(value);
+            } else if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+                documentContentQueryPredicates.remove(value);
+            } else {
+                System.err.println("problem while removing: unknown field " + field);
+            }
+        } else if (predicate instanceof SubstringMatchPredicate) {
+            if (field.equals(Constants.HOST_ID_FIELD)) {
+                hostNameMatchPredicates.remove(value);
+            } else if (field.equals(Constants.COLLECTION_ID_FIELD)) {
+                collectionNameMatchPredicates.remove(value);
+            } else {
+                System.err.println("problem while removing: unknown field " + field);
+            }
+        } else {
+            System.err.println("problem while removing: unknown class " + predicate.getClass().getName());
+        }
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java	(revision 14150)
@@ -0,0 +1,33 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.List;
+import java.util.Vector;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class Profile {
+    private List subscriptions;
+
+    private String user;
+
+    public Profile(String user) {
+        this.user = user;
+        subscriptions = new Vector();
+    }
+
+    public void add(Subscription sub) {
+        subscriptions.add(sub);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java	(revision 14150)
@@ -0,0 +1,37 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.*;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class QueryPredicate extends Predicate {
+    
+    private Set containsWords = new TreeSet();
+    private Set notContainsWords = new TreeSet();
+    
+    public QueryPredicate (String field, String query) {
+        super.field = field;
+        super.value = query;
+    }
+    public boolean isSatisfied(Map event) {
+        return true;
+    }
+    
+    public String toString() {
+        return field + " matches " + value;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java	(revision 14150)
@@ -0,0 +1,300 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.*;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class Subscription implements Comparable {
+    private Map map;
+    
+    private int id;
+    private String username;
+    private String name;
+    private String email;
+    private boolean rssNotification;
+    private boolean eventsSincePageNotification;
+    
+    private int numOfNonEqualsPredicates;
+    
+    public Subscription(Map valueMap) throws DatabaseException, SQLException {
+        valueMap = cleanHostCollectionIDs(valueMap);
+        map = new TreeMap();
+        for (Iterator iter = valueMap.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            
+            if (!Predicate.isFieldName(key)) {
+                continue;
+            }
+            
+            Object value = valueMap.get(key);
+            
+            if (value instanceof String) {
+                Predicate predicate = PredicateFactory.createPredicate(key, (String) value);  
+                map.put(key, predicate);
+
+                if (predicate != null && !(predicate instanceof EqualsPredicate)) {
+                    numOfNonEqualsPredicates++;
+                }
+            } else if (value instanceof List) {
+                List values = (List)value;
+                List predicates = PredicateFactory.createPredicates(key, values);
+                map.put(key, predicates);
+                for (Iterator iterator = predicates.iterator(); iterator
+                        .hasNext();) {
+                    Predicate predicate = (Predicate) iterator.next();
+                    
+                    if (predicate != null && !(predicate instanceof EqualsPredicate)) {
+                        numOfNonEqualsPredicates++;
+                    }
+                }
+            } 
+            
+        }
+        
+        System.out.println("finished creating predicates");
+        
+        username = (String) valueMap.get("username");
+        name = (String) valueMap.get("subscription_name");
+        email = (String) valueMap.get("email");
+        rssNotification = valueMap.containsKey("way") && ((List)valueMap.get("way")).contains("rss");
+        eventsSincePageNotification = valueMap.containsKey("way") && ((List)valueMap.get("way")).contains("page");
+        id = saveToDatabase();
+        
+        for (Iterator iter = getPredicates().iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            if (predicate != null) {
+                predicate.addSubscription(id);
+            }
+        }
+    }
+    
+    /**
+     * @param valueMap
+     * @return
+     */
+    private Map cleanHostCollectionIDs(Map valueMap) {
+        Map result = new TreeMap();
+        Pattern pattern = Pattern.compile("^(.*)\\|(.*)$");
+        for (Iterator iter = valueMap.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (!key.equals(Constants.HOST_COLLECTION_ID_FIELD)) {
+                result.put(key, valueMap.get(key));
+                continue;
+            }
+            if (key.equals(Constants.HOST_ID_FIELD)) { continue; }
+            
+            List values = (List) valueMap.get(key);
+            List hostIDs = new Vector();
+            List collIDs = new Vector();
+            for (Iterator iterator = values.iterator(); iterator.hasNext();) {
+                String id = (String) iterator.next();
+                Matcher matcher = pattern.matcher(id);
+                if (matcher.matches()) {
+                    String host = null;
+                    String coll = null;
+                    try {
+                        host = matcher.group(1);
+                        coll = matcher.group(2);
+                    } catch (IndexOutOfBoundsException e) { /*ignore*/ }
+                    if (host != null && host.length() > 0) {
+                        if (coll != null && coll.length() > 0) {
+                            if (!collIDs.contains(id)) collIDs.add(id);
+                        } else {
+                            if (!hostIDs.contains(host)) hostIDs.add(host);
+                        }
+                    }
+                }
+            }
+            result.put(Constants.HOST_ID_FIELD, hostIDs);
+            result.put(Constants.COLLECTION_ID_FIELD, collIDs);
+        }
+        return result;
+    }
+
+    public boolean containsKey(Object key) {
+        return map.containsKey(key);
+    }
+    
+    public boolean containsValue(Object value) {
+        return map.containsValue(value);
+    }
+    
+    public Predicate getPredicate(String field) {
+        if (map.get(field) instanceof Predicate) {
+            return (Predicate) map.get(field);
+        }
+        return null;
+    }
+    
+    public List getPredicateList(String field) {
+        if (map.get(field) instanceof List) {
+            return (List) map.get(field);
+        }
+        return new Vector();
+    }
+    
+    public Set keySet() {
+        return map.keySet();
+    }
+    
+    public Set getPredicates() {
+        Set result = new HashSet();
+        for (Iterator iter = map.values().iterator(); iter.hasNext();) {
+            Object next = iter.next();
+            if (next == null) {
+                continue;
+            } else if (next instanceof Predicate) {
+                result.add(next);
+            } else if (next instanceof List) {
+                result.addAll((Collection) next);
+            }
+        }
+        return result;
+    }
+    
+    /**
+     * @return
+     */
+    public int getId() {
+        return id;
+    }
+    
+    public int getNumOfNonEqualsPredicates() {
+        return numOfNonEqualsPredicates;
+    }
+    
+    public String toString() {
+        StringBuffer result = new StringBuffer("Subscription ");
+        result.append(name != null ? name : "(no name) ");
+        result.append(" (id ");
+        result.append(id);
+        result.append("):\n");
+        for (Iterator iter = map.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            Object value = map.get(key);
+            if (value instanceof Predicate) {
+                Predicate pred = (Predicate) value;
+                result.append(pred);
+            }
+            if (value instanceof List) {
+                List list = (List) value;
+                for (Iterator iterator = list.iterator(); iterator.hasNext();) {
+                    Predicate pred = (Predicate) iterator.next();
+                    result.append(pred);
+                    if (iterator.hasNext()) {
+                        result.append(" or ");
+                    }
+                }
+            }
+            result.append("\n");
+        }
+        return result.toString();
+    }
+    
+    public boolean equals(Object other) {
+        if (other == null || !(other instanceof Subscription)) return false;
+        return id == ((Subscription)other).id;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Comparable#compareTo(java.lang.Object)
+     */
+    public int compareTo(Object arg0) {
+        Subscription other = (Subscription) arg0;
+        return new Integer(id).compareTo(new Integer(other.id));
+    }
+    
+    private int saveToDatabase() throws DatabaseException, SQLException {
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String sqlString;
+        boolean initial = true;
+        
+        sqlString = "SELECT id FROM subscriptions WHERE name like '" + name + 
+        			"' AND email like '" + email + "' AND rss=" + (rssNotification ? 1 : 0) 
+        			+ " AND page=" + (eventsSincePageNotification ? 1 : 0) + " AND user like '" +
+        			username + "';";
+        ResultSet result = statement.executeQuery(sqlString);
+        if (result.next()) {
+            initial = false;
+            this.id = result.getInt("id");
+        }
+        
+        if (initial) {
+            sqlString = "INSERT INTO subscriptions (name, email, rss, page, user) " +
+            		"VALUES ('" + name + "','" + email + "'," + (rssNotification ? 1 : 0) +
+            		"," + (eventsSincePageNotification ? 1: 0)+ ", '" + username + "');";
+        } else {
+            sqlString = "UPDATE subscriptions SET name='" + name + "', email='" +
+            		email + "', rss=" + (rssNotification ? 1 : 0) + ",page=" + 
+            		(eventsSincePageNotification ? 1 : 0)+ " WHERE id=" + this.id + ";";
+            // cannot change user
+        }
+        statement.executeUpdate(sqlString);
+        
+        sqlString = "SELECT id FROM subscriptions WHERE name like '" + name + 
+        			"' AND email like '" + email + "' AND rss=" + (rssNotification ? 1 : 0) 
+        			+ " AND page=" + (eventsSincePageNotification ? 1 : 0) + " AND user like '" +
+        			username + "';";
+        System.out.println(sqlString);
+        result = statement.executeQuery(sqlString);
+        
+        if (result.next()) {
+            id = result.getInt("id");
+        } else {
+            throw new DatabaseException("Couldn't save subscription");
+        }
+        
+        if (initial) {
+           for (Iterator iter = getPredicates().iterator(); iter.hasNext();) {
+               Predicate predicate = (Predicate) iter.next();
+               sqlString = "INSERT INTO subs_to_predicates (subscription, predicate) " +
+               		"VALUES (" + id + "," + predicate.getID() + ");";
+               statement.executeUpdate(sqlString);
+           }
+        }
+        return id;
+    }
+
+    /**
+     * @return
+     */
+    public boolean wantsEMailNotification() {
+        return email != null && email.length() > 1;
+    }
+
+    /**
+     * @return
+     */
+    public String getMailAddress() {
+        return email;
+    }
+    
+    public String getName() {
+        return name;
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java	(revision 14150)
@@ -0,0 +1,39 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.Map;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class SubstringMatchPredicate extends Predicate {
+
+    public SubstringMatchPredicate(String field, String substring) {
+        super.field = field;
+        super.value = substring;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#isSatisfied(java.util.Map)
+     */
+    public boolean isSatisfied(Map event) {
+        String fieldValue = (String) event.get(field);
+        return fieldValue != null && fieldValue.indexOf(value) >= 0;
+    }
+
+    public String toString() {
+        return field + " contains " + value;
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java	(revision 14150)
@@ -0,0 +1,21 @@
+/*
+ * Created on Dec 3, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class PasswordMismatchException extends Exception {
+    public PasswordMismatchException(String message) {
+        super(message);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java	(revision 14150)
@@ -0,0 +1,31 @@
+/*
+ * Created on Dec 3, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UserManagementException extends Exception {
+
+    /**
+     * @param e
+     */
+    public UserManagementException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    
+    public UserManagementException(String message) {
+        super(message);
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java	(revision 14150)
@@ -0,0 +1,121 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+import java.sql.*;
+import java.util.Map;
+
+import javax.servlet.http.HttpSession;
+
+import org.greenstone.gsdlas.database.DatabaseManager;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UserManager {
+    private static UserManager instance;
+    
+    private UserManager() {
+        // hide constructor
+    }
+    
+    static public UserManager getInstance() {
+        if (instance == null) {
+            instance = new UserManager();
+        }
+        return instance;
+    }
+
+    /**
+     * @param session
+     * @return
+     */
+    public boolean isLoggedIn(HttpSession session) {
+        if (session.getCreationTime() - session.getLastAccessedTime() > session.getMaxInactiveInterval())
+            return false;
+        return session.getAttribute("username") != null;
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     * @throws PasswordMismatchException
+     * @throws UserManagementException
+     */
+    public void createUser(Map arguments, HttpSession session) throws PasswordMismatchException, UserManagementException {
+        if (!arguments.get("password").equals(arguments.get("password2"))) {
+            throw new PasswordMismatchException("The passwords don't match");
+        }
+        String username = (String) arguments.get("username");
+        
+        byte[] password = ((String) arguments.get("password")).getBytes();
+        
+        byte[] pwdHash = password;
+//        try {
+//            pwdHash = MessageDigest.getInstance("MD5").digest(password);
+//        } catch (NoSuchAlgorithmException e) {
+//            e.printStackTrace();
+//            throw new UserManagementException("could not create user", e);
+//        }
+        
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            statement.executeUpdate("INSERT INTO users (username, password) " +
+            		"VALUES ('" + username + "','" + new String(pwdHash) + "')");
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new UserManagementException("could not create user", e);
+        }
+        
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     * @throws UserManagementException
+     */
+    public void loginUser(Map arguments, HttpSession session) throws UserManagementException {
+        String username = (String) arguments.get("username");
+        
+        if (isLoggedIn(session) && session.getAttribute("username").equals(username)) {
+            return; // already logged in
+        }
+        
+        byte[] password = ((String) arguments.get("password")).getBytes();
+        
+        byte[] pwdHash = password;
+        try {
+//            pwdHash = MessageDigest.getInstance("MD5").digest(password);
+            
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            ResultSet results = statement.executeQuery("SELECT password " +
+            		"FROM users WHERE username like '" + username + "';");
+            String pwdFromDB = "";
+            if(results.next()) {
+                pwdFromDB = results.getString("password");
+            }
+            if (!pwdFromDB.equals(new String(pwdHash))) {
+                throw new PasswordMismatchException("user " + username 
+                        + " is unknown, or the passwords don't match");
+            }
+            session.setAttribute("username", username);
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new UserManagementException("could not login user", e);
+        }
+
+    }
+    
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java	(revision 14150)
@@ -0,0 +1,40 @@
+/*
+ * Created on Dec 5, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.Arrays;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class ArrayHelper {
+    static public boolean contains(String[] array, String element) {
+        if (array == null || element == null) {
+            return false;
+        }
+        Arrays.sort(array);
+        int index = Arrays.binarySearch(array, element);
+        return index > -1 && index <= array.length;
+    }
+
+    static public String toString(String [] array) {
+	StringBuffer buffer = new StringBuffer();
+	buffer.append("[");
+	for (int i=0; i<array.length; i++) {
+	    buffer.append((String)array[i]);
+	    buffer.append(",");
+	}
+	buffer.append("]");
+	return buffer.toString();
+
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java	(revision 14150)
@@ -0,0 +1,58 @@
+/*
+ * Created on Jan 1, 2005
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.Properties;
+
+import javax.mail.*;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class Mailer {
+    
+    private String fromAddress;
+    private String mailHost;
+    
+    public Mailer (String from, String host) {
+        fromAddress = from;
+        mailHost = host;
+    }
+    
+    public void sendMail(String to, String subject, String message) throws MessagingException {
+        // Set the host smtp address
+        Properties props = new Properties();
+        props.put("mail.smtp.host", mailHost);
+        
+        // create some properties and get the default Session
+        //Session session = Session.getDefaultInstance(props, null);
+        Session session = Session.getInstance(props);
+        
+        // create a message
+        Message msg = new MimeMessage(session);
+        
+        // set the from and to address
+        InternetAddress addressFrom = new InternetAddress(fromAddress);
+        msg.setFrom(addressFrom);
+        
+        InternetAddress[] addressTo = new InternetAddress[1];
+        addressTo[0] = new InternetAddress(to);
+        msg.setRecipients(Message.RecipientType.TO, addressTo);
+        
+        // Setting the Subject and Content Type
+        msg.setSubject(subject);
+        msg.setContent(message, "text/plain");
+        Transport.send(msg);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java	(revision 14150)
@@ -0,0 +1,242 @@
+/*
+ * Created on Nov 11, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ * 
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.*;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class PatriciaTree {
+	
+	private Node root;
+	
+	public PatriciaTree () {
+	}
+	
+	public void put(CharSequence key, Object value) {
+		root = put(key, value, root, 0);
+	}
+	
+	public Set get(CharSequence key) {
+		return get(key, root, 0);
+	}
+	
+	public String toString() {
+	    return toPreorderString(root).toString();
+	}
+	
+	private StringBuffer toPreorderString(Node root) {
+	    StringBuffer buffer = new StringBuffer("(\n");
+	    buffer.append(root);
+	    buffer.append("\nchildren:");
+	    if (root.hasChildren()) {
+	        for (Iterator iter = root.children.keySet().iterator(); iter.hasNext();) {
+	            Character key = (Character) iter.next();
+	            Node child = root.getChild(key);
+	            buffer.append("\n");
+	            buffer.append(key);
+	            buffer.append("->");
+	            buffer.append(toPreorderString(child));
+	        }
+	    }
+	    
+	    buffer.append("\n)");
+	    return buffer;
+	}
+	
+	/**
+	 * @param key
+	 * @param root
+	 * @param position
+	 * @return the set of values associated with <code>key</code>
+	 */
+	private Set get(CharSequence key, Node root, int position) {
+		Set result = new HashSet();
+		
+		int i = findFirstMismatch(key, root, position);
+		
+		if (i == root.skipped.length()) {
+			Character nextChar = new Character(key.charAt(position + i));
+			if (key.length() > position + i + 1) { // nextChar is not last character
+				if (root.hasChild(nextChar)) {
+					result.addAll(get(key, root.getChild(nextChar), position + i + 1));
+				}	
+			} else {
+				result.addAll(root.getValues(nextChar));
+			}
+		}
+		
+		return result;
+	}
+
+	/**
+	 * @param key
+	 * @param value
+	 * @param root
+	 * @param position
+	 * @return the new root node
+	 */
+	private Node put(CharSequence key, Object value, Node root, int position) {
+	    if (root == null) {
+	        root = new Node();
+	        root.skipped = key.subSequence(position, key.length() - 1);
+	        root.addValue(new Character(key.charAt(key.length() - 1)), value);
+	        return root;
+	    }
+	    
+	    if (root.skipped == null || root.skipped.length() == 0) {
+	        addAsValueOrChild(key, value, root, position);
+	        return root;
+	    }
+	    
+	    int i = findFirstMismatch(key, root, position);
+	    if (i >= root.skipped.length() && position + i < key.length()) {
+	        addAsValueOrChild(key, value, root, position + i);
+	        return root;
+	    }
+	    
+	    if (i >= root.skipped.length() && isLastCharacter(key, position + i - 1)) {
+	        Node newRoot = new Node();
+	        newRoot.skipped = root.skipped.subSequence(0, i - 1);
+	        newRoot.addChild(new Character(root.skipped.charAt(i - 1)), root);
+	        try {
+		        root.skipped = root.skipped.subSequence(i, root.skipped.length());
+		    } catch (StringIndexOutOfBoundsException e) {
+		        root.skipped = "";
+		    }
+		    addAsValueOrChild(key, value, newRoot, position + i - 1);
+		    return newRoot;
+	    }
+	    
+	    Node newRoot = new Node();
+	    newRoot.skipped = root.skipped.subSequence(0, i);
+	    newRoot.addChild(new Character(root.skipped.charAt(i)), root);
+	    try {
+	        root.skipped = root.skipped.subSequence(i + 1, root.skipped.length());
+	    } catch (StringIndexOutOfBoundsException e) {
+	        root.skipped = "";
+	    }
+	    
+	    addAsValueOrChild(key, value, newRoot, position + i);
+	    
+	    return newRoot;
+	}
+
+    /**
+     * @param key
+     * @param value
+     * @param root
+     * @param position
+     */
+    private void addAsValueOrChild(CharSequence key, Object value, Node root, int position) {
+        Character nextChar = new Character(key.charAt(position)); 
+        if (isLastCharacter(key, position)) {
+            root.addValue(nextChar, value); 
+        } else {
+            Node newChild = put(key, value, root.getChild(nextChar), position + 1);
+            root.addChild(nextChar, newChild);
+        }
+    }
+
+    /**
+     * @param key
+     * @param position
+     * @return
+     */
+    private boolean isLastCharacter(CharSequence key, int position) {
+        return key.length() - position == 1;
+    }
+
+    /**
+	 * @param key
+	 * @param root
+	 * @param position
+	 * @return
+	 */
+	private int findFirstMismatch(CharSequence key, Node root, int position) {
+		int i = 0;
+		while (i < root.skipped.length()) {
+			try {
+				if (root.skipped.charAt(i) == key.charAt(position + i)) {
+					i++;
+				} else {
+					return i;
+				}
+			} catch (StringIndexOutOfBoundsException e) {
+				return i;
+			}
+		}
+		return i;
+	}
+
+
+	private class Node {
+		private CharSequence skipped = "";
+		private Map children;
+		private Map values;
+		
+		/**
+		 * @return
+		 */
+		public Set getValues(Character key) {
+			if (values == null || !values.containsKey(key))
+				return new HashSet();
+			return (Set) values.get(key);
+		}
+
+		boolean hasChild(Character key) {
+			return children != null && children.containsKey(key);
+		}
+		
+		boolean hasChildren() {
+			return children != null && !children.isEmpty();
+		}
+		
+		void addChild(Character key, Node child) {
+			if (children == null ) {
+				children = new TreeMap();
+			}
+			children.put(key, child);
+		}
+		
+		Node getChild(Character key) {
+			if (children == null) return null;
+			return (Node) children.get(key);
+		}
+		
+		void addValue(Character key, Object value) {
+			if (values == null) {
+				values = new TreeMap();
+			}
+			Set valueSet;
+			if (values.containsKey(key)) {
+				valueSet = (Set) values.get(key);
+			} else {
+				valueSet = new HashSet();
+				values.put(key, valueSet);
+			}
+			valueSet.add(value);
+		}
+		
+		public String toString() {
+		    StringBuffer buffer = new StringBuffer();
+		    buffer.append(skipped);
+		    buffer.append("\nvalues:");
+		    if (values != null) {
+		        buffer.append(values);
+		    }
+		    return buffer.toString();
+		}
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java	(revision 14150)
@@ -0,0 +1,68 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public class AbstractClientResponse implements ClientResponse
+{ private boolean success;
+  private Map     values;
+
+  public AbstractClientResponse()
+  { this.success = true;
+    this.values  = new HashMap();
+  }
+  
+  public boolean isSuccessful()
+  { return this.success;
+  }
+  
+  public void setSuccessful(boolean success)
+  { this.success = success;
+  }
+  
+  public void addValue(String label, String value)
+  { if (this.values.get(label) == null)
+    { this.setValue(label, value);
+    }
+    else
+    { Vector set = (Vector) this.values.get(label);
+      set.addElement(value);
+    }
+  }
+  
+  public void setValue(String label, String value)
+  { Vector set = new Vector();
+    set.addElement(value);
+    this.values.put(label, set);
+  }
+  
+  public Vector getValues(String label)
+  { return (Vector) this.values.get(label);
+  }
+
+  public Set getLabels()
+  { return this.values.keySet();
+  }
+
+  public String toString()
+  { Iterator keys = this.values.keySet().iterator();
+
+    StringBuffer buffer = new StringBuffer();
+
+    while (keys.hasNext())
+    { String key = keys.next().toString();
+
+      buffer.append("  " +  key + ":\n");
+
+      Vector vals = (Vector) this.values.get(key);
+      Enumeration valist = vals.elements();
+      while (valist.hasMoreElements())
+	{ String nextVal = valist.nextElement().toString();
+
+	  buffer.append("    ");
+	  buffer.append(nextVal);
+	  buffer.append("\n");
+	}
+    }
+    return buffer.toString();
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java	(revision 14150)
@@ -0,0 +1,71 @@
+package org.greenstone.p2p;
+
+import java.net.*;
+import java.util.*;
+
+public abstract class AbstractServerAction implements NameServerAction
+{ NameServer host;
+
+  public AbstractServerAction(NameServer host)
+  { this.host = host;
+  }
+
+  public abstract String getActionName();
+  
+  public void propagateAction(URL forwarder, Map parameters)
+  { Enumeration subServers = host.subServers();
+  
+    // poll all child servers
+    while (subServers.hasMoreElements())
+    { URL server = (URL) subServers.nextElement();
+      
+      if (!server.equals(forwarder))
+      { // send server the action
+        String response = this.propagateActionViaServer(server, parameters, forwarder);
+      }
+    }
+    
+    // poll upwards to parent
+    if (host.getParent() != null &&
+	!host.getParent().equals(forwarder))
+    { this.propagateActionViaServer(host.getParent(), parameters, forwarder);
+    }
+  }
+
+  public void doAction(URL forwarder, Map parameters)
+  { // start propagating the action outwards
+    this.propagateAction(forwarder, parameters);
+  
+    // now execute it on our children
+    Enumeration servers = host.servers();
+  
+    while (servers.hasMoreElements())
+    { URL server = (URL) servers.nextElement();
+      
+      if (!server.equals(forwarder))
+      { // send server the action
+        String response = this.doActionOnServer(server, parameters, forwarder);
+      }
+    }
+  }
+  
+  String propagateActionViaServer(URL server, Map paramters, URL responseTo)
+  { String actionName = this.getActionName();
+  
+    // send action to server, in a separate thread
+    // get responses
+    // forward those back
+    return "";
+  }
+  
+  String doActionOnServer(URL server, Map parameters, URL responseTo)
+  { String actionName = this.getActionName();
+  
+    // send action to the destination server (as a URL)
+    
+    // get response (as HTML)
+    
+    // forward it back...
+    return "";
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/Client.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/Client.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/Client.java	(revision 14150)
@@ -0,0 +1,40 @@
+import java.io.*;
+import java.net.*;
+
+public class Client {
+    public static String read() throws IOException {
+
+        Socket echoSocket = null;
+        PrintWriter out = null;
+        BufferedReader in = null;
+
+        try {
+            echoSocket = new Socket("taranis", 7);
+            out = new PrintWriter(echoSocket.getOutputStream(), true);
+            in = new BufferedReader(new InputStreamReader(
+                                        echoSocket.getInputStream()));
+        } catch (UnknownHostException e) {
+            System.err.println("Don't know about host: taranis.");
+            System.exit(1);
+        } catch (IOException e) {
+            System.err.println("Couldn't get I/O for "
+                               + "the connection to: taranis.");
+            System.exit(1);
+        }
+
+        BufferedReader stdIn = new BufferedReader(
+                                   new InputStreamReader(System.in));
+        String userInput;
+
+        while ((userInput = stdIn.readLine()) != null) {
+            out.println(userInput);
+            System.out.println("echo: " + in.readLine());
+        }
+
+        out.close();
+        in.close();
+        stdIn.close();
+        echoSocket.close();
+	return "";
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ClientResponse.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ClientResponse.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ClientResponse.java	(revision 14150)
@@ -0,0 +1,11 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public interface ClientResponse
+{ boolean isSuccessful(); // obvious
+  Vector  getValues(String label); // get the values associated with a given
+                                   // label; which values are available varies
+                                   // from action to action
+  Set     getLabels();             // get the labels in this response
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java	(revision 14150)
@@ -0,0 +1,13 @@
+package org.greenstone.p2p;
+
+public class DefaultServerAction extends AbstractServerAction
+{ public static final String DEFAULT_ACTION = "default";
+
+  public DefaultServerAction(NameServer host)
+  { super(host);
+  }
+
+  public String getActionName()
+  { return DEFAULT_ACTION;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java	(revision 14150)
@@ -0,0 +1,31 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+import java.net.URL;
+
+public interface GreenstoneClient
+{ /**
+   * This interface needs to be fulfilled by a Greenstone machine wanting to
+   * register and receive messages from the Greenstone Distributed Service.
+   *
+   */
+
+  /**
+   * This method is called by a GDS server when an action (message) has a response
+   * sent in reply.  The Greenstone server that sends the action will have this
+   * method called for each response received from the GDS network.
+   *
+   * @param: <code>NameServerAction</code> - at present, </code>null</code>.
+   * This method *must* be synchronized
+   * @param: <code>ClientResponse</code> - is a simple wrapper for the Map class.
+   * 
+   */
+  public void receiveResponse(NameServerAction action, ClientResponse response);
+
+  /**
+   *
+   */
+  public void receiveMessage(String message, Map parameters);
+  public String getIdentifier();
+  public URL    getURL();
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java	(revision 14150)
@@ -0,0 +1,94 @@
+
+package org.greenstone.p2p;
+
+import java.net.URL;
+import java.util.*;
+
+/**
+ * This is the class that provides a set of functions for a Greenstone server to call
+ * in order to send information/messages into a GDS network.
+ */
+
+public class GreenstoneInterface
+{ private GreenstoneClient client;
+  private URL              server;
+
+  // temporary variables which will eventually disapper - ignore!
+  private static GreenstoneClient lastClient;
+  private static NameServer       nameServer = null;
+  // end of temporary variables
+
+  /**
+   * Make a new interface with a client that will receive the responses
+   * to any messages sent.
+   *
+   * @param: the client to receive responses to messages sent through this
+   *         interface.
+   */
+  public GreenstoneInterface(GreenstoneClient client)
+  { this.client = client;
+    lastClient = this.client;
+    //    /*this.*/nameServer = null;
+  }
+
+  /**
+   *  Set up a registration for this machine with a GDS server.
+   *
+   *  The Greenstone machine should take care to store the URL of the
+   *  server for later use - don't continually reregister the same
+   *  server!
+   *
+   *  @param: <code>URL</code> the location of the GDS server to register with
+   *
+   *  @return: <code>boolean</code> whether the registration was successful or not.
+   */
+  public boolean registerWithServer(URL server)
+  { // connect to server
+    // 
+    if (/*this.*/nameServer == null) {
+      /*this.*/nameServer = new NameServer(server.toString());
+    }
+    this.server = server;
+    /*this.*/nameServer.registerServer(this.client.getURL());
+
+    return true;
+  }
+
+  /**
+   *  Method to unregister from the current GDS server for this Greenstone
+   *  machine.
+   *
+   *  @return: <code>boolean</code> whether the registration was successful or not.
+   */
+  public boolean unregisterFromServer()
+  { /*this.*/nameServer = null;
+    return true;
+  }
+
+  /**
+   *  Send a message into the GDS system through the current GDS server.
+   *
+   *  Success does not guarantee that you will receive a response.
+   *
+   *  @param: <code>String</code> the identifier for this type of message
+   *  @param: <code>Map</code> a <code>Vector</code> item is associated with
+   *          each key in this map.
+   *
+   *  @return: <code>boolean</code> whether the sending was successful or not.
+   */
+  public boolean sendMessage(String message, Map parameters)
+  { if (/*this.*/nameServer != null)
+      { /*this.*/nameServer.sendMessage(this.client.getURL(), message, parameters);
+    }
+    return false;
+  }
+
+  /**
+   *  Receive a response to a message sent from this Greenstone machine and
+   *  forward it to the current client - won't be needed in the long run, it
+   *  is here at present to provide a route for returning responses.
+   */
+  static public void receiveResponse(URL greenstoneURL, ClientResponse response)
+  { lastClient.receiveResponse(null, response);
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java	(revision 14150)
@@ -0,0 +1,20 @@
+package org.greenstone.p2p;
+
+import java.net.URL;
+
+/**
+ *  Simple class to hold the location of a Greenstone server.
+ *
+ *  This class should be subclassed and methods overridden for 
+ *  Greenstone 2, 3 or any special installations such as the
+ *  Music DL.
+ */
+
+public class GreenstoneServer
+{ private URL server;
+  
+  public GreenstoneServer(URL url)
+  { this.server = url;
+  }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java	(revision 14150)
@@ -0,0 +1,70 @@
+package org.greenstone.p2p;
+
+import java.net.URL;
+import java.util.*;
+
+public class GreenstoneTest implements GreenstoneClient
+{ String name;
+  URL    url;
+  
+  public GreenstoneTest(String id, String url)
+  { try {
+      this.url = new URL(url);
+    }
+    catch (java.net.MalformedURLException malEx)
+      { this.url = null;
+      }
+    this.name = id;
+  }
+
+  public void receiveResponse(NameServerAction action, ClientResponse response)
+  {
+    System.out.println("\n----\nReceived a response:");
+    System.out.println(response.toString());
+  }
+
+  public void receiveMessage(String message, Map parameters)
+  {
+  }
+
+  public String getIdentifier()
+  { return this.name;
+  }
+
+  public URL getURL()
+  { return this.url;
+  }
+
+  /**
+   *  arg[0] => identifier for this machine
+   *  arg[1] => a url for this machine
+   *  arg[2] => a url for the GDS server
+   *
+   * e.g. run with:
+   * "java org.greenstone.p2p.GreenstoneTest localhost http://localhost http://www.nzdl.org"
+   */
+  public static void main(String args[])
+  { GreenstoneClient client = new GreenstoneTest(args[0], args[1]);
+    GreenstoneClient gsdl   = new GreenstoneTest("gsdl", "http://www.gsdl.org");
+    
+    URL server;
+    try
+      {
+	server = new URL(args[2]);
+      }
+    catch (java.net.MalformedURLException malEx)
+      { server = null;
+      }
+    
+    GreenstoneInterface gsdlIface = new GreenstoneInterface(gsdl);
+    gsdlIface.registerWithServer(server);
+
+    GreenstoneInterface iface = new GreenstoneInterface(client);
+    iface.registerWithServer(server);
+
+    Map parameters = new HashMap();
+    iface.sendMessage("ListServers", parameters);
+
+    iface.sendMessage("DocDelta", parameters);
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ListServersAction.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ListServersAction.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ListServersAction.java	(revision 14150)
@@ -0,0 +1,35 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+import java.net.URL;
+
+public class ListServersAction extends AbstractServerAction
+{ public String LISTSERVERS = "ListServers";
+
+  public ListServersAction(NameServer host)
+  { super(host);
+  }
+
+  public String getActionName()
+  { return LISTSERVERS;
+  }
+  
+  public void doAction(URL forwarder, Map parameters)
+  { this.propagateAction(forwarder, parameters);
+  
+    // Build a list of all servers
+    AbstractClientResponse response = new AbstractClientResponse();
+
+    Enumeration servers = this.host.servers();
+    while (servers.hasMoreElements())
+      { URL gsServer = (URL) servers.nextElement();
+	System.out.println("Adding listserver item " + gsServer);
+
+	response.addValue("Server", gsServer.toString());
+      }
+    response.setSuccessful(true);
+
+    // Send it back to the forwarder right now...
+    this.host.receiveResponse(forwarder, response);
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServer.java	(revision 14150)
@@ -0,0 +1,124 @@
+package org.greenstone.p2p;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class NameServer
+{ ServerList list;
+  URL        parent;
+  Vector     childServers; // name servers
+  Vector     children;     // greenstone servers
+  String     identity;
+  Map        actionMap;
+ 
+  public NameServer(String identity)
+  { this.list     = new ServerList();
+    this.identity = identity;
+    this.parent   = null;
+    this.children = new Vector();
+    this.childServers = new Vector();
+
+    this.actionMap = new HashMap();
+
+    NameServerAction listServersAction = new ListServersAction(this);
+    this.addAction(listServersAction);
+    NameServerAction defaultAction = new DefaultServerAction(this);
+    this.addAction(defaultAction);
+    
+    // set up registration listener
+    // set up messaging broadcaster
+    // set up messaging listener
+  }
+
+  private void addAction(NameServerAction action)
+  { this.actionMap.put(action.getActionName(), action);
+  }
+
+  private void loadChildren(BufferedReader in)
+  { try 
+      { String child = in.readLine();
+      }
+    catch (IOException ex)
+      {
+      }
+  }
+
+  private void loadServers(BufferedReader in)
+  { try
+      { String child = in.readLine();
+      }
+    catch (IOException io)
+      {
+      }
+  }
+
+  public void sendMessage(URL origin, String actionName, Map parameters)
+  { NameServerAction action = (NameServerAction) this.actionMap.get(actionName);
+    if (action != null)
+      { action.doAction(origin, parameters);
+      }
+    else
+      { System.out.println("Sending action " + actionName + " through <default>");
+
+	// TODO: add action name to the parameter map so that it is not lost...
+	action = (NameServerAction) this.actionMap.get("Default");
+	if (action != null)
+	{ action.doAction(origin, parameters);
+	}
+      }
+    //NameServerAction action = NameServerActionFactory.createAction(action, parameters);
+    //action.doAction();
+  }
+
+  public void receiveResponse(URL responseTo, ClientResponse response)
+  { GreenstoneInterface.receiveResponse(responseTo, response);
+  }
+    
+  private void loadParent()
+  {
+  }
+
+  public String getIdentity()
+  { return this.identity;
+  }
+  
+  public void setParent(URL parent)
+  { this.parent = parent;
+  }
+  
+  public URL getParent()
+  { return this.parent;
+  }
+  
+  public void addSubServer(URL subserver)
+  { this.childServers.add(subserver);
+  }
+  
+  public void registerServer(URL child)
+  { System.out.println("Receiving registration from " + child.toString());
+    this.children.add(child);
+  }
+  
+  public Enumeration subServers()
+  { return this.childServers.elements();
+  }
+  
+  public Enumeration servers()
+  { return this.children.elements();
+  }
+
+  public static boolean pollServer(URL server)
+  { return false;
+  }
+
+  /**
+   *  
+   */
+  public static void main(String args[])
+  { NameServer server = new NameServer(args[0]);
+
+    // Load the action items
+    // Wait for signals
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerAction.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerAction.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerAction.java	(revision 14150)
@@ -0,0 +1,10 @@
+package org.greenstone.p2p;
+
+import java.util.Map;
+import java.net.URL;
+
+public interface NameServerAction
+{ public String getActionName();
+  public void   doAction(URL forwarder, Map parameters);
+  public void   propagateAction(URL forwarder, Map parameters);
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java	(revision 14150)
@@ -0,0 +1,9 @@
+package org.greenstone.p2p;
+
+import java.util.Map;
+
+public class NameServerActionFactory
+{ public NameServerAction createAction(String actionName, Map parameters)
+  { return null;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java	(revision 14150)
@@ -0,0 +1,16 @@
+package org.greenstone.p2p;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class NameServerRegistrationThread extends Thread
+{ public void run() 
+  { /*try
+    {
+    }
+    catch (IOException ex)
+    {
+    }*/
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ServerList.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ServerList.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/src/org/greenstone/p2p/ServerList.java	(revision 14150)
@@ -0,0 +1,19 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public class ServerList
+{ Map map;
+
+  public ServerList()
+  { this.map = new HashMap();
+  }
+  
+  public void addServer(GreenstoneServer server)
+  { this.map.put(server, server);
+  }
+  
+  Iterator servers()
+  { return ((Collection)this.map.values()).iterator();
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java	(revision 14150)
@@ -0,0 +1,163 @@
+/*
+ * Created on Nov 24, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+import org.apache.velocity.exception.*;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class AlertingServiceTest extends TestCase {
+
+    private AlertingService as;
+    
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(AlertingServiceTest.class);
+    }
+
+    public final void testSubscribe() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put(Constants.TYPE_FIELD, "new_document");
+            arguments.put(Constants.COLLECTION_ID_FIELD, "testing");
+            arguments.put(Constants.DOCUMENT_CONTENT_FIELD, "albatros");
+            
+            String template = as.createSubscription(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testListSubscriptions() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            
+            String template = as.listSubscriptions(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testReceiveEvent() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put(Constants.COLLECTION_ID_FIELD, "testing");
+            arguments.put(Constants.TYPE_FIELD, "new_document");
+            arguments.put(Constants.DOCUMENT_ID_FIELD, "20041124:1");
+            arguments.put(Constants.DOCUMENT_TITLE_FIELD, "keine albatrosse in sicht");
+            arguments.put(Constants.HOST_ID_FIELD, "http://localhost:8080/soap/servlet/rpcrouter");
+            
+            as.receiveEvent(arguments);
+            
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testShowFeed() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put("subscriptionID", "0");
+            
+            String template = as.showFeed(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+    protected void setUp() throws Exception {
+        as = new AlertingService();
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java	(revision 14150)
@@ -0,0 +1,75 @@
+/*
+ * Created on Nov 17, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class GreenstoneCommunicatorTest extends TestCase {
+
+    private GreenstoneCommunicator gsComm;
+    
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(GreenstoneCommunicatorTest.class);
+    }
+    
+    protected void setUp() throws Exception {
+        URL host;
+        try {
+            host = new URL("http://localhost:8080/soap/servlet/rpcrouter");
+            gsComm = new GreenstoneCommunicator(host);
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testGetSiteList() {
+        String[] sites;
+        try {
+            sites = gsComm.getCollectionNames();
+            for (int i = 0; i < sites.length; i++) {
+                System.out.println(sites[i]);
+                
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testSupportsFullTextSearch() {
+        try {
+            assertTrue(gsComm.supportsTextSearch("gs2mgppdemo"));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testFullTextSearch() {
+        try {
+            Set result = gsComm.fullTextSearch("gs2mgppdemo", "Burundi");
+            assertEquals(14, result.size());
+            assertTrue(result.contains("HASH4632a8a51d33c47a75c559.7.1"));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java	(revision 14150)
@@ -0,0 +1,87 @@
+/*
+ * Created on Nov 18, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.*;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class ProfileStoreTest extends TestCase {
+
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(ProfileStoreTest.class);
+    }
+
+    public final void testAddSubscription() {
+        try {
+            addSubscriptions();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    public final void testFilter() {
+        try {
+            addSubscriptions();
+            
+            URL url = new URL("http://localhost:8080/soap/servlet/rpcrouter");
+            GreenstoneCommunicator gsComm = new GreenstoneCommunicator(url);
+            
+            Map event = new TreeMap();
+            
+            event.put(Constants.DOCUMENT_ID_FIELD, "20041118:5");
+            event.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+            event.put(Constants.HOST_ID_FIELD, "http://localhost:8080/soap/servlet/rpcrouter");
+            
+            Set result = ProfileStore.getInstance().filter(event, gsComm);
+            
+            System.out.println(result.size());
+            
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    private void addSubscriptions() throws Exception {
+        Map valueMap = new TreeMap();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "Tomatensalat");
+        
+        addSubscription(valueMap);
+        
+        valueMap.clear();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "Water+Buffalo");
+        
+        addSubscription(valueMap);
+        
+        valueMap.clear();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "gender+equality");
+        
+        addSubscription(valueMap);
+    }
+    
+    private void addSubscription(Map valueMap) throws Exception {  
+        ProfileStore.getInstance().createSubscription(valueMap);    
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java	(revision 14150)
@@ -0,0 +1,35 @@
+/*
+ * Created on Jan 1, 2005
+ *
+ * TODO To change the template for this generated file go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+package org.greenstone.gsdlas.util;
+
+import javax.mail.MessagingException;
+
+import junit.framework.TestCase;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class MailerTest extends TestCase {
+
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(MailerTest.class);
+    }
+
+    public final void testSendMail() {
+        Mailer mailer = new Mailer("andrea@localhost", "localhost");
+        try {
+            mailer.sendMail("andrea@localhost", "test", "test message testing...\n");
+        } catch (MessagingException e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/WEB-INF/web.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/WEB-INF/web.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/WEB-INF/web.xml	(revision 14150)
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<web-app>
+
+
+    <!-- General description of your web application -->
+
+    <display-name>Alerting Service for Greenstone</display-name>
+    <description>
+	An alerting service for Greenstone, 
+	written by Andrea Schweer (schweer@cs.waikato.ac.nz).
+    </description>
+
+
+    <!-- Context initialization parameters that define shared
+         String constants used within your application, which
+         can be customized by the system administrator who is
+         installing your application.  The values actually
+         assigned to these parameters can be retrieved in a
+         servlet or JSP page by calling:
+
+             String value =
+               getServletContext().getInitParameter("name");
+
+         where "name" matches the <param-name> element of
+         one of these initialization parameters.
+
+         You can define any number of context initialization
+         parameters, including zero.
+    -->
+
+    <context-param>
+      <param-name>webmaster</param-name>
+      <param-value>schweer@cs.waikato.ac.nz</param-value>
+      <description>
+        The EMAIL address of the administrator to whom questions
+        and comments about this application should be addressed.
+      </description>
+    </context-param>
+
+
+    <!-- Servlet definitions for the servlets that make up
+         your web application, including initialization
+         parameters.  With Tomcat, you can also send requests
+         to servlets not listed here with a request like this:
+
+           http://localhost:8080/{context-path}/servlet/{classname}
+
+         but this usage is not guaranteed to be portable.  It also
+         makes relative references to images and other resources
+         required by your servlet more complicated, so defining
+         all of your servlets (and defining a mapping to them with
+         a servlet-mapping element) is recommended.
+
+         Servlet initialization parameters can be retrieved in a
+         servlet or JSP page by calling:
+
+             String value =
+               getServletConfig().getInitParameter("name");
+
+         where "name" matches the <param-name> element of
+         one of these initialization parameters.
+
+         You can define any number of servlets, including zero.
+    -->
+
+    <servlet>
+      <servlet-name>service</servlet-name>
+      <description>
+        An alerting service for Greenstone 3
+      </description>
+      <servlet-class>org.greenstone.gsdlas.AlertingService</servlet-class>
+      <init-param>
+            <param-name>properties</param-name>
+            <param-value>/velocity.properties</param-value>
+      </init-param>
+    </servlet>
+<!--
+    <servlet>
+      <servlet-name>subscribe</servlet-name>
+      <description>
+        This servlet creates new subscriptions.
+      </description>
+      <servlet-class>org.greenstone.gsdlas.SubscriptionServlet</servlet-class>
+    </servlet> -->
+
+	<!-- Define mappings that are used by the servlet container to
+         translate a particular request URI (context-relative) to a
+         particular servlet.  The examples below correspond to the
+         servlet descriptions above.  Thus, a request URI like:
+
+           http://localhost:8080/{contextpath}/graph
+
+         will be mapped to the "graph" servlet, while a request like:
+
+           http://localhost:8080/{contextpath}/saveCustomer.do
+
+         will be mapped to the "controller" servlet.
+
+         You may define any number of servlet mappings, including zero.
+         It is also legal to define more than one mapping for the same
+         servlet, if you wish to.
+    -->
+
+<!--    <servlet-mapping>
+      <servlet-name>receive</servlet-name>
+      <url-pattern>/receive</url-pattern>
+    </servlet-mapping>
+
+    <servlet-mapping>
+      <servlet-name>subscribe</servlet-name>
+      <url-pattern>/subscribe</url-pattern>
+    </servlet-mapping> -->
+    <servlet-mapping>
+      <servlet-name>service</servlet-name>
+      <url-pattern>/service</url-pattern>
+    </servlet-mapping>
+</web-app>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/about.html
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/about.html	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/about.html	(revision 14150)
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+      <title>Greenstone Version 3</title>
+			<link rel="stylesheet" type="text/css" href="default.css"/>
+			</head>
+  <body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+    <div id="content"><h3>Alerting Service for Greenstone &ndash; About</h3>
+		<p></p>
+	
+	
+	(description of what an alerting service does)<br/>
+	<a href="index.html">back</a> to the alerting service.<br/>
+	</div>
+  </body></html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/default.css
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/default.css	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/default.css	(revision 14150)
@@ -0,0 +1,52 @@
+body {
+	color:             black;
+	background-color:  white;
+	/*font-family:       "Trebuchet MS", Arial, helvetica, sans-serif;*/
+	text-align:        center;
+	margin:            0px;
+	background-image:  url("images/chalk.gif");
+	background-repeat: repeat-y;
+}
+
+#content {
+	background-image: url("images/gsdlhead.gif");
+	background-repeat: no-repeat;
+	background-position: center top;
+	padding-top:  88px;
+	width:        537px;
+	position:     relative;
+	text-align:   left;
+	margin-left:  auto;
+	margin-right: auto;
+	margin-top:   10px;
+}
+
+
+.error {
+        color: red;
+        font-weight: bold;
+}
+
+.details {
+        font-size: small;
+}
+
+table, thead {
+        border: 1pt solid black;
+}
+
+thead {
+        background-color: gray;
+}
+
+a:link {
+	color: #006666;
+}
+
+a:active {
+	color: #cc9900;
+}
+
+a:visited {
+	color: #666633;
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/descriptor.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/descriptor.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/descriptor.xml	(revision 14150)
@@ -0,0 +1,8 @@
+<isd:service
+  xmlns:isd="http://xml.apache.org/xml-soap/deployment"
+  id="alerting" checkMustUnderstands="false">
+  <isd:provider type="java" scope="Session" methods="receiveEvent">
+    <isd:java class="org.greenstone.gsdlas.AlertingService" static="false"/>
+  </isd:provider>
+  <isd:faultListener>org.apache.soap.server.DOMFaultListener</isd:faultListener>
+</isd:service>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/feed.css
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/feed.css	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/feed.css	(revision 14150)
@@ -0,0 +1,45 @@
+@import url("default.css");
+
+#channelhead {
+	margin-bottom: 20px;
+}
+
+#channelhead a:link {
+	text-decoration: none;
+	color:           black;
+}
+
+#channelhead a:visited {
+	text-decoration: none;
+	color:            black;
+}
+
+
+#channelhead a:hover, a:focus, a:active {
+	color:            blue;
+	text-decoration:  underline;
+}
+
+#channelinfo {
+	border-top: 2px solid black;
+	font-size: small;
+	margin-top: 20px;
+}
+
+div.item {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
+span.date, span.category {
+	font-style: italic;
+}
+
+div.description {
+	font-size: small;
+}
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/form.css
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/form.css	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/form.css	(revision 14150)
@@ -0,0 +1,72 @@
+@import url("default.css");
+
+form {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
+td, th {
+	padding: 0px 5px;
+}
+
+thead tr {
+	background-color: #CCCCCC;
+}
+
+tr.even {
+	background-color: #CCCCCC;
+}
+
+.explanation {
+	font-size: small;
+}
+
+#progress {
+	font-size: x-small;
+	border: 0px solid black;
+	width: 80%;
+	margin: 5px auto;
+}
+
+#progress td {
+	width: 25%;
+	vertical-align: top;
+	text-align: center;
+	color: #555;
+	border-top: 10px solid #CCC;
+	font-family: "Trebuchet MS", Arial, helvetica, sans-serif;
+}
+
+#progress td.completed {
+	border-top-color: #9CBE9C;
+	color: #000;
+}
+
+#progress td.current {
+	background-color: #F0E68C;
+	color: #000;
+}
+
+table.subscription {
+	font-size: small;
+}
+
+.rssbutton {
+	background: #ff6600;
+	color: #ffffff;
+	border-left:   1px solid #cc9966;
+	border-top:    1px solid #ccaa99;
+	border-right:  1px solid #993300;
+	border-bottom: 1px solid #331100;
+	padding: 0px 0.5em 0px 0.5em;
+	font-family: helvetica, arial, sans-serif;
+	font-weight: bold;
+	font-size: xx-small;
+	margin-top: 1em;
+}
+
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/index.html
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/index.html	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/index.html	(revision 14150)
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		<title>Greenstone Version 3</title>
+		<link rel="stylesheet" type="text/css" href="default.css"/>
+	</head>
+	<body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>What do you want to do today?</p>
+			
+			<ul>
+				<li>
+					<a href="/alerting/service?action=showLoginForm">Log in</a></li>
+				</li>
+				<li>
+					<a href="subscribe.html">Create</a> a subscription.</li>
+				<li>
+					<a href="/alerting/service?action=listSubscriptions">View and edit</a> your existing subscriptions.</li>
+				<li>
+					<a href="about.html">Read more</a> about the alerting service.</li>
+				<li>
+					<a href="/alerting/service?action=logout">Log out</a></li?
+			</ul>
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/list.css
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/list.css	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/list.css	(revision 14150)
@@ -0,0 +1,12 @@
+@import url("default.css");
+
+div.subscription {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/rss2html.xsl
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/rss2html.xsl	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/rss2html.xsl	(revision 14150)
@@ -0,0 +1,65 @@
+<?xml version="1.0" ?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+	<xsl:output method="xml" 
+  	doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd" 
+	  doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN" indent="yes"/>
+	<!-- <xsl:output method="html"/> -->
+
+	<xsl:template match="/">
+		<html>
+			<xsl:attribute name="lang">
+				<xsl:value-of select="//channel/language"/>
+			</xsl:attribute>
+			<head>
+				<link rel="stylesheet" type="text/css" href="/alerting/feed.css"/>
+				<title>
+					<xsl:value-of select="//channel/title"/>
+				</title>
+			</head>
+			<body>
+				<div id="content">
+					<xsl:apply-templates/>
+				</div>
+   		</body>
+		</html>
+  </xsl:template>
+	
+	<xsl:template match="channel">
+		<div id="channelhead">
+      <h3>
+				<a>
+      		<xsl:attribute name="href">
+        		<xsl:value-of select="link"/>
+					</xsl:attribute>
+					<xsl:value-of select="title"/>
+				</a>
+			</h3>
+			<div class="description">
+        	<xsl:value-of select="description" disable-output-escaping="yes"/>
+			</div>
+		</div>
+    <xsl:apply-templates/>
+		<div id="channelinfo">
+			Last rebuilt <xsl:value-of select="lastBuildDate"/>.<br/>
+			This file has been automatically generated by <xsl:value-of select="generator"/>; please <a><xsl:attribute name="href">mailto:<xsl:value-of select="webMaster"/></xsl:attribute>contact the webmaster</a> if you have any questions.<br/>
+		</div>
+  </xsl:template>
+
+  <xsl:template match="item">
+		<div class="item">
+			<span class="date"><xsl:value-of select="pubDate"/></span>: 
+			<span class="category"><xsl:value-of select="category"/></span><br/>
+    	<a class="item">
+      	<xsl:attribute name="href">
+        	<xsl:value-of select="link"/>
+				</xsl:attribute>
+				<xsl:value-of select="title"/>
+      </a><br/>
+		</div>
+      <xsl:apply-templates/>
+  </xsl:template>
+
+  <xsl:template match="text()"/>
+	
+</xsl:stylesheet>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/subscribe.html
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/subscribe.html	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/subscribe.html	(revision 14150)
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		<title>Greenstone Version 3</title>
+		<link rel="stylesheet" type="text/css" href="form.css"/>
+	</head>
+	<body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>Create a subscription</p>
+			
+			You can subscribe to be notified of events about these elements:
+			<ul>
+				<li><a href="/alerting/service?action=createSubscription">documents</a> (new, modified or deleted documents)</li>
+				<li><a href="sub_docparts.html">parts of documents</a> (new, modified or deleted parts of documents)</li>
+				<li><a href="sub_metadata.html">document metadata</a> (new, modified or deleted metadata of documents)</li>
+				<li><a href="sub_collection.html">collections</a> (new, rebuilt or deleted collections)</li>
+				<li><a href="sub_classifiers.html">classifiers of collections</a> (new or deleted classifiers) <img alt="help" src="images/help.png" longdesc="this influences the ways you can browse a collection" title="this influences the ways you can browse a collection"/></li>
+				<li><a href="sub_indexes.html">indexes of a collection</a> (new or deleted indexes) <img alt="help" src="images/help.png" longdesc="this influences the ways you can search in a collection" title="this influences the ways you can search in a collection"/></li>
+			</ul>
+			
+			<a href="/alerting">back</a> to the alerting service
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/test.html
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/test.html	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/test.html	(revision 14150)
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Alerting Service for Greenstone</title>
+		<link href="default.css" rel="stylesheet" type="text/css"/>
+		<style type="text/css">
+		<!--
+		table { border:1pt solid black; padding:2px; background:#F0F0F0; }
+		th { text-align:left; background:lightblue; }
+		th,td { border-width:0px; }
+		-->
+		</style>
+	</head>
+	<body>
+		<div id="content">
+		<!-- <h1>Test</h1>
+		<form action="/alerting/service?test" method="post">
+			<input type="text" name="text"/><br/>
+			<input type="submit" value="submit"/>
+		</form>
+		<hr/> -->
+		
+		<h3>Subscribe</h3>
+		<form action="/alerting/service" method="post" name="subscribe">
+			<input type="hidden" name="action" value="subscribe"/>
+			<table border="1">
+				<tr>
+					<th>Event Type</th>
+					<td>
+						<input type="radio" name="type" value="new_document" checked="checked"/> new document
+						<input type="radio" name="type" value="document_deleted"/> document deleted
+						<input type="radio" name="type" value="document_content_changed" /> content changed
+						<input type="radio" name="type" value="document_md_changed"/> metadata changed
+					</td>
+				</tr>
+				<tr>
+					<th>Document ID</th>
+					<td>
+						<input type="text" name="documentID"/> 
+					</td>
+				</tr>
+				<tr>
+					<th>Document Title</th>
+					<td>
+						<input type="text" name="document_title"/>
+					</td>
+				</tr>
+				<tr>
+					<th>Document Content</th>
+					<td>
+						<input type="text" name="document_content"/>
+					</td>
+				</tr>
+				<tr>
+					<th>Collection ID</th>
+					<td>
+					 <input type="text" name="collectionID"/>
+					</td>
+				</tr>
+			</table>
+			<br/>
+			<input type="submit" value="submit"/>
+		</form>
+		</div>
+	</body>
+</html>
Index: /other-projects/trunk/greenstone3-extension/gsdl-as/web/velocity.properties
===================================================================
--- /other-projects/trunk/greenstone3-extension/gsdl-as/web/velocity.properties	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/gsdl-as/web/velocity.properties	(revision 14150)
@@ -0,0 +1,155 @@
+#----------------------------------------------------------------------------
+# These are the default properties for the
+# Velocity Runtime. These values are used when
+# Runtime.init() is called, and when Runtime.init(properties)
+# fails to find the specificed properties file.
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+# R U N T I M E  L O G
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+#  default LogSystem to use: default: AvalonLogSystem
+#----------------------------------------------------------------------------
+
+# runtime.log.logsystem.class = org.apache.velocity.runtime.log.AvalonLogSystem,org.apache.velocity.runtime.log.SimpleLog4JLogSystem
+
+runtime.log.logsystem.class = org.apache.velocity.runtime.log.SimpleLog4JLogSystem
+
+#---------------------------------------------------------------------------
+# This is the location of the Velocity Runtime log.
+#----------------------------------------------------------------------------
+
+runtime.log = ../logs/velocity.log
+
+#----------------------------------------------------------------------------
+# This controls if Runtime.error(), info() and warn() messages include the
+# whole stack trace. The last property controls whether invalid references
+# are logged.
+#----------------------------------------------------------------------------
+
+runtime.log.error.stacktrace = true
+runtime.log.warn.stacktrace = false
+runtime.log.info.stacktrace = false
+runtime.log.invalid.reference = true
+
+#----------------------------------------------------------------------------
+# Configuration for the Log4JLogSystem.
+# You must define the runtime.log.logsystem.class property to be:
+#   org.apache.velocity.runtime.log.Log4JLogSystem
+#
+# You must also include Log4J's .jar files into your classpath. They are
+# included with the Velocity distribution in the build/lib directory.
+#
+# There are several different options that you can configure.
+# Uncomment the ones that you want and also define their settings.
+#----------------------------------------------------------------------------
+#runtime.log.logsystem.log4j.pattern=%d - %m%n
+#runtime.log.logsystem.log4j.file.size=100000
+#runtime.log.logsystem.log4j.file.backups=1
+#runtime.log.logsystem.log4j.syslogd.host=my.syslog.server.com
+#runtime.log.logsystem.log4j.syslogd.facility=LOG_DAEMON
+#runtime.log.logsystem.log4j.remote.host=my.remote.server.com
+#runtime.log.logsystem.log4j.remote.port=1099
+#runtime.log.logsystem.log4j.email.server=localhost
+#runtime.log.logsystem.log4j.email.from=root@localhost
+#runtime.log.logsystem.log4j.email.to=root@localhost
+#runtime.log.logsystem.log4j.email.subject=Velocity Error Report
+#runtime.log.logsystem.log4j.email.buffer.size=512
+
+#----------------------------------------------------------------------------
+# T E M P L A T E  E N C O D I N G
+#----------------------------------------------------------------------------
+
+input.encoding=ISO-8859-1
+output.encoding=ISO-8859-1
+
+#----------------------------------------------------------------------------
+# F O R E A C H  P R O P E R T I E S
+#----------------------------------------------------------------------------
+# These properties control how the counter is accessed in the #foreach
+# directive. By default the reference $velocityCount will be available
+# in the body of the #foreach directive. The default starting value
+# for this reference is 1.
+#----------------------------------------------------------------------------
+
+directive.foreach.counter.name = velocityCount
+directive.foreach.counter.initial.value = 1
+
+#----------------------------------------------------------------------------
+# I N C L U D E  P R O P E R T I E S
+#----------------------------------------------------------------------------
+# These are the properties that governed the way #include'd content
+# is governed.
+#----------------------------------------------------------------------------
+
+directive.include.output.errormsg.start = <!-- include error :
+directive.include.output.errormsg.end   =  see error log -->
+
+#----------------------------------------------------------------------------
+# P A R S E  P R O P E R T I E S
+#----------------------------------------------------------------------------
+
+directive.parse.max.depth = 10
+
+#----------------------------------------------------------------------------
+# T E M P L A T E  L O A D E R S
+#----------------------------------------------------------------------------
+#
+#
+#----------------------------------------------------------------------------
+
+# resource.loader = file
+# 
+# file.resource.loader.description = Velocity File Resource Loader
+# file.resource.loader.class = org.apache.velocity.runtime.resource.loader.FileResourceLoader
+# file.resource.loader.path = .
+# file.resource.loader.cache = false
+# file.resource.loader.modificationCheckInterval = 2
+resource.loader = class
+class.resource.loader.description = Velocity Class Resource Loader
+class.resource.loader.class = org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
+
+#----------------------------------------------------------------------------
+# VELOCIMACRO PROPERTIES
+#----------------------------------------------------------------------------
+# global : name of default global library.  It is expected to be in the regular
+# template path.  You may remove it (either the file or this property) if
+# you wish with no harm.
+#----------------------------------------------------------------------------
+velocimacro.library = VM_global_library.vm
+
+velocimacro.permissions.allow.inline = true
+velocimacro.permissions.allow.inline.to.replace.global = false
+velocimacro.permissions.allow.inline.local.scope = false
+
+velocimacro.context.localscope = false
+
+#----------------------------------------------------------------------------
+# INTERPOLATION
+#----------------------------------------------------------------------------
+# turn off and on interpolation of references and directives in string
+# literals.  ON by default :)
+#----------------------------------------------------------------------------
+runtime.interpolate.string.literals = true
+
+
+#----------------------------------------------------------------------------
+# RESOURCE MANAGEMENT
+#----------------------------------------------------------------------------
+# Allows alternative ResourceManager and ResourceCache implementations
+# to be plugged in.
+#----------------------------------------------------------------------------
+resource.manager.class = org.apache.velocity.runtime.resource.ResourceManagerImpl
+resource.manager.cache.class = org.apache.velocity.runtime.resource.ResourceCacheImpl
+
+#----------------------------------------------------------------------------
+# PLUGGABLE INTROSPECTOR
+#----------------------------------------------------------------------------
+# Allows alternative introspection and all that can of worms brings
+#----------------------------------------------------------------------------
+
+runtime.introspector.uberspect = org.apache.velocity.util.introspection.UberspectImpl
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/README
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/README	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/README	(revision 14150)
@@ -0,0 +1,82 @@
+Vishnu package for ...
+
+If you are running this as part of Greenstone, just installing Greenstone will install Vishnu.
+
+If you want to run it standalone:
+
+You need to have Java 1.4 or higher and Ant installed. The environment variable JAVA_HOME needs to be set. 
+You need to have MG installed - this should be the version from Greenstone, and you need the mg.jar file in your CLASSPATH and the libmgjni.so in your LD_LIBRARY_PATH.
+You need the lucene 1.4 or higher jar file in your CLASSPATH.
+Other jar files needed in CLASSPATH: xercesImpl.jar, servlet-api.jar.
+These jar files all come as part of the Greenstone download. Some in gsdl3/lib/java, and servlet.jar in gsdl3/comms/jakarta/tomcat/lib (when unpacked).
+
+Edit the build.xml file and change the prefix property to be the current directory.
+
+Then run 'ant' (or "ant all').
+
+
+COLLECTION BUILDING:
+
+Make a directory called collections inside the main vishnu directory. Inside this, create a new directory for your collection. The name of this directory is the collection identifier. Inside the collection directory create a new direcotry called import. Inside here put all the documents that you want in your collection. Documents must be plain text, xml or html.
+
+Then you can build the collection:
+
+Command line: 
+java vishnu.builder.CollectionBuilder -collsHome (directory where collections live: path-to-vishnu/collections) -collection (directory name of collection) -name (descriptive name for collection) -description (collection description) -index ("mg" or "luc")
+
+If you want to use views, add the option "-use-views" to the above command. This will create a separate index for each sub-directory in the import folder, and you can search each sub-view independently.
+
+Using a GUI tool:
+java vishnu.builder.IndexerGUI
+
+Fill in the fields as appropriate, and click "Start indexing".
+
+Once the collection is built, you need to rename the building directory to index. Collections are built into the building directory, but served from the index directory. This allows you to serve a built collection while rebuilding.
+
+For collections in the Greenstone 3 context, you will need to create the files etc/collectionConfig.xml and index/buildConfig.xml.
+These should look like the following:
+
+etc/collectionConfig.xml:
+<collectionConfig>
+  <metadataList>
+    <metadata name="creator">creator's email address</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Collection name</displayItem>
+    <displayItem name='description' lang='en'>Collection description
+    </displayItem>
+  </displayItemList>
+</collectionConfig>
+
+index/buildConfig.xml:
+<buildConfig>
+  <metadataList/>
+  <serviceRackList>
+    <serviceRack name="Visualizer">
+      <engineType name="MG"/> <!-- or LUCENE, depending on how you built the collection -->
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
+
+Note that you cannot use views in the Greenstone context yet.
+
+RUNNING THE SERVLET:
+
+
+If you are running this standalone, you need to run a servlet container for VisServlet.
+We have provided an example configuration for use with Tomcat (version 4.1).
+
+The vishnu/web directory is setup to be the context directory for vishnu in Tomcat. 
+You need to edit Tomcat's conf/server.xml file, and add in the context for vishnu. Add the following element after the tomcat root context element:
+
+<Context path="/vishnu" docBase="full-path-to-vishnu/web" debug="1" reloadable="true"><Resources allowLinking='true'/></Context>
+(replacing "full path-to-vishnu" with the path to the vishnu directory.)
+Edit the vishnu/web/WEB-INF/web.xml file and put in the correct path to your collections directory.
+
+The vishnu.jar file needs to be in your CLASSPATH, and make sure that Tomcat is using your CLASSPATH.
+
+RUNNING THE APPLET:
+
+The file vishnu/web/index.html is an introductory page about vishnu, and contains the applet. Make sure that the library parameter has the correct address for your visualiser servlet. The default is http://localhost:8080/vishnu/server. This will be correct if you have installed Tomcat to use port 8080, and have set the vishnu servlet up as described above. But note, if you are using Mozilla, it may not like localhost here, so if you have problems, try putting the full name of the server running Tomcat in place of localhost.
+
+If you are running Tomcat as described above, you can view this index file at http://localhost:8080/vishnu/index.html
Index: /other-projects/trunk/greenstone3-extension/vishnu/build.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/build.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/build.xml	(revision 14150)
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+
+<!-- ====================================================================== 
+     April 2005                                                    
+
+     Vishnu build script
+                   
+     kjdon                                                                
+     ====================================================================== -->
+<project name="vishnu" default="all" basedir=".">
+
+  <property environment="env"/>
+  <property name="build.dir" value="${basedir}/build"/>
+  <property name="src.dir" value="${basedir}/src"/>
+
+  <!-- if vishnu is installed as a package inside greenstone -->
+  <property name="prefix" value="../.."/>
+  <!-- use if vishnu is standalone -->
+  <property name="prefix" value="${basedir}"/>
+  <property name="catalina.home" value="${env.CATALINA_HOME}"/>
+  <property name="web.lib" value="${prefix}/web/WEB-INF/lib"/>
+  <property name="web.classes" value="${prefix}/web/WEB-INF/classes"/>
+
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="true"/>
+  <property name="compile.optimize"    value="true"/>
+
+  <!-- classpath : this is for vishnu with Greenstone. For standalone vishnu,
+  make sure all the specified jar files are on the environment CLASSPATH -->
+  <path id="compile.classpath">
+    <pathelement location="${prefix}/lib/jni/mg.jar"/>
+    <pathelement location="${web.lib}/lucene-1.4.1.jar"/>
+    <pathelement location="${catalina.home}/common/lib/servlet-api.jar"/>
+    <pathelement location="${catalina.home}/common/endorsed/xercesImpl.jar"/>
+    <pathelement path="${env.CLASSPATH}"/>
+    <!-- if not usign this with greenstone, don't need this -->
+    <pathelement location="${web.lib}/gsdl3.jar"/>
+  </path>
+
+  <path id="ld.library.path">
+    <pathelement location="${prefix}/lib/jni"/>
+  </path>
+  <target name="all" depends="clean,compile"/>
+
+  <target name="clean" depends=""
+    description="Remove the compiled code">
+    <delete dir="${build.dir}"/>
+  </target>
+  
+  <target name="compile" depends=""
+    description="Compile the code">
+    <mkdir dir="${build.dir}"/>
+    <javac srcdir="${src.dir}"
+      destdir="${build.dir}"
+      debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+      optimize="${compile.optimize}">
+      <classpath>
+	<path refid="compile.classpath"/>
+      </classpath>
+      <!-- uncomment this if not in a greenstone context -->
+      <!--<exclude name="org/**"/>-->
+    </javac>
+    <jar destfile="${build.dir}/vishnu.jar">
+      <fileset dir="${build.dir}">
+	<include name="vishnu/**"/>
+	<include name="org/**"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>      
+    <copy file="${build.dir}/vishnu.jar" todir="${web.lib}"/>
+  </target>
+</project>
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java	(revision 14150)
@@ -0,0 +1,295 @@
+/*
+ *    Visualiser.java
+ *    Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Text; 
+import java.util.Vector;
+import java.util.HashMap;
+import java.io.File;
+import java.io.*;
+import vishnu.server.*;
+import vishnu.server.Search.*;
+import vishnu.datablock.*;
+import vishnu.util.Base64;
+
+public class Visualizer
+    extends ServiceRack {
+    
+    // the services on offer
+    private static final String VIS_SERVICE = "VisApplet";
+
+    // other internal strings
+    private static final String ENGINE_TYPE_ELEM = "engineType";
+    private static final String LUCENE_ENGINE = "LUCENE";
+    private static final String MG_ENGINE = "MG";
+
+    private Element applet_description = null;
+    
+    private String engine_type = LUCENE_ENGINE; // lucene is default
+    private String collection_home = null;
+    private SearchInterface engine = null;
+    public Visualizer () {
+
+    }
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	Element e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+	e.setAttribute(GSXML.NAME_ATT, VIS_SERVICE);
+	short_service_info.appendChild(e);
+
+	applet_description = this.doc.createElement(GSXML.SERVICE_ELEM);
+        applet_description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+        applet_description.setAttribute(GSXML.NAME_ATT, VIS_SERVICE);
+
+
+	String app_info = "<"+GSXML.APPLET_ELEM+" CODEBASE='lib' CODE='vishnu.testvis.visual.VishnuSingle.class' ARCHIVE='vishnu.jar,gsdl3.jar,xercesImpl.jar,xml-apis.jar' WIDTH='1000' HEIGHT='800'>";
+	app_info += "<PARAM NAME='library' VALUE=''/>"; // filled in by receptionist
+	app_info += "<PARAM NAME='viscgi' VALUE='?";
+	app_info += GSParams.ACTION +"=a&amp;"+GSParams.REQUEST_TYPE +"=r&amp;"+GSParams.SERVICE+"="+VIS_SERVICE+"&amp;"+GSParams.OUTPUT+"=xml&amp;"+GSParams.RESPONSE_ONLY+"=1'/>";
+	app_info += "<PARAM NAME='collection'   VALUE='" + this.cluster_name + "'/>";
+	app_info += "<PARAM NAME='engine'   VALUE='GSDLEngine' />";
+	// add view info if appropriate
+	app_info += "The visualization applet.</"+GSXML.APPLET_ELEM+">";
+	   
+	Document dom = converter.getDOM(app_info);
+	if (dom == null) {
+	    System.err.println("Visualizer.configure Error: Couldn't parse applet info");
+	    return false;
+	}
+	Element app_elem = dom.getDocumentElement();
+	applet_description.appendChild(this.doc.importNode(app_elem, true));
+
+	// get engine type from config file
+	Element engine_elem = (Element)GSXML.getChildByTagName(info, ENGINE_TYPE_ELEM);
+	if (engine_elem != null) {
+	    engine_type = engine_elem.getAttribute(GSXML.NAME_ATT);
+	    if (engine_type.equals("")) {
+		engine_type = LUCENE_ENGINE;
+	    }
+	}
+	
+	collection_home = this.site_home + File.separator + "collect"+ File.separator + this.cluster_name + File.separator;
+	if (engine_type.equals(LUCENE_ENGINE)) {
+	    // make the full path the arg to MGSearcher??
+	    engine = new LUCSearcher(collection_home, null);
+	} else if (engine_type.equals(MG_ENGINE)) {
+	    engine = new MGSearcher(collection_home, this.cluster_name, null);
+	} else {
+	    System.err.println("Visualiser: invalid engine type: "+engine_type);
+	    return false;
+	}
+	return true;
+    }	
+    
+    
+    
+    protected Element getServiceDescription(String service, String lang, String subset) {
+	if (!service.equals(VIS_SERVICE)) {
+	    return null;
+	}
+	Element describe = (Element) applet_description.cloneNode(true);
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  
+							    getTextString(VIS_SERVICE+".name", lang)));
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  
+							    getTextString(VIS_SERVICE+".description", lang)));
+	return describe;
+    }
+    
+    protected Element processVisApplet(Element request)
+    {
+
+	Element param_elem = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_elem, false);
+
+	String type = (String)params.get("type");
+
+	// the result element
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, VIS_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// applet result info must be in appletInfo element
+	Element applet_data = this.doc.createElement(GSXML.APPLET_DATA_ELEM);
+	result.appendChild(applet_data);
+	Element vis_data = this.doc.createElement("visData");
+	applet_data.appendChild(vis_data);
+
+	String results = "";
+	if (type.equals("search")) {
+
+	    //Element cluster_results = this.doc.createElement("cluster");
+	    //vis_data.appendChild(cluster_results);
+	    //Element description_results = this.doc.createElement("descriptions");
+	    //vis_data.appendChild(description_results);
+
+	    String query = (String)params.get("q");
+	    System.err.println("the query was "+query);
+	    
+	    Vector doc_nums = new Vector();
+	    Vector descriptions = new Vector();
+	    
+	    engine.search(query);
+	    doc_nums = engine.getDocIdentifiers();
+	    descriptions = engine.getDocDescriptions();
+	    
+	    DataBlock db = generateDataBlock(collection_home, null, doc_nums, descriptions);
+	    System.err.println("got back data, now converting to string");
+	    try {
+		results = Base64.encodeObject(db);
+	    } catch (Exception e) {
+		System.err.println("trying to base64 encode the datablock, but exception happened: "+e);
+	    }
+	    System.err.println("after converting to string");
+// 	    // serialise the data block
+// 	    try {
+// 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+// 		ObjectOutputStream oos = new ObjectOutputStream(baos);
+// 		oos.writeObject(db);
+// 		oos.close();
+		
+// 		//results = baos.toString("UTF-8");
+// 		results = Base64.encode(baos.toByteArray());
+// 	    } catch (Exception e) {
+// 		System.err.println("Visualizer serialise data block error: "+e);
+// 	    }
+	    //System.err.println("num docs = "+doc_nums.size());
+	    //Cluster c = new Cluster();
+	    //results = c.getCluster(collection_home,doc_nums);
+	    //results.trim(); 
+	    //System.err.println("results = "+results);
+
+	    Text t = this.doc.createTextNode(results);
+	    vis_data.appendChild(t);
+
+// 	    for (int i=0; i<descriptions.size(); i++) {
+// 		String d = (String)descriptions.get(i);
+// 		Element de = GSXML.createTextElement(this.doc, "desc", d);
+// 		description_results.appendChild(de);
+// 	    }
+	    System.err.println("end of search");
+	} else if (type.equals("fetch")) {
+	    String docNum = (String)params.get("d");
+	
+	    Vector doc = engine.getDocContent(Integer.parseInt(docNum));
+  	    for (int i=0; i<doc.size(); i++) {
+		results += (String)doc.get(i) +"\n";
+	    }
+	    Text t = this.doc.createTextNode(results);
+	    vis_data.appendChild(t);
+	} else {
+	    System.err.println("invalid type sent to Visualiser process: "+type);
+	    return result;
+	}
+	System.err.println("returning result");
+	return result;
+    }
+
+    private DataBlock generateDataBlock(String collection_home, String view, Vector docNums, Vector descriptions) {
+
+	if (docNums.size()==0) {
+	    return null;
+	}
+	DataBlock data = new DataBlock();
+
+	CKServer ck_server = new CKServer(collection_home, view);
+	
+	/**** set data fields one by one and pass one what ever gets assembled ****/
+		
+	try{
+	    
+	    ck_server.setDescriptions(descriptions);
+	    
+	    /**** get candidate keywords ****/
+	    
+	    String[] keywords = ck_server.computeKeywords(docNums);
+	    
+	    data.words = keywords;
+	    
+	    
+	    /**** get sparse document * keyword matrix ****/
+	    
+	    SparseMatrix matrix = ck_server.getSparseMatrix(docNums);
+	    
+	    data.matrix = matrix;
+	    
+	    
+	    /**** get document indices, this is a subset of the original ****/
+	    /**** those without keywords are excluded                    ****/
+	    
+	    int[] docs = ck_server.getHitDocuments();
+	    
+	    data.docs = docs;
+	    
+	    
+	    String[] desc = ck_server.getHitDescriptions();
+	    
+	    data.descriptions = desc;
+	    
+	    
+	    /**** get 10 or so clusters ****/
+	    
+	    Vector[] clusters = ck_server.getClusters();
+	    
+	    data.clusters = clusters;
+	    
+	    
+	    double[][] centroids = ck_server.getCentroids();
+	    
+	    
+	    /**** send their centroids through sammon mapping ****/
+	    
+	    Point2D[] sammon = ck_server.getSammonMap(centroids);
+	    
+	    System.err.println("Num descriptions: " + descriptions.size());
+	    System.err.println("Num docs: " + docs.length);
+	    System.err.println("Num desc: " + desc.length);
+	    
+	    data.sammon = sammon;
+	    System.err.println("end of try in generatedatablock");
+	} catch (Exception e) {
+	    System.err.println("VisServlet: computing clustering Error: "+e);
+	    e.printStackTrace();
+	}
+	ck_server = null;
+	System.err.println("returning data");
+	return data;
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/CKIndexer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/CKIndexer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/CKIndexer.java	(revision 14150)
@@ -0,0 +1,395 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+import java.util.Hashtable;
+import vishnu.util.*;
+
+public class CKIndexer extends Indexer
+{
+
+    static private String delimiter = "()<>.,?!'\" "; 
+
+
+    private static void usage()
+    {
+        System.out.println("java Indexer/CKIndexer \n -i InputDirectory [$COLL_HOME/import/]) \n -o OutputDirectory [$COLL_HOME/index/ck_index/java] \n -r root directory of collection \n -list (name of file listing the docs to be indexed)[docs.lst]\n -d string containing chars to be eliminated from words\n");
+	System.exit(0);
+    }
+    
+    
+    public static void main(String[] args)
+    {
+        String in = null;
+        String out = null;
+        //String COLL_HOME = null;
+	//String docList = null;
+	
+        if( args.length == 0 ){
+            usage();
+            System.exit(0);
+        }
+	
+        int a = 0;
+        while( a < args.length ){
+            if( args[a].equals("-i") )
+                in = args[++a];
+            else if( args[a].equals("-o") )
+                out = args[++a];
+            //else if( args[a].equals("-r") )
+            //    COLL_HOME = args[++a];
+	    //else if( args[a].equals("-list") )
+	    //	docList = args[++a];
+	    else if( args[a].equals("-d") )
+		delimiter = args[++a];
+            a++;
+        }
+	
+	CKIndexer cki = new CKIndexer();
+
+	if (out==null || in==null) {
+	    usage();
+	    System.exit(0);
+	}
+	
+	//if( COLL_HOME == null ) usage();
+	//cki.setRootDirectory(COLL_HOME);
+	
+	//if( out == null ) out = COLL_HOME + "/index/ck_index/java/";
+	cki.setOutputDirectory(out);
+	
+	//if( in == null ) in = COLL_HOME + "/import/";
+	cki.setInputDirectory(in);
+	
+	//if( docList != null ) cki.setDocList(docList);
+	
+	cki.startIndexing();
+    }
+    
+    
+    public CKIndexer(){
+	super("CKIndexer");
+    }
+    
+    
+//     public void setDocList(String doclist)
+//     {
+// 	docList = doclist;
+//     }
+
+
+    public void startIndexing(){
+	
+	initialize();
+	/**** read in archive ****/
+	
+	Vector docs = getDocList(inputDir); //readDocList(docList);
+	
+	System.out.println(docs.size() + " documents read");
+	
+	/****  hash of ALL words of corpus + their document frequency ****/
+	
+	Hashtable corpus_hash = new Hashtable();
+	
+	
+	/**** readDocuments and populate hash ****/
+
+	System.out.println("First run: getting document frequencies of all words");
+	 
+	getDocFreqs(docs, corpus_hash);
+	
+	
+	/**** prune words to get candidate keywords ****/
+
+	int min = 3;
+	int max = (int)(0.3*docs.size());
+	int minLength = 4;
+	
+	int num_ck = pruneKeywords(corpus_hash, minLength, min, max);
+	
+	
+	/**** sort hashtable by value ****/
+	
+	Enumeration enumer = corpus_hash.keys();
+	String[] words_ck = new String[num_ck];
+	int[] freqs_ck = new int[num_ck];
+	
+	int count = 0;
+	while( enumer.hasMoreElements() ){
+	    words_ck[count] = (String)enumer.nextElement();
+	    freqs_ck[count] = -((Integer)corpus_hash.get(words_ck[count])).intValue();
+	    count++;
+	}
+	
+	Misc.quickSort(freqs_ck,words_ck);
+	
+	
+	/**** normalize document frequencies ****/
+	
+	float[] freqs_f = new float[num_ck];
+	
+	for( int i = 0; i < num_ck; i++ ){
+	    freqs_f[i] = (float)(-1*freqs_ck[i])/docs.size();
+	}
+	
+	
+	/**** save document frequencies and candidate keywords ****/
+	
+	printToFile(freqs_f,outputDir + "doc_freq");
+	printToFile(words_ck,outputDir + "ck_words");	
+	
+	
+	/**** create (ck_word, index hash) ****/
+	
+	Hashtable ck_hash = new Hashtable();
+	
+	for( int i = 0; i < words_ck.length; i++ ){
+	    ck_hash.put(words_ck[i],new Integer(i));
+	}
+	
+	
+	/**** read documents and store for each document the ck words it contains and their frequencies ****/
+
+	System.out.println("Second run: getting term frequencies of candidate keywords");
+	
+	Vector termFreq = getTermFreqs(docs,ck_hash);
+	
+	
+	/**** save term frequency vector ****/ 
+	
+	printToFile(termFreq,outputDir + "term_freq");
+    }
+    
+    
+    private int pruneKeywords(Hashtable corpus_hash, int minLength, int min, int max){
+	Set ck_set = corpus_hash.keySet();
+	Enumeration enumer = corpus_hash.keys();
+	
+	int count = 0;
+	int pcount = 0;
+      	
+	while( enumer.hasMoreElements() ) {
+	    
+	    String word = (String)enumer.nextElement();
+	    int freq = ((Integer)corpus_hash.get(word)).intValue();
+	    
+	    
+	    /**** if word is a number remove ****/
+	    
+	    try{
+		int i = Integer.parseInt(word);
+		corpus_hash.remove(word);
+		ck_set.remove(word);
+		continue;
+	    }catch(NumberFormatException ex){}
+
+
+	    /**** if word too rare or too freq remove ****/
+	    
+	    if( word.length() < minLength || freq < min || freq > max ){
+		corpus_hash.remove(word);
+		ck_set.remove(word);
+		continue;
+	    }
+	}
+	return corpus_hash.size();
+    }
+    
+    
+    private Vector getTermFreqs(Vector docs, Hashtable ck_hash)
+    {
+	Vector termFreqVector = new Vector();
+	
+	for( int i = 0; i < docs.size(); i++ ){
+
+	    Hashtable doc_ck_hash = new Hashtable();
+	    String doc = (String)docs.elementAt(i);
+	    
+	    try{
+		BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(doc)));
+		String line = "";
+		StringTokenizer strtok;
+		
+		while( (line=dis.readLine())!=null ){
+		    
+		    /**** read word by word ****/
+		    
+		    strtok = new StringTokenizer(line,delimiter);
+		    
+		    while( strtok.hasMoreTokens() ){
+			String word = strtok.nextToken();
+			Integer index = null;
+			if( (index = (Integer)ck_hash.get(word)) != null ){
+			    Integer count;
+			    
+			    if( (count = (Integer)doc_ck_hash.get(index)) != null ){
+				int newCount = count.intValue() + 1;
+				doc_ck_hash.put(index,new Integer(newCount));
+			    }else{
+				doc_ck_hash.put(index,new Integer(1));				
+			    }
+			}
+		    }
+		}
+		dis.close();
+	    }catch(Exception ex){ex.printStackTrace();}
+	
+	
+	    
+	    termFreqVector.addElement(doc_ck_hash);
+	}
+	
+	return termFreqVector;
+    }
+    
+
+    private void getDocFreqs(Vector docs, Hashtable corpus_hash)
+    {
+	for( int i = 0; i < docs.size(); i++ ){
+	    
+	    String doc = (String)docs.elementAt(i);
+	    
+	    Hashtable doc_hash = new Hashtable();
+	    
+	    try{
+		BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(doc)));
+		String line = "";
+		StringTokenizer strtok;
+		
+		while( (line=dis.readLine())!=null ){
+		    
+		    
+		    /**** read word by word ****/
+		    
+		    strtok = new StringTokenizer(line,delimiter);
+		    
+		    while( strtok.hasMoreTokens() ){
+			String word = strtok.nextToken();
+			
+			
+			if( (Object)doc_hash.get(word) == null ){
+			    
+			    
+			    /**** make a note that there is at least one occurrence ****/
+			    
+			    doc_hash.put(word,word);
+			    
+			    
+			    /**** add it to corpus_hash if necessary or increment the counter ****/
+			    
+			    Integer count;
+			    if( (count = (Integer)corpus_hash.get(word)) == null ){
+				
+				
+				/**** add it to corpus_hash ****/
+				
+				corpus_hash.put(word,new Integer(1));
+			    }
+			    else{
+				
+				/**** increment counter in doc_freq ****/
+				
+				Integer newCount = new Integer(count.intValue()+1);
+				corpus_hash.put(word,newCount);
+			    }
+			}
+		    }
+		}
+		dis.close();
+		
+	    } catch(IOException ex){ex.printStackTrace();}
+	}
+    }
+    
+    private Vector getDocList(String input_dir_name) {
+
+	File input_dir = new File(input_dir_name);
+	if (!input_dir.exists()) {
+	    System.err.println("CKIndexer Error: input directory "+input_dir_name+ " does not exist!");
+	    return null;
+	}
+
+	Vector file_vector = new Vector();
+	processDirectory(input_dir, file_vector);
+	return file_vector;
+    }
+    private void processDirectory(File dir, Vector file_list) {
+	File [] files = dir.listFiles();
+	for (int i=0; i<files.length; i++) {
+	    if (files[i].isDirectory()) {
+		processDirectory(files[i], file_list);
+	    } else {
+		file_list.add(files[i].getPath());
+	    }
+	}
+
+    }
+//     private Vector readDocList(String list)
+//     {
+// 	list = inputDir + "/" + "docs.lst";
+	
+// 	File docList = new File(list);
+	
+// 	if (!docList.exists()){
+// 	    System.out.println("No documents found");
+// 	    System.exit(0);
+// 	}
+	
+// 	Vector docs = new Vector();
+	
+// 	try{
+//             BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(docList)));
+// 	    String line = "";
+//             while( (line=dis.readLine())!=null ){
+// 		docs.addElement(new String(inputDir+"/" + line));
+// 	    }
+//             dis.close();
+// 	} catch(IOException ex){ex.printStackTrace();}
+	
+// 	return docs;
+//     }
+    
+    
+    private void printToFile(Object obj, String name){
+	
+	FileOutputStream fos = null;
+	ObjectOutputStream out = null;
+	
+	try{
+	    fos = new FileOutputStream(name);
+	    out = new ObjectOutputStream(fos);
+	    out.writeObject(obj);
+	    out.close();
+	}
+	catch(IOException ex){
+	    ex.printStackTrace();
+	}
+    }
+}
+
+
+	/*Hgram hg = new Hgram();
+	hg.setBinNumber(50);
+	hg.setLabel("Document frequency");
+	hg.setData(freqs_f,Hgram.RAW_DATA);
+	hg.update();*/
+
+ /*    private Map.Entry[] sort(Hashtable h)
+    {
+	Set set = h.entrySet();
+	Map.Entry [] entries = 
+	    (Map.Entry[])set.toArray(
+				     new Map.Entry[set.size()]);
+	Arrays.sort(entries, new Comparator() {
+		public int compare(Object o1, Object o2) {
+		    int i1 = ((Integer)((Map.Entry)o1).getValue()).intValue();
+		    int i2 = ((Integer)((Map.Entry)o2).getValue()).intValue();
+		    if( i1 < i2 ) return 0;
+		    else return 1;
+		}
+		public boolean equals(Object that) {
+		    return this==that;
+		}
+	    });
+	return entries;
+	}*/
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/CollectionBuilder.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/CollectionBuilder.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/CollectionBuilder.java	(revision 14150)
@@ -0,0 +1,252 @@
+package vishnu.builder;
+
+import java.util.*;
+import java.io.*;
+
+public class CollectionBuilder
+{
+    //String site;
+    String collID = null; // directory name
+    String collHome = null; // full path to collections directory
+    String collName = null; // descriptive name
+    String collDesc = null; // descriptive description :-)
+    String indexer = null; // mg or lucene
+    boolean use_views = false;
+    long numDocs=0;
+    Vector views = null;
+    private static void usage()
+    {
+	System.out.println("java vishnu.builder.CollectionBuilder\n -collsHome (directory where collections live) \n -collection (directory name of collection)\n -name (descriptive name for collection) \n -description (collection description)\n -index (\"mg\" or \"luc\")\n [-use_views ]");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	String collID = null;
+	String collsHome = null;
+	String collName = "";
+	String collDesc = "";
+	String indexer = null;
+	boolean use_views = false;
+	int a = 0;
+	while( a < args.length ){
+	    if (args[a].equals("-collsHome"))
+		collsHome = args[++a];
+	    else if (args[a].equals("-collection"))
+		collID = args[++a];
+	    else if( args[a].equals("-name") )
+		collName = args[++a];
+	    else if( args[a].equals("-description") )
+		collDesc = args[++a];
+	    else if( args[a].equals("-index") )
+		indexer = args[++a];
+	    else if (args[a].equals("-use_views")) 
+		use_views = true;
+	    a++;
+	}
+	
+	if (collsHome==null || collID==null || indexer==null) {
+	    usage();
+	    System.exit(0);
+	}
+	
+	if (collName.equals("")) {
+	    collName = collID;
+	}
+	if( !indexer.equals("mg") && !indexer.equals("luc") )
+	    indexer = "luc";
+	
+	if (!collsHome.endsWith(File.separator)) {
+	    collsHome += File.separator;
+	}
+	String coll_home = collsHome + collID + File.separator;
+	
+	CollectionBuilder cb = new CollectionBuilder(coll_home, collID, collName, collDesc, indexer, use_views);
+    }
+    
+    
+    
+    public CollectionBuilder(String coll_home, String coll_id, String coll_name,String coll_desc,  String ind, boolean use_views)
+    {
+	this.collHome = coll_home;
+	this.collID = coll_id;
+	this.collName = coll_name;
+	this.collDesc = coll_desc;
+	this.indexer = ind;
+	this.use_views = use_views;
+	
+	if (use_views) {
+	    views = new Vector();
+	}
+	if (!setupCollection()) {
+	    System.err.println("Couldn't setup the collection. Quitting build");
+	    return;
+	}
+	
+	/**** get text annotation ****/
+
+	FTIndexing();
+
+
+	/**** extract keywords ****/
+
+	CKIndexing();
+
+	/**** write out the config file ****/
+	printConfig();
+    }
+
+
+
+    private boolean setupCollection()
+    {
+	// check the import directory
+	File import_dir = new File(collHome + "import");
+	if (!import_dir.isDirectory()) {
+	    System.err.println("the import directory "+import_dir.getPath()+" doesn't exist. Quitting build");
+	    return false;
+	}
+	
+	
+	// create the building directory
+	try {
+	    File index_dir = new File(collHome + "building");
+	    if (!index_dir.exists()) {
+		index_dir.mkdir();
+	    } else if (!index_dir.isDirectory()) {
+		// index is present but is not a directory
+		System.err.println("Trying to create building dir but a file already exists at "+index_dir.getPath());
+		return false;
+	    }
+	} catch (Exception e) {
+	    System.err.println("CollectionBuilder.setupCollection Error: "+e);
+	    return false;
+	}
+
+	// create the view list
+	if (use_views) {
+	    // create a new view for each sub directory in import
+	    File [] sub_dirs = import_dir.listFiles();
+	    for (int i=0; i<sub_dirs.length; i++) {
+		if (sub_dirs[i].isDirectory()) {
+		    views.add(sub_dirs[i].getName());
+		}
+	    }
+	    if (views.size()==0) {
+		// we haven't found any subdirectories
+		System.err.println("no sub directories were found in the import dorectory ("+import_dir.getPath()+"), not using views");
+		use_views = false;
+
+	    }
+	}
+	return true;
+    }
+
+    private void printConfig() {
+
+	String index_type="";
+	if (indexer.equals("luc")) {
+	    index_type = "LUCENE";
+	} else if (indexer.equals("mg")) {
+	    index_type = "MG";
+	}
+	
+	try{
+	    File target = new File(collHome + "config.xml");
+	    PrintWriter out = new PrintWriter(new FileWriter(target),true);
+	
+	    StringBuffer buffer = new StringBuffer();
+	    buffer.append("<collection name=\"");
+	    buffer.append(collName);
+	    buffer.append("\" description=\"");
+	    buffer.append(collDesc);
+	    buffer.append("\" engine=\"");
+	    buffer.append(index_type);
+	    buffer.append("\" size=\"");
+	    buffer.append(String.valueOf(numDocs));
+	    buffer.append("\">\n");
+	    
+	    // add in views
+	    if (use_views) {
+		buffer.append("<viewList>\n");
+		for (int i=0; i<views.size(); i++) {
+		    buffer.append("<view name=\"");
+		    buffer.append(views.get(i));
+		    buffer.append("\"/>");
+		}
+		buffer.append("\n</viewList>\n");
+	    } 
+	    buffer.append("</collection>\n");
+		
+	    out.print(buffer.toString());
+	    out.close();
+
+	} catch (IOException e){ System.out.println("Error writing config file: " + e.toString());
+	}
+    }
+    
+
+    private void CKIndexing()
+    {
+	CKIndexer cki = new CKIndexer();
+	cki.setCollectionName(collID);
+	String in = collHome + "import"+File.separator;
+	String out = collHome + "building"+File.separator+"ck_index"+File.separator;
+	if (use_views) {
+	    for (int i=0; i<views.size(); i++) {
+		String view = (String)views.get(i);
+		System.out.println("calculating candidate keywords for view "+view);
+		cki.setOutputDirectory(out+view+File.separator);
+		cki.setInputDirectory(in+view+File.separator);
+		cki.startIndexing();
+	    }
+	} else {
+	    System.out.println("calculating candidate keywords");
+	    // just do the import dir as a whole
+	    cki.setOutputDirectory(out);
+	    cki.setInputDirectory(in);
+	    cki.startIndexing();
+	}
+    }
+    
+
+
+    private void FTIndexing()
+    {
+	String in = collHome + "import"+File.separator;
+	String out = collHome + "building"+File.separator + indexer + "_index"+File.separator;
+	Indexer the_indexer;
+	if (indexer.equals("luc") ) {
+	    the_indexer = new LuceneWrapper();
+	} else {
+	    the_indexer = new MGWrapper();
+	}
+	the_indexer.setCollectionName(collID);
+
+	if (use_views) {
+	    for (int i=0; i<views.size(); i++) {
+		String view = (String)views.get(i);
+		System.out.println("indexing the text for view "+view);
+		the_indexer.setOutputDirectory(out+view+File.separator);
+		the_indexer.setInputDirectory(in+view+File.separator);
+		the_indexer.startIndexing();
+	    }
+
+	} else {
+	    System.out.println("indexing the text");
+	    the_indexer.setOutputDirectory(out);
+ 	    the_indexer.setInputDirectory(in);
+ 	    the_indexer.startIndexing();
+	}
+
+    }
+}
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/FileDocument.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/FileDocument.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/FileDocument.java	(revision 14150)
@@ -0,0 +1,57 @@
+package vishnu.builder;
+
+import java.io.File;
+import java.io.Reader;
+import java.io.FileInputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.DateField;
+
+/** A utility for making Lucene Documents from a File. */
+
+public class FileDocument {
+  /** Makes a document for a File.
+    <p>
+    The document has three fields:
+    <ul>
+    <li><code>path</code>--containing the pathname of the file, as a stored,
+    tokenized field;
+    <li><code>modified</code>--containing the last modified date of the file as
+    a keyword field as encoded by <a
+    href="lucene.document.DateField.html">DateField</a>; and
+    <li><code>contents</code>--containing the full contents of the file, as a
+    Reader field;
+    */
+  public static Document Document(File f, String relative_path)
+       throws java.io.FileNotFoundException {
+	 
+    // make a new, empty document
+    Document doc = new Document();
+
+    // Add the path of the file as a field named "path".  Use a Text field, so
+    // that the index stores the path, and so that the path is searchable
+    //doc.add(Field.Text("path", f.getPath()));
+    doc.add(Field.Text("path", relative_path));
+    // Add the last modified date of the file a field named "modified".  Use a
+    // Keyword field, so that it's searchable, but so that no attempt is made
+    // to tokenize the field into words.
+    doc.add(Field.Keyword("modified",
+			  DateField.timeToString(f.lastModified())));
+
+    // Add the contents of the file a field named "contents".  Use a Text
+    // field, specifying a Reader, so that the text of the file is tokenized.
+    // ?? why doesn't FileReader work here ??
+    FileInputStream is = new FileInputStream(f);
+    Reader reader = new BufferedReader(new InputStreamReader(is));
+    doc.add(Field.Text("contents", reader));
+
+    // return the document
+    return doc;
+  }
+
+  private FileDocument() {}
+}
+    
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/Indexer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/Indexer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/Indexer.java	(revision 14150)
@@ -0,0 +1,89 @@
+package vishnu.builder;
+
+import java.io.File;
+abstract public class Indexer
+{ 
+    
+    // views stuff
+//     public static final int DEFAULT = 0;
+//     public static final int DIP = 1;
+//     public static final int REG = 2;
+//     public static final int LEM = 3;
+//     public static final int SYN = 4;
+
+//     public static final String[] types = new String[]{"full","dip","reg","lem","syn"};
+    
+//     boolean use_views = false;
+//     int type = -1;
+    String name = null;
+    //String limiter = null;
+    String outputDir = null;
+    String inputDir = null;
+    String collName = null;
+
+    public Indexer(String n)
+    {
+	name = n;		
+    }
+    
+    
+//     public void setUseViews(boolean use_v) {
+// 	use_views = use_v;
+//     }
+    
+//     public void setIndexingType(int t)
+//     {
+// 	type = t;
+//     }    
+    public void setOutputDirectory(String s)
+    {
+	outputDir = s;
+    }
+    
+//     public void setLimiter(String s)
+//     {
+//         limiter = s;
+//     }
+    
+    public void setInputDirectory(String s)
+    {
+        inputDir = s;
+    }
+    
+     
+    public void setCollectionName(String c)
+    {
+	collName = c;
+    }
+
+    // this should be called at the start of startIndexing()
+    protected void initialize() {
+
+	if( inputDir == null || outputDir == null ){
+	    System.err.println("Input dir or output dir is null");
+	    System.exit(0);
+	}
+	
+	/* check if index dir exists */
+	File in = new File(inputDir);
+	if (!in.isDirectory()) {
+	    System.err.println("input directory ("+inputDir+") not a directory, exiting...");
+	    System.exit(0);
+	}
+
+	/**** check if output directory exists, if not create it ****/	
+        File f = new File(outputDir);
+        if( !f.exists() ){
+	    try {
+		f.mkdirs();
+	    } catch (Exception e) {
+		System.err.println("Couldn't create output directory "+outputDir);
+		System.err.println(e);
+		System.exit(0);
+	    }
+	}
+	
+    }
+    
+    abstract public void startIndexing();
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/IndexerGUI.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/IndexerGUI.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/IndexerGUI.java	(revision 14150)
@@ -0,0 +1,126 @@
+package vishnu.builder;
+
+import java.awt.event.*;
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.SpringLayout;
+import java.io.File;
+import vishnu.util.SpringUtilities;
+
+public class IndexerGUI {
+    public final static boolean RIGHT_TO_LEFT = false;
+    public Toolkit toolkit = Toolkit.getDefaultToolkit();
+    private static JTextField[] textFields; 
+    private static JTextArea ja;
+
+    public static void addComponentsToPane(Container pane)
+    {
+	String [] labels = {"Collections directory: ", "Collection id (directory name): ", "Collection name: ","Collection description: ", "Indexer (\"mg\" or \"luc\"): ", "Use views (\"yes\" or \"no\"): "};
+        int numPairs = labels.length;
+        textFields = new JTextField[numPairs];
+	
+        pane.setLayout(new BoxLayout(pane,BoxLayout.Y_AXIS));
+	
+        JPanel p = new JPanel(new SpringLayout());
+	
+        for (int i = 0; i < numPairs; i++) {
+	    JLabel l = new JLabel(labels[i], JLabel.TRAILING);
+	    p.add(l);
+	    textFields[i] = new JTextField(30);
+	    l.setLabelFor(textFields[i]);
+	    p.add(textFields[i]);
+	}
+	
+	SpringUtilities.makeCompactGrid(p,numPairs, 2, 6, 6, //initX, initY
+                                6, 6);       //xPad, yPad
+	
+	pane.add(p);
+	
+	JPanel butPan = new JPanel();
+	butPan.setLayout(new BoxLayout(butPan,BoxLayout.X_AXIS));
+
+	Dimension minSize = new Dimension(10, 50);
+	Dimension prefSize = new Dimension(10, 50);
+	Dimension maxSize = new Dimension(Short.MAX_VALUE, 50);
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+	
+	JButton button = new JButton(" Start indexing ");
+        button.setBorder(BorderFactory.createBevelBorder(0));
+	button.addActionListener(new ActionListener(){
+                public void actionPerformed(ActionEvent e){
+			build();
+                }
+            });
+        butPan.add(button);
+
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+	
+        butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+
+	pane.add(butPan);
+
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+
+	ja = new JTextArea(5,5);
+        JScrollPane scrollingArea = new JScrollPane(ja);
+	scrollingArea.setBorder(BorderFactory.createBevelBorder(1));
+	pane.add(scrollingArea);
+     }
+
+    private static void build()
+    {
+	// get data
+	String colls_home = textFields[0].getText();
+	String coll_id = textFields[1].getText();
+	String coll_name = textFields[2].getText();
+	String coll_desc = textFields[3].getText();
+	String indexer = textFields[4].getText();
+	String use_views_str = textFields[5].getText();
+	String coll_home = colls_home;
+	if (!coll_home.endsWith(File.separator)) {
+	    coll_home += File.separator;
+	}
+	coll_home += coll_id + File.separator;
+
+	if (!indexer.equals("mg") || !indexer.equals("luc")) {
+	    indexer = "mg";
+	}
+	boolean use_views = false;
+	if (use_views_str.equals("yes")) {
+	    use_views = true;
+	}
+	String output = "Launching Collection Builder with \n collections home = "+colls_home + "\n collection id = "+coll_name+"\n indexer = "+indexer;
+	if (use_views) {
+	    output += "\n using views";
+	} else {
+	    output += "\n not using views";
+	}
+	ja.setText(output);
+
+	CollectionBuilder builder = new CollectionBuilder(coll_home, coll_id, coll_name, coll_desc, indexer, use_views); 
+	output += "\n Indexing Finished";
+	ja.setText(output);
+    }
+
+    private static void createAndShowGUI() {
+        JFrame.setDefaultLookAndFeelDecorated(true);
+        JFrame frame = new JFrame("Indexing Tool");
+        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+        addComponentsToPane(frame.getContentPane());
+
+        frame.pack();
+
+	Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+        int w = frame.getSize().width;
+        int h = frame.getSize().height;
+        int x = (dim.width-w)/2;
+        int y = (dim.height-h)/2;
+        frame.setLocation(x,y);
+
+	frame.setVisible(true);
+    }
+
+    public static void main(String[] args) {
+	createAndShowGUI();
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/LuceneWrapper.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/LuceneWrapper.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/LuceneWrapper.java	(revision 14150)
@@ -0,0 +1,162 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexWriter;
+
+
+public class LuceneWrapper extends Indexer
+{
+    
+    private static void usage()
+    {
+	System.out.println("java LucenceWrapper -i InputDirectory \n -o OutputDirectory \n -c CollectionName");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	String in = null;
+	String out = null;
+	String coll = null;
+
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	int a = 0;
+	while( a < args.length ){
+	    if( args[a].equals("-i") )
+		in = args[++a];
+	    if( args[a].equals("-o") )
+		out = args[++a];
+	    if( args[a].equals("-c") )
+		coll = args[++a];
+	    a++;
+	}
+	if (out==null || in==null|| coll==null) {
+	    usage();
+	    System.exit(0);
+	}
+	if (!out.endsWith(File.separator)) {
+	    out += File.separator;
+	}
+	if (!in.endsWith(File.separator)) {
+	    in += File.separator;
+	}
+
+	LuceneWrapper luc = new LuceneWrapper();
+	
+	out = out + "luc_index";
+	luc.setCollectionName(coll);
+        luc.setOutputDirectory(out);
+	luc.setInputDirectory(in);
+	luc.startIndexing();
+    }
+    
+    
+    public LuceneWrapper()
+    {
+	super("Lucene Indexer");
+    }
+    
+    public void startIndexing() {
+	initialize();
+	try {
+	    IndexWriter writer = new IndexWriter(outputDir, new StandardAnalyzer(), true);
+	    // this is a bit of a hack putting import here
+	    indexFiles(writer, new File(inputDir), "import/");
+	    writer.optimize();
+	    writer.close();
+	} catch (Exception e) {
+	    System.err.println("Error creating the Lucene index: "+e);
+	    System.exit(0);
+	}
+
+    }
+
+    private void indexFiles(IndexWriter writer, File input_dir, String current_path) {
+	File [] doc_list = input_dir.listFiles();
+	for (int i=0; i<doc_list.length; i++) {
+	    File f = doc_list[i];
+	    if (f.isDirectory()) {
+		indexFiles(writer, f, current_path+f.getName()+File.separator);
+	    } else {
+		try {
+		    writer.addDocument(FileDocument.Document(f, current_path+f.getName()));	
+		} catch (Exception e) {
+		    System.err.println("Error indexing document "+current_path+f.getName()+": "+e);
+		}
+	    }
+	}
+
+    }
+
+	
+    public void startIndexingOld()
+    {
+	
+	if( inputDir == null || outputDir == null ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	
+	/**** check if output directory exists, if not create it ****/
+	
+	
+        File f = new File(outputDir);
+        if( !f.exists() ){
+            System.out.println("Creating output directory..");
+	    
+            StringTokenizer strt = new StringTokenizer(outputDir,"/");
+            String d = "";
+            while(strt.hasMoreTokens()){
+                String str = strt.nextToken();
+                if( str.indexOf(".") == -1 ){
+		    d += "/" + str;
+		    if( !(new File(d)).exists() ){
+			try{
+			    Runtime rt = Runtime.getRuntime();
+			    Process proc = rt.exec("mkdir " + d);
+			    int exitVal = proc.waitFor();
+			}catch(Exception ex){ex.printStackTrace();}
+		    }
+		}
+	    }
+        }	
+	
+	File docList = new File(inputDir + "/docs.lst");
+	
+	if (!docList.exists()){
+	    System.out.println("No documents found");
+	    System.exit(0);
+	}
+	
+	Vector docs = new Vector();
+	
+	try{
+            BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(docList)));
+	    String line = "";
+            while( (line=dis.readLine())!=null ){
+		docs.addElement(new String(inputDir+"/" + line));
+	    }
+            dis.close();
+	} catch(IOException ex){ex.printStackTrace();}
+	
+	try{
+	    
+	    IndexWriter writer = new IndexWriter(outputDir, new StandardAnalyzer(), true);
+	    
+	    for( int i = 0; i < docs.size(); i++ ){
+		File file = new File((String)docs.elementAt(i));		
+		//writer.addDocument(FileDocument.Document(file));
+	    }
+	    writer.optimize();
+	    writer.close();
+        }catch(Exception ex){ex.printStackTrace();}
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/MGWrapper.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/MGWrapper.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/builder/MGWrapper.java	(revision 14150)
@@ -0,0 +1,123 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+
+public class MGWrapper extends Indexer
+{
+    private static void usage()
+    {
+	System.out.println("java Indexer MGWrapper -i InputDirectory \n -o OutputDirectory \n -c CollectionName");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	String in = null;
+	String out = null;
+	String coll = null;
+
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	int a = 0;
+	while( a < args.length ){
+	    if( args[a].equals("-i") )
+		in = args[++a];
+	    if( args[a].equals("-o") )
+		out = args[++a];
+	    if( args[a].equals("-c") )
+		coll = args[++a];
+	    a++;
+	}
+	
+	if (out==null || in==null|| coll==null) {
+	    usage();
+	    System.exit(0);
+	}
+	if (!out.endsWith(File.separator)) {
+	    out += File.separator;
+	}
+	if (!in.endsWith(File.separator)) {
+	    in += File.separator;
+	}
+	
+	out = out + "mg_index" + File.separator;
+	
+	MGWrapper mgw = new MGWrapper();
+	mgw.setCollectionName(coll);
+        mgw.setOutputDirectory(out);
+	mgw.setInputDirectory(in);
+	mgw.startIndexing();
+    }
+    
+    
+    public MGWrapper()
+    {
+	super("Managing Gigabytes Indexer");
+    }
+    
+    
+    public void startIndexing()
+    {
+
+	initialize();
+	// new mg requires a .mg_getrc to work
+	// lets put it in the output dir
+	File rc_file = new File(outputDir, ".mg_getrc");
+	if (!rc_file.exists()) {
+	    // if its there, assume it is correct
+	    try {
+		rc_file.createNewFile();
+		BufferedWriter rc_out = new BufferedWriter(new FileWriter(rc_file, true));
+		String data_line = collName+"\tDIR2\t"+inputDir;
+		rc_out.write(data_line, 0, data_line.length());
+		rc_out.newLine();
+		rc_out.flush();
+		rc_out.close();
+	    } catch (Exception ex) {
+		System.err.println(ex);
+		System.err.println("Unable to create file "+rc_file.getPath()+", cancelling build");
+		System.exit(0);
+	    }
+	}
+	/*
+	if( !f.exists() ){
+	    System.out.println("Creating output directory..");
+	    
+            StringTokenizer strt = new StringTokenizer(outputDir,"/"); 
+            String d = "";
+            while(strt.hasMoreTokens()){
+                String str = strt.nextToken();
+                if( str.indexOf(".") == -1 ){
+		    d += "/" + str;
+		    if( !(new File(d)).exists() ){
+			try{
+			    Runtime rt = Runtime.getRuntime();
+			    Process proc = rt.exec("mkdir " + d);
+			    int exitVal = proc.waitFor();
+			}catch(Exception ex){ex.printStackTrace();}
+		    }
+		}
+	    }	
+	}
+	*/
+	try{
+	    //String command = /*GSDL3_HOME +*/ "/packages/VIS/" + "MG/mg/mg-1.2.1/bin/mgbuild -d " + outputDir + " -i " + inputDir + " " + collName;
+	    //String command = "mgbuild -d " + outputDir + " -i " + inputDir + " " + collName;
+	    // requires .mg_getrc file
+	    String []env = new String [1];
+	    env[0] = "MG_GETRC="+outputDir+".mg_getrc";
+	    String command = "mgbuild -d " + outputDir + " " + collName;
+	    System.out.println("Executing.. " + command);
+	    Process proc = Runtime.getRuntime().exec(command, env);
+	    proc.waitFor();
+	    System.out.println(proc.exitValue());
+	}
+	catch(IOException ex){ex.printStackTrace();}
+	catch (InterruptedException ex){ System.out.println(ex); ex.printStackTrace();
+	} 
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/AveLink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/AveLink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/AveLink.java	(revision 14150)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class AveLink extends LinkFunc
+{
+    float link(float a, float b)
+	{
+  		return (a+b)/2;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Cluster.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Cluster.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Cluster.java	(revision 14150)
@@ -0,0 +1,76 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+public class Cluster
+{
+    public int id;
+    public Cluster  parent = null;
+    public Cluster  child_1 = null;
+    public Cluster  child_2 = null;
+    
+    public double children_sim;
+    
+    // size of cluster
+    public int items;
+    
+    public int split;
+    
+    // [] -> centroid for each vector element ?
+    public float[] centroid=null;
+    
+    public Vector iter_items = null;
+    public int iter_items_num = 0;
+    
+    public int x = 0;
+    public int y = 0;
+    
+    public float[] getCentroid(){return centroid;}
+    
+    public Cluster(int id, Cluster parent, Cluster c1, Cluster c2, double children_sim, int items)
+    {
+	this.id               = id;
+	this.children_sim     = children_sim;
+	this.items            = items;
+	split                 = 0;
+	this.parent           = parent;
+        iter_items            = new Vector(0);
+        
+        // probably to get a balanced tree
+	if (c1 != null && c2 != null){
+	    if (c2.items > c1.items){
+		child_1 = c1;
+		child_2 = c2;
+	    }else{
+	        child_1 = c2;
+		child_2 = c1;
+	    }
+	}else{ // if either or both are null
+	    child_1 = c1;
+	    child_2 = c2;
+	}
+    }
+    
+    // a nice recursive function to determine the depth of the hierarchy
+    public int depth()
+    {
+	if (id != -1)
+	    return 0;
+	else
+	    return Math.max(1 + child_1.depth(), 1 + child_2.depth());
+    }
+    
+    
+    void addChildrenToOutputVector(int []coding, Vector output)
+    {
+	if (id != -1){
+    	    Integer ii = new Integer(coding[id]);
+            output.addElement(ii);
+	}else{
+      	    child_1.addChildrenToOutputVector(coding,output);
+      	    child_2.addChildrenToOutputVector(coding,output);
+	}
+    }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/ClusterArray.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/ClusterArray.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/ClusterArray.java	(revision 14150)
@@ -0,0 +1,85 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+
+public class ClusterArray
+{
+    Vector contents;
+    
+    ClusterArray(int num_of_clusters)
+    {
+	contents = new Vector(num_of_clusters);
+	
+	for(int i = 0; i < num_of_clusters; i++)
+	    contents.addElement(new Cluster(i, null, null, null, 0, 1));
+    }
+    
+    
+    // counts the number of clusters that are less dissimilar than disim
+    private void sweep_tree (Cluster cluster, double disim, int min_c_items, int[] clusters)
+    {
+	cluster.split = 0;
+	
+	if (cluster.id == -1){
+	    if (cluster.children_sim <= disim && cluster.items >= min_c_items){
+		cluster.split = 1;
+		contents.addElement(cluster);
+		clusters[0]++;
+	    }else{
+		sweep_tree (cluster.child_1, disim, min_c_items, clusters);
+		sweep_tree (cluster.child_2, disim, min_c_items, clusters);
+	    }
+    	}else{
+	    if (cluster.items >= min_c_items){
+		cluster.split = 1;
+		contents.addElement(cluster);
+		clusters[0]++;
+	    }
+	}
+    }
+
+
+    double split (Cluster cluster, double[] heights, int n, int target, int min_c_items, int[] clusters)
+    {
+	
+	/*
+	  Try to split 'cluster', made up of 'n' vectors, into 'target'
+	  clusters by incrementing iteratively the mimimum required
+	  similarity between the 'cluster\'s' children. The result is an
+	  array of clusters, 'cluster_ary' which contains 'clusters'
+	  clusters.  Obviously the final number of clusters may not be the
+	  same as the desired number of clusters. The function return the
+	  threshold similarity which resulted in the clusters being split
+	  in 'clusters' clusters.
+	*/
+	
+	int i;
+	
+	if (target < 2) target = 2;
+	
+	if (target > n) target = n;
+	
+	// height store minimum similarity values
+	// this is then the iterative incrementing loop
+	for (i = target - 2; i <= n - 1; i++){
+	    clusters[0] = 0;
+	    contents.removeAllElements();
+	    sweep_tree (cluster, heights[i], min_c_items, clusters);
+	    
+	    if (clusters[0] >= target) break;
+	}
+	
+	// if target - 2 > n - 1, i.e. if the previous loop has not been 
+	// entered at all
+	if (i > n - 1){
+	    /* ignore the min_c_size requirement */
+	    i = target - 1;
+	    clusters[0] = 0;
+	    contents.removeAllElements();
+	    sweep_tree (cluster, heights[i], 1, clusters);
+	}
+	
+	return heights[i];
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Clustering.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Clustering.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Clustering.java	(revision 14150)
@@ -0,0 +1,128 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+
+public class Clustering
+{
+    public static final int COMPLETE_LINKAGE = 0;
+    public static final int AVERAGE_LINKAGE  = 1;
+    public static final int SINGLE_LINKAGE   = 2;
+    public static final int BUCKSHOT = 0;
+    public static final int COMPLETE = 1;
+    
+    private Vector [] clusters;
+    private double [][] centroids;
+    
+    public Vector [] getClusters()
+    {
+        return clusters;
+    }
+
+    public double[][] getCentroids()
+    {
+        return centroids;
+    }
+    
+    
+    public Clustering (int linkage, float[][] KDMatrix, int min_c_items, int target_clusters, int limit)
+    {
+        SimMatrix sim_matrix;
+	LinkFunc linkage_func=null;
+	
+        int wordCount = KDMatrix[0].length;
+	int hitDocs = KDMatrix.length;
+        
+	Cluster toplevel_cluster;
+        ClusterArray splitclusters = null;
+	int[] n_clusters = new int[]{0};
+	
+        double threshold;
+	
+        Distance sim_func = new EuclideanDistance();
+	
+	double[] heights;
+
+        int[] coding = new int[hitDocs];
+        
+
+        for (int c = 0; c < hitDocs; c++) coding[c] = c;
+	
+	switch (linkage){
+	case COMPLETE_LINKAGE:
+	    linkage_func = new MaxLink();
+	    break;
+	case AVERAGE_LINKAGE:
+	    linkage_func = new AveLink();
+	    break;
+	case SINGLE_LINKAGE:
+	    linkage_func = new MinLink();
+	    break;
+	default:
+	    System.exit(0);
+	}
+	
+
+        /**** make sure upper bound does not exceed #docs ****/
+        
+	if(limit > hitDocs) limit = hitDocs;
+	limit = (int)hitDocs/2;
+	
+
+	/**** get similarity matrix of the first limit docs with each other ****/
+	
+        sim_matrix = new SimMatrix(KDMatrix, limit, sim_func);
+	
+	heights = new double[hitDocs];
+	
+        Hierarchical hierarchical = new Hierarchical(limit);
+        
+        toplevel_cluster = hierarchical.clustering(sim_matrix, limit, heights, linkage_func);
+	if( toplevel_cluster == null )
+	    return;
+	
+        System.out.println("Clustered items: " + toplevel_cluster.items);
+	Cluster c1 = toplevel_cluster.child_1;
+	Cluster c2 = toplevel_cluster.child_2;
+	System.out.println("Child1: " + c1.items);
+	System.out.println("Child2: " + c2.items);
+	
+	splitclusters = new ClusterArray(0);
+        
+	threshold = splitclusters.split(toplevel_cluster, heights, limit, target_clusters, min_c_items, n_clusters);
+	
+    	System.out.println(n_clusters[0] + " clusters");
+	System.out.println("Threshold: " + threshold);
+	
+        System.out.println("Target_clusters " + target_clusters);
+	System.out.println("Limit " + limit);
+	System.out.println("#docs: " + hitDocs);
+	
+        Iterative iter = new Iterative(wordCount);
+	
+	iter.clustering(splitclusters, n_clusters, KDMatrix, limit, hitDocs, sim_func);
+	
+	System.out.println("Finished iterative clustering");
+
+        clusters = new Vector[n_clusters[0]];
+        centroids = new double[n_clusters[0]][wordCount];
+
+	for (int i = 0; i < n_clusters[0]; i++){
+	    clusters[i]=new Vector();
+	    Cluster clust = (Cluster)splitclusters.contents.elementAt(i);
+	    clust.addChildrenToOutputVector(coding,clusters[i]);
+	    
+	    for (int j = 0; j < clust.iter_items_num; j++){
+		Integer ii = (Integer)clust.iter_items.elementAt(j);
+		Integer jj = new Integer (coding[ii.intValue()]);
+		clusters[i].addElement(jj);
+	    }
+	    
+	    float [] centroid = clust.getCentroid();
+	    
+	    for (int j = 0; j < wordCount; j++){
+		centroids[i][j]=centroid[j];
+	    }
+	}
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Distance.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Distance.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Distance.java	(revision 14150)
@@ -0,0 +1,10 @@
+package vishnu.cluster;
+
+// Abstract class
+
+public abstract class Distance
+{
+	public abstract float get(float[] v1, float[] v2);
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/EuclideanDistance.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/EuclideanDistance.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/EuclideanDistance.java	(revision 14150)
@@ -0,0 +1,15 @@
+package vishnu.cluster;
+
+public class EuclideanDistance extends Distance
+{
+    public float get(float[] v1, float[] v2)
+    {
+  	float sum = 0;
+
+	for(int c=0;c<v1.length;c++){
+      	    float x = v1[c] - v2[c];
+	    sum += x*x;
+	}
+  	return (float)Math.sqrt(sum);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Hierarchical.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Hierarchical.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Hierarchical.java	(revision 14150)
@@ -0,0 +1,234 @@
+package vishnu.cluster;
+
+public class Hierarchical
+{
+    public float[] minrow;
+    public int[] colind;
+    public TriangleIndex triangle;
+    
+    public Hierarchical(int matrix_rows){
+	// create Triangle with "limit" arg = maximum #clusters
+	// an object to help with indexing the cells 
+	// in a triangular matrix
+	triangle = new TriangleIndex(matrix_rows);
+    }
+    
+    
+    // set each minrow[i] to the value of the minimum element of that row
+    void set_minrow (SimMatrix m, int row)
+    {
+	// triangular matrix without diagonal, hence zeroth row is all zero
+        if(row == 0) return;
+	
+	int i;
+	int index_start, index_end;
+        
+        // first column
+	index_start = triangle.index (row, 0);
+
+	// last column (row(!)-1 because as many rows as columns)
+	index_end   = triangle.index (row, row - 1); 
+	
+	minrow[row] = m.matrix[index_start];
+	colind[row] = index_start;
+	
+	for (i = index_start + 1; i <= index_end; i++){
+	    if (m.matrix[i] < minrow[row]){
+		minrow[row] = m.matrix[i];
+		colind[row] = i;
+	    }
+	}
+    }
+    
+    
+    // return the ROW in which the minimum disimilarity resides
+    int get_minrow (int last_row){
+	int i;
+	int row;
+	float value;
+	
+	row   = 1;
+	value = minrow[1];
+	
+	for (i = 2; i <= last_row; i++){
+	    if (minrow[i] < value){
+		row = i;
+		value = minrow[i];
+	    }
+    	}
+	
+	return row;
+    }
+
+    
+    // num_of_rows is "limit" = max docs to be clustered in hierarchical clustering
+    Cluster clustering (SimMatrix sim_matrix, int num_of_rows, double[] heights, LinkFunc linkage_func)
+    {
+	Cluster child_1;
+	Cluster child_2;
+	Cluster parent=null;
+	ClusterArray cluster_ary;
+	
+        int min_row, index, row, col, i;
+	double sim;
+	
+        int last_row_index, sim_matrix_size;
+	
+        sim_matrix_size = num_of_rows *(num_of_rows - 1) / 2;
+	
+	// Place each of the documents in a cluster of each own 
+	// there are vector.length docs but only num_of_rows = limit places in the array
+	// may be two-stage clustering process, i.e. cluster subset of docs and then add the rest
+	cluster_ary = new ClusterArray (num_of_rows);
+	last_row_index = num_of_rows - 1;
+	
+	// minimum distance for each row of the sim_matrix
+	minrow = new float[num_of_rows];
+	
+	// column index for minimum distance entry
+	colind = new int[num_of_rows];
+	
+	for (i = 1; i < num_of_rows; i++) set_minrow(sim_matrix, i);
+	
+	// do all the hierarchical clustering by running over all rows, updating the matrix 
+	// as a new cluster is formed each time
+	for (i = 0; i < num_of_rows - 1; i++){
+
+	    // find the row with the lowest disimilarity
+	    min_row = get_minrow(last_row_index);
+	    sim = minrow[min_row];
+	    
+	    System.out.println("Min sim: " + sim);
+	    
+	    // the column with the lowest distance in that row
+	    index  = colind[min_row];
+	    
+	    // array of heights, DESCENDING disim, i.e. sim goes last
+	    heights[num_of_rows - 2 - i] = sim; 
+	    
+	    row = triangle.i_ind[index];
+	    col = triangle.j_ind[index];
+	    
+	    // merges rows and cols to create a smaller sim_matrix
+	    merge (row, col, sim_matrix, last_row_index, linkage_func);
+	    System.out.println("Merging: " + row + " and " + col);
+	    
+	    sim_matrix_size -= last_row_index;
+	    
+	    child_1 = (Cluster)cluster_ary.contents.elementAt(row);
+	    child_2 = (Cluster)cluster_ary.contents.elementAt(col);
+	    
+	    // parents get id = -1 to indicate that they are not representing single docs
+	    // all leaves have an id (1 - num_rows) that points to the doc they contain
+	    int sum = child_1.items + child_2.items;
+	    System.out.println("Create new parent with " + child_1.items + " + " + child_2.items + " leaves");
+	    parent = new Cluster (-1, null, child_1, child_2, sim, child_1.items + child_2.items);
+	    
+	    child_1.parent = parent;
+	    child_2.parent = parent;
+	    
+	    cluster_ary.contents.setElementAt(cluster_ary.contents.elementAt(last_row_index),row);
+	    cluster_ary.contents.setElementAt(parent,col);
+	    
+	    last_row_index--;
+	}
+	
+	return parent;
+    }
+    
+    // form a new cluster from the two most similar documents
+    void merge (int i, int j, SimMatrix m, int n, LinkFunc linkage_func)
+    {
+	
+	/* i and j are the matrix elements with maximum similarity         */
+	/* matrix is the triangle similarity matrix with last row index n  */
+	/* linkage_func is a pointer to the linkage function               */
+	
+	/* i > j */
+	
+	int c, k, index1, index2;
+	
+        // go over all rows
+	for (k = 0; k <= n; k++){
+	    
+    	    // if k < j the affected elements are on row j
+	    if (k < j){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	    }else if (k > j && k < i){
+		/* if j < k < i then only one element is updated per row */
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+	    }else if (k > i && k < n){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		//printf ("3 - [%d, %d] = link ([%d, %d], [%d, %d]) = %f\n", k, j, k, i, k, j, matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+		
+		index1 = triangle.index (k, i);
+		index2 = triangle.index (k, n);
+		m.matrix[index1] = m.matrix[index2];
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+	    }else if(k == n  && i != n){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+		
+		for (c = 0; c < i; c++){
+		    index1 = triangle.index (c, i);
+		    index2 = triangle.index (c, n);
+		    m.matrix[index1] = m.matrix[index2];
+		}
+		
+	    }
+    	}
+	set_minrow(m, i);
+	set_minrow(m, j);
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Iterative.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Iterative.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/Iterative.java	(revision 14150)
@@ -0,0 +1,102 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+class Iterative
+{
+    Vector vector;
+    int wordCount;
+    
+    Iterative(int vsize)
+    {
+	wordCount = vsize;
+    }
+    
+    public void printVector(double[] a)
+    {
+        int i;
+	
+	for (i = 0; i < wordCount; i++)
+    	    System.out.println("v[" + i + "] = " + a[i]);
+    }
+
+
+    void cluster_center (Cluster cluster, float [][] vectors, float[] centroid)
+    {
+        int i;
+	
+      	if (cluster.id == -1){
+      	    cluster_center (cluster.child_1, vectors, centroid);
+      	    cluster_center (cluster.child_2, vectors, centroid);
+    	}else{
+            float[] doubleArray = vectors[cluster.id];
+      	    for (i = 0; i < doubleArray.length; i++)
+		centroid[i] += doubleArray[i];
+        }
+    }
+    
+    void compute_cluster_centers (ClusterArray clusters, int num_of_clusters, float [][] vectors)
+    {
+	int i, j; double ssum;
+	Cluster clust;
+	
+
+	/**** for each hierarchical cluster, compute centroid ****/
+	for (i = 0; i < num_of_clusters; i++){
+	    clust = (Cluster)clusters.contents.elementAt(i);
+	    clust.centroid = new float[wordCount];
+	    cluster_center (clust, vectors, clust.centroid);
+	    
+	    for (ssum = j = 0; j < wordCount; j++)
+		ssum += clust.centroid[j] * clust.centroid[j];
+	    
+	    if(ssum>0)                // normalise
+		for(ssum = 1.0/Math.sqrt(ssum), j = 0; j < wordCount; j++)
+		    clust.centroid[j] *= ssum;
+	}
+    }
+    
+    
+    void place_item (ClusterArray clusters, int num_of_clusters, float [][] vectors, int item_id,  Distance sim_func)
+    {
+        Cluster clust = (Cluster)clusters.contents.elementAt(0);
+	double minDist = sim_func.get(clust.centroid,vectors[item_id]);	
+	int x_minDist = 0;
+	
+	for (int i = 1; i < num_of_clusters; i++){
+            clust = (Cluster)clusters.contents.elementAt(i);
+      	    double dist = sim_func.get(clust.centroid,vectors[item_id]);
+	    
+            if (dist < minDist){
+		x_minDist = i;
+		minDist = dist;
+	    }
+    	}
+	
+	//System.out.println("in " + x_minDist); 
+	clust = (Cluster)clusters.contents.elementAt(x_minDist);
+        clust.iter_items.addElement(new Integer(item_id));
+	clust.iter_items_num++;
+    }
+    
+    
+    // add all documents not yet added to the existing cluster hierarchy (which is based on the 
+    // limit first documents. Hence there are vector.length - limit docs to add
+    // consequently index runs from limit (start_index) to vector.length (end_index)
+    void clustering (ClusterArray clusters, int[] num_of_clusters, float[][] vectors, int start_index, int end_index,Distance sim_func)
+    {
+        int i;
+
+	System.out.println("Start: " + start_index + " Finish: " + end_index);
+	
+	if (start_index >= end_index)
+	    return;
+	
+	compute_cluster_centers(clusters, num_of_clusters[0], vectors);
+	
+	for (i = start_index; i < end_index; i++){
+	    System.out.println("Placing document " + i);
+	    place_item(clusters, num_of_clusters[0], vectors, i, sim_func);
+	}
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/LinkFunc.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/LinkFunc.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/LinkFunc.java	(revision 14150)
@@ -0,0 +1,8 @@
+package vishnu.cluster;
+
+//Abstract linkfunctions class
+
+public abstract class LinkFunc
+{
+	abstract float link(float a, float b);
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/MaxLink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/MaxLink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/MaxLink.java	(revision 14150)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class MaxLink extends LinkFunc
+{
+	public float link(float a, float b)
+	{
+  		return (a > b? a*99+b: b*99+a)/100;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/MinLink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/MinLink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/MinLink.java	(revision 14150)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class MinLink extends LinkFunc
+{
+	public float link(float a, float b)
+	{
+  		return (a < b? a*999+b: b*999+a)/1000;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/SimMatrix.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/SimMatrix.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/SimMatrix.java	(revision 14150)
@@ -0,0 +1,70 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+public class SimMatrix
+{
+    
+    public float[] matrix;
+    
+
+    public SimMatrix(float[][] KDMatrix, int limit, Distance simfunc){
+	
+	float iarray[];
+	
+	int hitDocs = KDMatrix.length;
+	int wordCount = KDMatrix[0].length;
+	int index = 0;
+	
+	// to exclude diagonal and conjugates
+	// - - - - -
+	//         |
+	// *       |
+	// * *     |
+	// * * *   |    
+	
+	int realSize = (hitDocs * (hitDocs - 1)/2);
+	matrix = new float[realSize];
+	
+	// this gets the sim matrix for all docs rather 
+	// than only the first limit - mistake?
+
+	for(int i = 1; i < hitDocs; i++){
+	    iarray = KDMatrix[i];
+	    for(int j = 0; j < i; j++){
+		matrix[index] = simfunc.get(iarray,KDMatrix[j]);
+	        index++;
+	    }
+	}
+    }
+
+    public void printMatrix()
+    {
+	int countA = 0;
+	int countB = 0;
+
+	for( int i = 0; i < matrix.length; i++ ){
+	    System.out.print(matrix[i] + " ");
+	    if( countA >= countB ){
+		System.out.println();
+		countB++;
+		countA = 0;
+	    }
+	    else countA++;
+	}
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TheInt.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TheInt.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TheInt.java	(revision 14150)
@@ -0,0 +1,11 @@
+package vishnu.cluster;
+
+public class TheInt
+{
+	public int val;
+
+	TheInt(int x)
+	{
+		val = x;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TheVectors.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TheVectors.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TheVectors.java	(revision 14150)
@@ -0,0 +1,79 @@
+package vishnu.cluster;
+
+import java.io.*;
+import java.util.*;
+
+public class TheVectors
+{
+    public Vector theData;
+    public int size;
+    public Vector coding;
+    public int zeros;
+
+
+    TheVectors(String file, int VECTOR_SIZE)
+    {
+        theData= new Vector(VECTOR_SIZE);
+	coding= new Vector(VECTOR_SIZE);
+        int index,code =0, zerolist=0;
+	double ssum,value;
+    	try
+	{
+      	    FileReader inFile = new FileReader(file);
+      	    BufferedReader inLines = new BufferedReader(inFile);
+	    size=0;
+            String line = null, tokenString;
+	    do
+	    {
+
+		line = inLines.readLine();
+		if (line!= null)
+		{
+
+		  double[] row= new double[VECTOR_SIZE];
+ 					StringTokenizer tokens = new StringTokenizer(line," ");
+                  ssum=0.0;
+   		  for (index=0; index<VECTOR_SIZE; index++)
+		  {
+                      if(tokens.hasMoreTokens())
+                      {
+                          tokenString = tokens.nextToken();
+           	          value = Double.parseDouble(tokenString);
+		          row[index]= value;
+		          ssum+= value*value;
+                      }
+                      else
+                      {
+                          row[index]=0.0;
+                      }
+	          }
+	          if(index==0) continue;
+                  ssum=Math.sqrt(ssum);
+	          if(ssum!=0.0)
+	          {
+	              for(int i=0;i<index;i++)
+		      row[i]/=ssum;
+		      theData.addElement(row);
+		      coding.addElement(new TheInt(code));
+                      size++;
+		  }
+		  else
+		  {
+		      if(zerolist==0)
+		      {
+			//System.out.print("0 ");
+	                  zerolist=1;
+		      }
+		      System.out.print(code + " ");
+		  }
+	      }
+	      code++;
+	  }while(line != null);
+	}
+	catch(IOException e)
+	{
+	    System.err.println("IO Exception in read_vectors");
+	    e.printStackTrace();
+        }
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TriangleIndex.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TriangleIndex.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/cluster/TriangleIndex.java	(revision 14150)
@@ -0,0 +1,85 @@
+package vishnu.cluster;
+
+class TriangleIndex
+{
+	short [] i_ind;    /* from index to i */
+	short [] j_ind;    /* from index to j */
+
+
+
+	TriangleIndex(int matrix_rows)
+	{
+
+  		short i, j;
+        int theIndex;
+
+  		int num_of_elements = matrix_rows * (matrix_rows - 1) / 2;
+
+  		/* Allocate space for lookup tables */
+  		i_ind = new short[num_of_elements];
+  		j_ind = new short [num_of_elements];
+
+	  	for (i = 1; i < matrix_rows; i++)
+    	{
+      		for (j = 0; j < i; j++)
+			{
+	  			theIndex = TriangleIndex.index (i, j);
+				i_ind[theIndex] = i;
+	  			j_ind[theIndex] = j;
+			}
+    	}
+	}
+
+	int row_size (int size)
+	{
+  /* Don't forget that we actually start counting in the second row, since the first one is assumed to be all zeroes */
+  		int i = 1;
+  		int sum = 0;
+
+  		while (sum != size)
+    		sum += i++;
+  		return i;
+	}
+
+	static int index (int i, int j)
+	{
+  /* Assumes lower triangular matrix, i.e. i>j */
+
+  		int temp;
+
+  /* The matrix in this case is symmetric so (i, j) = (j, i) */
+  		if (i < j)
+    	{
+      		temp = i; i = j; j = temp;
+    	}
+
+  		if (i != j)
+    		return (i*i-i)/2 + j;
+    		//(i - 1) * (i - 2) / 2 + i - 1 + j;
+  		else         /* The elements on the diagonal are all 0 */
+    	{
+      		return -1; /* so just return the dummy value -1      */
+    	}
+	}
+	static void print (double[] matrix, int row_size)
+	{
+  		int i, j, theIndex;
+
+  		for (i = 0; i < row_size; i++)
+    	{
+      		for (j = 0; j < row_size; j++)
+			{
+	  			if (i <= j)
+	    			System.out.print(" ");
+	  			else
+	    		{
+	      			theIndex = index (i, j);
+	      			System.out.print(matrix[theIndex] + " ");
+	    		}
+			}
+      		System.out.println();
+    	}
+	}
+
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Collection.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Collection.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Collection.java	(revision 14150)
@@ -0,0 +1,23 @@
+package vishnu.datablock;
+
+import java.io.*;
+
+public class Collection implements Serializable
+{
+    public String dir;
+    public String name;
+    public String desc;
+    public int size;
+    public String engine;
+    public String [] views;
+    
+    public Collection(String di, String n, String d, int s, String e, String [] v)
+    {
+	dir = di;
+	name = n; 
+	desc = d; 
+	size = s;
+	engine = e;
+	views = v;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/DataBlock.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/DataBlock.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/DataBlock.java	(revision 14150)
@@ -0,0 +1,35 @@
+package vishnu.datablock;
+
+import java.io.*;
+import java.util.*;
+
+
+
+public class DataBlock implements Serializable
+{
+    public int [] docs = null;
+    public String [] descriptions = null;
+    public String [] words = null;
+    public SparseMatrix matrix = null;
+    public Vector [] clusters = null;
+    public Point2D [] sammon = null;
+
+    public DataBlock() {
+    }
+    public DataBlock(int[] docs,
+		     String [] descriptions,
+		     String [] words,
+		     SparseMatrix matrix,
+		     Vector [] clusters,
+		     Point2D [] sammon)
+    {
+	this.docs = docs;
+	this.descriptions = descriptions;
+        this.words = words;
+	this.matrix = matrix;
+	this.clusters = clusters;
+	this.sammon = sammon;
+    }
+    
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Node.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Node.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Node.java	(revision 14150)
@@ -0,0 +1,37 @@
+package vishnu.datablock;
+
+public class Node
+{
+    Node next;
+    int value;
+    
+    public Node(int i, Node n)
+    {
+        value = i;
+        next = n;
+    }
+    
+    public Node(int i)
+    {
+        value = i;
+        next = null;
+    }
+    
+    public Node push(Node n)
+	{
+        if (next != null) next.push(n);
+        else next = n;
+        return this;
+    }
+
+	public Node pop()
+	{
+		return next;
+	}
+	
+	public int getValue()
+	{
+	    return value;
+	}
+  
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Point2D.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Point2D.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Point2D.java	(revision 14150)
@@ -0,0 +1,21 @@
+package vishnu.datablock;
+import java.io.*;
+/** 
+* Point2D.java: Class for points in logical coordinates.
+* From Section 1.5 of
+* @author Ammeraal, L. (1998) Computer Graphics for Java Programmers,
+* Chichester: John Wiley.
+*/       
+public class Point2D  implements Serializable
+{
+   /** x cartesian coordinate */
+   public float x;
+   /** y cartesian coordinate */
+   public float y;
+   /**
+   * constructor
+   * @param cartesian x coordinate
+   * @param cartesian y coordinate
+   */
+   public Point2D(float x, float y){this.x = x; this.y = y;}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Query.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Query.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/Query.java	(revision 14150)
@@ -0,0 +1,69 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+
+package vishnu.datablock;
+import java.io.*;
+
+public class Query implements Serializable
+{
+    public String query = null;
+    public String collection = null;
+    public int docNum = -1;
+    public String action = null;
+
+    public Query(){}
+
+    public Query(String q, String c, String s)
+    {
+	query = q;
+	collection = c;
+    }
+
+    public void setCollection(String c)
+    {
+	collection = c;
+    }
+
+    public void setDocNum(int id)
+    {
+	docNum = id;
+    }
+
+    public void setQuery(String q)
+    {
+	query = q;
+    }
+
+    public void setAction(String a)
+    {
+	action = a;
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/ShortFileException.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/ShortFileException.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/ShortFileException.java	(revision 14150)
@@ -0,0 +1,6 @@
+package vishnu.datablock;
+public class ShortFileException extends Exception
+{
+    public ShortFileException(){super();}
+    public ShortFileException(String s){super(s);}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/SparseMatrix.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/SparseMatrix.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/SparseMatrix.java	(revision 14150)
@@ -0,0 +1,81 @@
+package vishnu.datablock;
+import java.io.*;
+
+public class SparseMatrix implements Serializable
+{
+    private SparseNode [] nodes;
+    private int n_cols;
+    private int n_rows;
+    
+    public SparseMatrix(int rows, int cols)
+    {
+        n_rows = rows;
+	n_cols = cols;
+	
+        nodes = new SparseNode[n_rows];
+	
+        for (int c = 0; c < n_rows; c++)
+	    nodes[c] = null;
+    }
+    
+    
+    public void addDataItem(int row, int column, float value)
+    {
+	SparseNode node; 
+	node = new SparseNode(value, column);
+	nodes[row] = node.push(nodes[row]);
+    }
+    
+    
+    public SparseNode getRow(int row)
+    {
+	return nodes[row];
+    }
+
+
+    public int getRowCount()
+    {
+	    return n_rows;
+    }
+
+
+    public int getColumnCount()
+    {
+	    return n_cols;
+    }
+    
+
+    public float[][] getExpanded()
+    {
+        float[][] full = new float[n_rows][n_cols];
+	
+	SparseNode n;
+	
+	for(int r = 0; r < n_rows; r++){
+	    for (int c = 0; c < n_cols; c++)
+		full[r][c]=0;
+	    
+	    n=nodes[r];
+	    
+	    while(n!= null){
+		full[r][n.getColumn()]=n.getValue();
+		n=n.getNext();
+	    }
+	}
+	return full;
+    }
+    
+    
+    public void print()
+    {
+	for( int i = 0; i < nodes.length; i++ ){
+	    SparseNode n = getRow(i);
+	    while(n != null ){
+		System.out.print(n.getColumn()+ " ");
+		n = n.getNext();
+	    }
+	    System.out.println();
+	}
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/SparseNode.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/SparseNode.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/datablock/SparseNode.java	(revision 14150)
@@ -0,0 +1,59 @@
+package vishnu.datablock;
+import java.io.*;
+
+public class SparseNode implements Serializable
+{
+    private float value;
+    
+    private int column;
+    private SparseNode next;
+    
+    
+    SparseNode(float f, int c)
+    {
+	value = f;
+	column = c;
+	next = null;
+    }
+    
+    
+    SparseNode(float f, int c, SparseNode n)
+    {
+	value = f;
+	column = c;
+	next = n;
+    }
+    
+
+    public SparseNode push(SparseNode n)
+    {
+        if (next != null) next.push(n);
+        else next = n;
+        return this;
+    }
+
+
+    public SparseNode pop()
+    {
+	return next;
+    }
+
+    
+    public SparseNode getNext()
+    {
+        return next;
+    }
+
+    
+    public int getColumn()
+    {
+        return column;
+    }
+    
+
+    public float getValue()
+    {
+        return value;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/CKServer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/CKServer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/CKServer.java	(revision 14150)
@@ -0,0 +1,293 @@
+package vishnu.server;
+
+import java.io.*;
+import java.util.Vector;
+import java.util.Hashtable;
+import java.util.Enumeration;
+import vishnu.util.Misc;
+import vishnu.util.Sammon;
+import vishnu.cluster.*;
+import vishnu.datablock.*;
+
+public class CKServer 
+{
+    private String index_directory = null;
+
+    private final static String FILE_DOC_FREQ = "doc_freq";
+    private final static String FILE_TERM_FREQ = "term_freq";
+    private final static String FILE_CK_WORDS = "ck_words";
+    
+    private float[] docFreq = null;
+    private String[] ck_words = null;
+    private Vector termFreq = null;
+
+
+    private int[] x_keywords;
+    private int[] x_hitdocs;
+    private float[][] KDMatrix;
+    private Vector[] clusters;
+    private double[][] centroids;
+    private Point2D[] sammon;
+    private Vector descriptions;
+    
+    public CKServer(String collection_home, String view) {
+	
+	this.index_directory = collection_home;
+	if (!this.index_directory.endsWith(File.separator)) {
+	    this.index_directory += File.separator;
+	}
+	this.index_directory += "index" + File.separator + "ck_index" + File.separator;
+	if (view != null) {
+	    this.index_directory += view + File.separator;
+	}
+	    
+	loadIndices();
+    }
+    
+    private void loadIndices(){
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	
+	try {
+	    docFreq = (float[])getObject(this.index_directory  + FILE_DOC_FREQ);
+	    ck_words = (String[])getObject(this.index_directory + FILE_CK_WORDS);
+	    termFreq = (Vector)getObject(this.index_directory + FILE_TERM_FREQ);
+	} catch (Exception e) {
+	    System.err.println("CKServer Error: Couldn't load up ck index data: "+e);
+	}
+    }
+
+    
+    private Object getObject(String str)
+	throws IOException, FileNotFoundException, ClassNotFoundException {
+	Object o = null;
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	fis = new FileInputStream(str);
+	in = new ObjectInputStream(fis);
+	o = in.readObject();
+	in.close();
+	return o;
+    }
+
+
+    public void setDescriptions(Vector desc)
+    {
+	descriptions = desc;
+    }
+    
+
+    public String[] computeKeywords(Vector docNum)
+    {
+	int hits = docNum.size();
+	
+	Hashtable keyHash = new Hashtable();
+	
+	int count_ck_all = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    count_ck_all += ht.size();
+	    
+	    Enumeration enumer = ht.keys();
+	    while( enumer.hasMoreElements() ){
+		Integer term_index = (Integer)enumer.nextElement();
+		
+		Integer set_freq;
+		if( (set_freq = (Integer)keyHash.get(term_index)) == null ){
+		    keyHash.put(term_index,new Integer(1));
+		}else{
+		    keyHash.put(term_index,new Integer(set_freq.intValue()+1));
+		}
+	    }
+	}
+	
+
+	/**** compute interesting weight for each keyword ****/
+	
+	Enumeration enumer = keyHash.keys();
+	
+	float[] weight_ck = new float[keyHash.size()];
+	Integer[] weight_index = new Integer[keyHash.size()];
+	int count = 0;
+	
+	while( enumer.hasMoreElements() ){
+	    Integer index_ck = (Integer)enumer.nextElement();
+	    Integer set_freq = (Integer)keyHash.get(index_ck);
+	    
+	    float set_freq_f = (float)set_freq.intValue()/hits;
+	    
+	    float f1 = (float)set_freq_f/docFreq[index_ck.intValue()];
+	    float f2 = -(float)Math.log(set_freq_f);
+	    
+	    weight_index[count] = index_ck;
+	    weight_ck[count] = - f1 * f2;
+	    
+	    count++;
+	}
+	
+	Misc.quickSort(weight_ck,weight_index);
+	
+	
+	/**** store indices of up to 100 top ranked keywords ****/
+	
+	int wordCount = Math.min(100,count);
+	int[] indices_kw = new int[wordCount];
+	
+	for( int i = 0; i < wordCount; i++ ){
+	    indices_kw[i] = weight_index[i].intValue();
+	    
+	    float weight = -weight_ck[i];
+	    String ck = ck_words[indices_kw[i]];
+	}
+	
+	
+	/**** sort the indices_kw array in increasing order of indices ****/
+	
+	Misc.quickSort(indices_kw);
+
+	x_keywords = indices_kw;
+	
+	String[] keywords = new String[x_keywords.length];
+	for( int i = 0; i < x_keywords.length; i++ ){
+	    keywords[i] = ck_words[x_keywords[i]];
+	}
+
+	return keywords;
+    }
+    
+    
+    public SparseMatrix getSparseMatrix(Vector docNums){
+		
+	int hits = docNums.size();
+	int wordCount = x_keywords.length;
+	int scale = 100; // length of document vector
+	
+
+	/**** build full matrix: rows indices_kw (wordCount), col docNum (hits) ****/
+	
+	float[][] matrix_kd = new float[hits][wordCount];
+	float[] sum_rows = new float[hits];
+	int emptyRows = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNums.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		Integer term_freq;
+		
+		if( (term_freq = (Integer)ht.get(new Integer(x_keywords[j]))) != null ){
+		    float w = -(float)(Math.log(1+term_freq.intValue()) * Math.log(docFreq[x_keywords[j]]));
+		    w = term_freq.intValue();
+		    matrix_kd[i][j] = w;
+		    sum_rows[i] += w*w;
+		}
+	    }
+	    
+	    
+	    /**** sqrt sum to obtain |z| ****/
+	    
+	    if( sum_rows[i] != 0 ) sum_rows[i] = (float)Math.sqrt(sum_rows[i])/scale; else emptyRows++;
+	}
+	
+	/**** populate sparseMatrix and full matrix dividing each entry by the row_sum ****/
+	
+	SparseMatrix matrix = new SparseMatrix(hits-emptyRows,wordCount);
+	KDMatrix = new float[hits-emptyRows][wordCount];
+		
+	int ci = 0;
+
+	for( int i = hits-1; i >= 0; i-- ){
+	    if( sum_rows[i] == 0 ){
+		docNums.removeElementAt(i);
+		descriptions.removeElementAt(i);
+	    }
+	}
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    if( sum_rows[i] == 0 ) continue;
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		float m = matrix_kd[i][j];
+		
+		if( m != 0 ){
+		    float norm_m = m; //(float)m/sum_rows[i];
+		    matrix.addDataItem(ci, j, norm_m);
+		    KDMatrix[ci][j] = norm_m; //(float)(norm_m);
+		}
+		else KDMatrix[ci][j] = 0;
+	    }
+	    ci++;
+	}
+	
+	x_hitdocs = new int[docNums.size()];
+	
+	for( int i = 0; i < docNums.size(); i++ ){
+	    x_hitdocs[i] = ((Integer)docNums.elementAt(i)).intValue();
+	}
+	
+	return matrix;
+    }
+
+
+    public Vector[] getClusters(){
+
+	int wordCount = KDMatrix[0].length;
+	int docHits = KDMatrix.length;
+	System.err.println("getClusters: word count = "+wordCount+", docHits = "+docHits);
+        int clusterNum =  docHits>20?10:docHits;
+      
+        int limit =  docHits/2;
+	
+        Clustering clustering = new Clustering(Clustering.COMPLETE_LINKAGE,KDMatrix,1,clusterNum,limit);
+	
+        clusters = clustering.getClusters();
+	centroids= clustering.getCentroids();
+	
+	// what are these for??
+	//Sammon map = new Sammon(centroids);
+	//Point2D [] sammon = map.getMapping();
+	return clusters;
+    }
+    
+    
+    public int[] getHitDocuments()
+    {
+	return x_hitdocs;
+    }
+
+
+    public double[][] getCentroids()
+    {
+	return centroids;
+    }
+
+
+    public String[] getHitDescriptions()
+    {
+	String[] desc = new String[descriptions.size()];
+	for( int i = 0; i < desc.length; i++ ){
+	    desc[i] = (String)descriptions.elementAt(i);
+	}
+
+	return desc;
+    }
+    
+    public Point2D[] getSammonMap(double[][] centroids)
+    {
+	if (centroids == null) {
+	    return null;
+	}
+	Sammon sammon = new Sammon(centroids);
+	Point2D[] p = sammon.getMapping();
+	
+	return p;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/CKTester.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/CKTester.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/CKTester.java	(revision 14150)
@@ -0,0 +1,230 @@
+package vishnu.server;
+
+import java.io.*;
+import java.util.*;
+import vishnu.server.Search.*;
+import vishnu.util.*;
+import vishnu.datablock.*;
+
+public class CKTester
+{
+
+    public static void main(String[] args){
+	CKTester ckt = new CKTester(args);
+    }
+
+    
+    public CKTester(String[] args){
+	
+	Vector docNum = new Vector();
+	Vector descriptions = new Vector();
+	
+	String query = args[0];
+	String coll = args[1];
+	String vis_home = args[2];
+	String view = null;
+	if (args.length >= 4) {
+	    view = args[3];
+	}
+
+	/**** read in ck words, doc freq and term freq ****/
+
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	
+	String root = vis_home;
+	if (!root.endsWith(File.separator)) {
+	    root += File.separator;
+	}
+	root += "collections" + File.separator+ coll + File.separator + "index" + File.separator + "ck_index" + File.separator;
+	if (view != null) {
+	    root += view+File.separator;
+	}
+	String fileDocFreq = "doc_freq";
+	String fileTermFreq = "term_freq";
+	String fileCKWords = "ck_words";
+	
+	float[] docFreq = (float[])getObject(root+fileDocFreq);
+	String[] ck_words = (String[])getObject(root + fileCKWords);
+	Vector termFreq = (Vector)getObject(root + fileTermFreq);
+
+
+	/**** get relevant docs ****/
+
+	LUCSearcher lus = new LUCSearcher(coll, view); // coll should be fullpath
+	//lus.setRootDirectory(vis_home);
+	lus.search(query);
+	docNum = lus.getDocIdentifiers();
+	descriptions = lus.getDocDescriptions();
+	
+	int hits = docNum.size();
+	
+
+	/**** pool the candidate keywords and note their frequencies ****/
+	
+	Hashtable keyHash = new Hashtable();
+	
+	int count_ck_all = 0;
+
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    count_ck_all += ht.size();
+	    
+	    Enumeration enumer = ht.keys();
+	    while( enumer.hasMoreElements() ){
+		Integer term_index = (Integer)enumer.nextElement();
+		
+		Integer set_freq;
+		if( (set_freq = (Integer)keyHash.get(term_index)) == null ){
+		    keyHash.put(term_index,new Integer(1));
+		}else{
+		    keyHash.put(term_index,new Integer(set_freq.intValue()+1));
+		}
+	    }
+	}
+
+	System.out.println("all keywords: " + count_ck_all);
+	System.out.println("pooled: " + keyHash.size());
+
+
+	/**** compute interesting weight for each keyword ****/
+
+	Enumeration enumer = keyHash.keys();
+
+	float[] weight_ck = new float[keyHash.size()];
+	Integer[] weight_index = new Integer[keyHash.size()];
+	int count = 0;
+
+	while( enumer.hasMoreElements() ){
+	    Integer index_ck = (Integer)enumer.nextElement();
+	    Integer set_freq = (Integer)keyHash.get(index_ck);
+	    
+	    float set_freq_f = (float)set_freq.intValue()/hits;
+
+	    float f1 = (float)set_freq_f/docFreq[index_ck.intValue()];
+	    float f2 = -(float)Math.log(set_freq_f);
+
+	    weight_index[count] = index_ck;
+	    weight_ck[count] = - f1 * f2;
+
+	    //System.out.println(set_freq_f+" "+docFreq[index_ck.intValue()]+" "+weight_ck[count]);
+	    count++;
+	}
+	
+	Misc.quickSort(weight_ck,weight_index);
+
+
+	/**** store indices of up to 100 top ranked keywords ****/
+
+	int wordCount = Math.min(100,count);
+	int[] indices_kw = new int[wordCount];
+	
+	for( int i = 0; i < wordCount; i++ ){
+	    indices_kw[i] = weight_index[i].intValue();
+	    
+	    float weight = -weight_ck[i];
+	    String ck = ck_words[indices_kw[i]];
+	    System.out.println(indices_kw[i] + " " + ck + " " + weight + " ");
+	}
+
+
+	/**** sort the indices_kw array in increasing order of indices ****/
+
+	Misc.quickSort(indices_kw);
+
+
+	/**** build full matrix: rows indices_kw (wordCount), col docNum (hits) ****/
+
+	float[][] matrix_kd = new float[hits][wordCount];
+	float[] sum_rows = new float[hits];
+	int emptyRows = 0;
+
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		Integer term_freq;
+		
+		if( (term_freq = (Integer)ht.get(new Integer(indices_kw[j]))) != null ){
+		    float w = -(float)(Math.log(1+term_freq.intValue()) * Math.log(docFreq[indices_kw[j]]));
+		    matrix_kd[i][j] = w;
+		    sum_rows[i] += w*w;
+		}
+		System.out.print(matrix_kd[i][j] + " ");
+	    }
+	    
+	    System.out.println();
+	    
+
+	    /**** sqrt sum to obtain |z| ****/
+
+	    if( sum_rows[i] != 0 ) sum_rows[i] = (float)Math.sqrt(sum_rows[i]); else emptyRows++;
+	}
+	
+	/**** populate sparseMatrix dividing each entry by the row_sum ****/
+	
+	SparseMatrix matrix = new SparseMatrix(hits-emptyRows,wordCount);
+
+	int ci = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    if( sum_rows[i] == 0 )
+		continue;
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		float m = matrix_kd[i][j];
+		
+		if( m != 0 ){
+		    float norm_m = (float)m/sum_rows[i];
+		    matrix.addDataItem(ci, j, norm_m);
+		    System.out.print(j + " (" + norm_m + ") ");
+		}
+	    }
+
+	    ci++;
+	    System.out.println();
+	}
+    }
+    
+
+
+    private Object getObject(String str){
+	Object o = null;
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	try{
+	    fis = new FileInputStream(str);
+	    in = new ObjectInputStream(fis);
+	    o = in.readObject();
+	    in.close();
+	}
+	catch(IOException ex){
+	    ex.printStackTrace();
+	}
+	catch( ClassNotFoundException ex){
+	    ex.printStackTrace();
+	}
+	return o;
+    }
+}
+
+
+class Node{
+    Node next = null;
+    float value = -1;
+    int docIndex = -1;
+    int kwIndex = -1;
+
+    public Node(int d, int k)
+    {
+	docIndex = d;
+	kwIndex = k;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Packaging.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Packaging.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Packaging.java	(revision 14150)
@@ -0,0 +1,376 @@
+package vishnu.server;
+
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+//import org.apache.xpath.XPathAPI;
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.*;
+import java.awt.*;
+
+public class Packaging
+{
+    static Document doc = null;
+    static boolean out = false;
+
+    static String matrixString = "";
+
+    public Packaging()
+    {
+    }
+
+
+    public static int[] inInts(StringTokenizer st)
+    {
+	String line;
+	int intCount;
+
+	if (!st.hasMoreTokens()) {
+	    return new int[0];
+	}
+
+	intCount = Integer.parseInt(st.nextToken());
+	if(out) System.out.println(intCount);
+
+	int[] intarray = new int[intCount];
+	for (int c = 0; c < intCount;c++)
+	    {
+		line = st.nextToken();
+		int n = Integer.parseInt(line);
+		intarray[c] = n;
+		if(out) System.out.println(n);
+	    }
+	return intarray;
+    }		
+    
+    
+    public static String [] inStrings(StringTokenizer st)
+    {
+	String line;
+	String [] instrings;
+	int strCount;
+
+	if (!st.hasMoreTokens()) {
+	    return new String[0];
+	}
+	
+	strCount = Integer.parseInt(st.nextToken());
+	if(out)	System.out.println(strCount);
+	   
+	instrings = new String[strCount];
+	for (int c = 0; c < strCount;c++)
+	    {
+		instrings[c] = st.nextToken();
+	    }
+
+	return instrings;
+    }
+
+    public static SparseMatrix inMatrix(StringTokenizer st, int wordCount)
+    {
+	String line;
+	int rowCount,column,value;
+
+	if (!st.hasMoreTokens()) {
+	    return new SparseMatrix(0, wordCount);
+	}
+
+	rowCount = Integer.parseInt(st.nextToken());
+	matrixString += String.valueOf(rowCount) + "\n";
+	if(out) System.out.println(rowCount);
+	
+	SparseMatrix matrix = new SparseMatrix(rowCount,wordCount);
+	
+	for (int c = 0; c < rowCount;c++)
+	    {
+		line = st.nextToken();
+		matrixString += line + "\n";
+
+		if(out) System.out.println(line);
+		
+		if(line.length() > 1)
+		    {
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				String tokenString = tokens.nextToken();
+				value = Integer.parseInt(tokenString);
+				
+				if(tokens.hasMoreTokens())
+				    { 
+					tokenString = tokens.nextToken();
+					column = Integer.parseInt(tokenString);
+					matrix.addDataItem(c, column, value);
+				    }
+			    }// while				
+		    } //if
+	    } // for 
+	return matrix;
+    } // inMatrix
+
+    
+    public static Vector [] inCluster(StringTokenizer st, int docCount)
+    {
+	String line;
+	int clusterCount,value;
+	
+	if (!st.hasMoreTokens()) {
+	    return new Vector[0];
+	}
+	clusterCount = Integer.parseInt(st.nextToken());
+	if(out)	System.out.println(clusterCount);
+	Vector [] clusters = new Vector[clusterCount];
+	
+	for (int c = 0; c < clusterCount;c++)
+	    {
+		Vector v = new Vector();
+
+		line = st.nextToken();
+		if(out) System.out.println(line);
+		if (line.length()>0)
+		    {
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				String tokenString = tokens.nextToken();
+				
+				value = Integer.parseInt(tokenString);
+				if(value >= 0 && value <= docCount)
+				    {
+					v.addElement(new Integer(value));
+				    } // if
+			    } // while
+		    } // if
+		
+		clusters[c]=v;
+	    } // for
+	return clusters;
+    }
+ 
+
+    public static Point2D[] inPoint2D(StringTokenizer st)
+    {
+	String line;
+	int count ;
+	float x = 0,y = 0;
+	
+	if (!st.hasMoreTokens()) {
+	    return new Point2D[0];
+	}
+	
+	count = Integer.parseInt(st.nextToken());
+	if(out) System.out.println(count);
+	
+	Point2D [] array = new Point2D[count];
+	
+	for (int c = 0; c < count;c++)
+	    {
+		line = st.nextToken();
+		if(out)	System.out.println(line);
+		StringTokenizer tokens = new StringTokenizer(line,",");
+
+		if(tokens.hasMoreTokens())
+		    {
+			String tokenString = tokens.nextToken();
+			x = Float.parseFloat(tokenString);
+		    }
+			
+		if(tokens.hasMoreTokens())
+		    {
+			String tokenString = tokens.nextToken();
+			y = Float.parseFloat(tokenString);
+		    } // if
+		    		
+		array[c]= new Point2D(x,y);
+	    } // for
+	return array;
+    }
+
+ 
+ public Element getXMLPackage(String result, Vector desc) 
+    {
+	String [] words;
+	int [] docs;
+	String [] descriptions;
+	SparseMatrix matrix;
+	Vector [] clusters;
+	Point2D [] sammon;
+
+	StringTokenizer st = new StringTokenizer(result);
+
+	docs = inInts(st);
+	words = inStrings(st);
+	matrix = inMatrix(st,words.length);
+	clusters = inCluster(st, docs.length);
+	sammon = inPoint2D(st);	
+	descriptions = new String[docs.length];
+
+	Element response = doc.createElement("response");
+	response.setAttribute("from","Packaging");
+	
+	Element docList = doc.createElement("docNodeList");
+	response.appendChild(docList);
+	for( int i = 0; i < docs.length; i++ )
+	    {
+		Element node = doc.createElement("docNode");
+		node.setAttribute("docNum", String.valueOf(docs[i]));
+		node.setAttribute("title",(String)desc.elementAt(i));
+		docList.appendChild(node);
+	    }
+
+	Element sammonList = doc.createElement("sammonList");
+	response.appendChild(sammonList);
+	for( int i = 0; i < sammon.length; i++ )
+	    {
+		float x = sammon[i].x;
+		float y = sammon[i].y;		
+		Element node = doc.createElement("sammonNode");
+		node.setAttribute("xpos",String.valueOf(x));
+		node.setAttribute("ypos",String.valueOf(y));
+		sammonList.appendChild(node);
+	    }
+
+	Element wordList = doc.createElement("wordList");
+	response.appendChild(wordList);
+	for( int i = 0; i < words.length; i++ )
+	    {
+		Element node = doc.createElement("wordNode");
+		node.setAttribute("term",words[i]);
+		wordList.appendChild(node);
+	    }
+
+	Element mat = doc.createElement("DocTermSparseMatrix");
+	response.appendChild(mat);
+
+	st = new StringTokenizer(matrixString);
+	String line;
+	int rowCount,column,value;
+
+	rowCount = Integer.parseInt(st.nextToken());
+	mat.setAttribute("rows",String.valueOf(rowCount));
+	mat.setAttribute("cols",String.valueOf(words.length));
+
+	Element rowList = doc.createElement("rowList");
+	mat.appendChild(rowList);
+
+	for (int c = 0; c < rowCount;c++)
+	    {
+		line = st.nextToken();
+		if(line.length() > 1)
+		    {
+			Element colList = doc.createElement("colList");
+			rowList.appendChild(colList);
+			colList.setAttribute("row",String.valueOf(c));
+			
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				Element entryElem = doc.createElement("entry");
+				colList.appendChild(entryElem);
+
+				String tokenString = tokens.nextToken();
+				value = Integer.parseInt(tokenString);
+
+				entryElem.setAttribute("docFreq",tokenString);
+				
+				if(tokens.hasMoreTokens())
+				    { 
+					tokenString = tokens.nextToken();
+					column = Integer.parseInt(tokenString);
+					entryElem.setAttribute("term",tokenString);
+				    }
+			    }// while				
+		    } //if
+	    } // for 
+
+	Element clusterList = doc.createElement("ClusterList");
+
+	response.appendChild(clusterList);
+
+	for( int i = 0; i < clusters.length; i++ )
+	    {
+		Element cluster = doc.createElement("Cluster");
+		clusterList.appendChild(cluster);
+
+		Vector vector = clusters[i];
+		for( int c = 0; c < vector.size(); c++ )
+		    {
+			Element entry = doc.createElement("leaf");
+			cluster.appendChild(entry);
+			
+			Integer I = (Integer)vector.elementAt(c);
+			entry.setAttribute("docNum",String.valueOf(I.intValue()));
+		    }
+	    }
+
+	for( int i = 0; i < docs.length; i++ ){
+            descriptions[i] = (String)desc.elementAt(i);
+	}
+
+	return response;
+    }
+
+
+   
+    public DataBlock getDataPackage(String result, Vector desc) 
+    {
+	String [] words;
+	int [] docs;
+	String [] descriptions;
+	SparseMatrix matrix;
+	Vector [] clusters;
+	Point2D [] sammon;
+
+	StringTokenizer st = new StringTokenizer(result);
+
+	docs = inInts(st);
+	words = inStrings(st);
+	matrix = inMatrix(st,words.length);
+	clusters = inCluster(st, docs.length);
+	sammon = inPoint2D(st);	
+	descriptions = new String[docs.length];
+
+	for( int i = 0; i < docs.length; i++ )
+	    {
+		descriptions[i] = (String)desc.elementAt(i);
+		//descriptions[i]	= "doc"+Integer.toString(i);
+	    }
+
+	DataBlock data = new DataBlock(docs,descriptions,words,matrix,clusters,sammon); 
+	return data;
+    }
+
+}
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/LUCSearcher.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/LUCSearcher.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/LUCSearcher.java	(revision 14150)
@@ -0,0 +1,173 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+
+import java.util.Vector;
+import java.util.Hashtable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+
+public class LUCSearcher implements SearchInterface
+{
+    protected Document doc=null;
+    private String collection_home;
+    private String index_dir;
+    private Vector docIDs;
+    private Vector docDescs;
+    private Hits hits;
+    private Hashtable ht;
+    
+    public LUCSearcher(String coll_home, String view)
+    {
+	collection_home = coll_home;
+	if (!collection_home.endsWith(File.separator)) {
+	    collection_home += File.separator;
+	}
+	index_dir = collection_home + File.separator + "index" + 
+	    File.separator + "luc_index" + File.separator;
+	if (view != null) {
+	    index_dir += view + File.separator;
+	}
+    }
+
+
+    public Vector getDocIdentifiers()
+    {
+	return docIDs;
+    }
+    
+    
+    public Vector getDocDescriptions()
+    {
+	return docDescs;
+    }	
+    
+    
+    public Vector getDocContent(int doc_num)
+    {
+	System.err.println("in doc content, doc num="+doc_num);
+	Integer in = null;
+	if (ht==null) {
+	    System.err.println("ht = null");
+	}
+	try {
+	    in = (Integer)ht.get(new Integer(doc_num));
+	} catch (Exception e) {
+	    System.err.println("exception happened: "+e);
+	}
+	System.err.println("*");
+	int id = 1;
+	System.err.println("*");
+	if( in != null ) id = in.intValue();
+	System.err.println("hit id ="+id);
+	Vector v = new Vector();
+
+	try {
+	    Directory directory = FSDirectory.getDirectory(new File(index_dir), false);
+
+	    IndexReader reader = IndexReader.open(directory);
+	    
+	    Document doc = reader.document(doc_num);
+	    String path = doc.get("path");
+	    // make sure the slashes go the right way
+	    path = path.replaceAll("[\\\\\\/]", File.separator);
+	    String full_path = collection_home + path;
+	    File f = new File(full_path);
+	    BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+	    String line;
+	    
+	    while( (line=dis.readLine()) != null ){
+		v.addElement(line);
+	    }
+	    dis.close();
+	} catch(IOException ex){v.addElement("Content of this document currently unavailable");}    
+	return v;
+    }
+
+    
+    public void search(String q)
+    {
+	docIDs = new Vector();
+	docDescs = new Vector();
+	ht = new Hashtable();
+	
+	try {
+	    Searcher searcher = new IndexSearcher(index_dir);
+	    Analyzer analyzer = new StandardAnalyzer();
+	    
+	    Query query = QueryParser.parse(q, "contents", analyzer);
+	    System.out.println("Searching for: " + query.toString("contents") + " in " + index_dir);
+	    
+	    hits = searcher.search(query);
+	    System.out.println(hits.length() + " total matching documents");
+	    int max_docs = Math.min(MAX_DOCS,hits.length());
+	    for (int i = 0; i < max_docs; i++) {
+
+		int id = hits.id(i);
+		ht.put(new Integer(id),new Integer(i));
+		docIDs.addElement(new Integer(id));
+		
+		Document doc = hits.doc(i);
+		String path = doc.get("path");
+		// make sure the slashes go the right way
+		path = path.replaceAll("[\\\\\\/]", File.separator);
+		String full_path = collection_home + path;
+		File f = new File(full_path);
+		
+		String line = "";
+		
+		try{
+		    BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+		    while( line.length() < MAX_DESCRIPTION_LENGTH ){
+			String tmp = dis.readLine();
+			line += tmp;
+		    }
+		    docDescs.addElement(line.substring(0,MAX_DESCRIPTION_LENGTH));
+		    dis.close();
+		}catch(IOException ex){docDescs.addElement(new String("No description available"));}    
+	    }
+	    searcher.close();
+	    
+	} catch (Exception ex) {ex.printStackTrace();}
+    }
+}
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/MGSearcher.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/MGSearcher.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/MGSearcher.java	(revision 14150)
@@ -0,0 +1,101 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+
+import org.greenstone.mg.*;
+
+public class MGSearcher implements SearchInterface
+{
+ 
+    private String collection_home;
+    private String collection_name;
+    private Vector docIDs;
+    private Vector docDescs;
+    private String mg_index_path;
+    private MGWrapper mg_src;
+    
+    public MGSearcher(String coll_home, String coll_name, String view)
+    {
+	collection_home = coll_home;
+	if (!collection_home.endsWith(File.separator)) {
+	    collection_home += File.separator;
+	}
+	collection_name = coll_name;
+	mg_src = new MGWrapper();
+	mg_index_path = "index"+File.separator+"mg_index"+ File.separator;
+	if (view != null) {
+	    mg_index_path += view + File.separator;
+	}
+	mg_index_path += collection_name+File.separator+collection_name;
+	mg_src.setIndex(mg_index_path);
+    }
+
+
+    public Vector getDocIdentifiers()
+    {
+	return docIDs;
+    }
+
+
+    public Vector getDocDescriptions()
+    {
+	return docDescs;
+    }	
+
+    // why do we want document in a vector ?? -- kjdon
+    public Vector getDocContent(int doc_num) {
+
+	Vector document = new Vector();
+	String doc_content = this.mg_src.getDocument(collection_home, mg_index_path, doc_num);
+	String [] lines = doc_content.split("\n");
+	for (int i=0; i<lines.length; i++) {
+	    document.addElement(lines[i]);
+	}
+	return document;
+
+    }
+
+    public void search(String query) {
+	this.mg_src.runQuery(collection_home, mg_index_path, query);
+	MGQueryResult mqr = this.mg_src.getQueryResult();
+	docIDs = new Vector();
+	docDescs = new Vector();
+	Vector doc_results =   mqr.getDocs();
+	int max_docs = Math.min(MAX_DOCS,doc_results.size());
+	for (int i=0; i<max_docs; i++) {
+	    int doc_num = (int)((MGDocInfo)doc_results.get(i)).num_;
+	    docIDs.addElement(new Integer(doc_num));
+	    String doc_content = this.mg_src.getDocument(collection_home, mg_index_path, doc_num);
+	    docDescs.addElement(doc_content.substring(0,MAX_DESCRIPTION_LENGTH));
+	}	
+    }
+    
+}
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/SearchInterface.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/SearchInterface.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/Search/SearchInterface.java	(revision 14150)
@@ -0,0 +1,41 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import vishnu.datablock.*;
+import java.util.*;
+import org.w3c.dom.Element;
+
+public interface SearchInterface
+{
+
+    public static final int MAX_DOCS = 100;
+    public static final int MAX_DESCRIPTION_LENGTH = 200;
+
+    public void search(String query);
+
+    public Vector getDocIdentifiers();
+
+    public Vector getDocDescriptions();
+
+    public Vector getDocContent(int doc_num);
+
+    //public void setRootDirectory(String root);
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/VisServlet.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/VisServlet.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/server/VisServlet.java	(revision 14150)
@@ -0,0 +1,328 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server;
+
+import vishnu.server.Search.*;
+import vishnu.datablock.*;
+import vishnu.util.XMLUtil;
+
+import java.io.*;
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.util.Vector;
+import java.util.Hashtable;
+
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+public class VisServlet extends HttpServlet
+{
+    final String DOCUMENT_FETCH_ACTION = "fetch";
+    final String DOCUMENT_SEARCH_ACTION = "search";
+    final String KEYWORDS_COMPUTE_ACTION = "keywords";
+    final String INITIALIZATION_ACTION = "initialise";
+    
+    protected String COLLS_HOME=null; 
+    
+    Hashtable collH;
+
+    public void init(ServletConfig config) throws ServletException {
+	// always call super.init;
+	super.init(config);
+
+	COLLS_HOME = config.getInitParameter("collectionsHome");
+	if (COLLS_HOME == null || COLLS_HOME.equals("")) {
+	    System.err.println("initialization parameter, collectionsHome, not set");
+	    System.exit(1);
+	}
+	if (!COLLS_HOME.endsWith(File.separator)) {
+	    COLLS_HOME += File.separator;
+	}
+    }
+
+    private Object receiveObject(HttpServletRequest req) throws Exception
+    {
+        ObjectInputStream in = new ObjectInputStream( req.getInputStream() );
+        Object obj = in.readObject();
+        in.close();
+        return obj;
+    }
+    
+    
+    private void sendObject(HttpServletResponse resp, Object obj) throws Exception
+    {
+        ObjectOutputStream out = new ObjectOutputStream( resp.getOutputStream() );
+        out.writeObject(obj);
+        out.close();
+    }
+    
+    public void doGet(HttpServletRequest req, HttpServletResponse resp) 
+	throws IOException {
+
+	resp.setContentType("text/html;charset=UTF-8");
+        PrintWriter out = resp.getWriter();
+	String output = "<html><head><title>Visualiser servlet</title></head>\n<body>The servlet is running, and accepting messages via POST</body></html>";
+	out.println(output);
+
+    }
+    public void doPost(HttpServletRequest req, HttpServletResponse resp)
+    {
+	Object obj;
+	
+        try{
+	    obj = receiveObject(req);
+	    Vector q = (Vector)obj;
+	    Vector response = new Vector();
+	    
+	    String action = (String)q.elementAt(0);
+	    System.out.println("Action.. " + action);
+	    
+ 	    if( action.equals(INITIALIZATION_ACTION) ){
+		Vector colls = activateCollections();
+		sendObject(resp,colls);
+	    }
+	    else if( action.equals(DOCUMENT_SEARCH_ACTION) ){
+		Vector docNums = new Vector();
+		Vector descriptions = new Vector();
+		String engine;
+		
+		String query = (String)q.elementAt(1);
+		String collection = (String)q.elementAt(2);
+		String view = null;
+		if (q.size() == 4) {
+		    view = (String)q.elementAt(3);
+		}
+		if (view != null && view.equals("")) {
+		    view = null;
+		}
+		System.out.println("Requesting..." + collection);
+		
+		try{
+		    engine = (String)collH.get(collection);
+		}catch(Exception ex){engine = "MG";}
+		
+		
+		System.out.println("Retrieving with " + engine);
+		
+		String collection_home = COLLS_HOME+collection+File.separator;
+		SearchInterface search_engine = null;
+		if( engine.equals("MG") ){
+		    search_engine = new MGSearcher(collection_home, collection, view);
+		} else if( engine.equals("LUCENE") ){
+		    search_engine = new LUCSearcher(collection_home, view);
+		} 
+		
+		search_engine.search(query);
+		docNums = search_engine.getDocIdentifiers();
+		descriptions = search_engine.getDocDescriptions();
+		System.out.println(docNums.size() + " documents in VisServlet");
+		DataBlock data = generateDataBlock(collection_home, view, docNums, descriptions);
+		if (data != null) {
+		    response.addElement(data);
+		}
+		sendObject(resp,response);
+		search_engine = null;
+		data = null;
+		docNums = null;
+		descriptions = null;
+		
+	    }
+	    else if( action.equals(DOCUMENT_FETCH_ACTION) ){
+		String docNum = (String)q.elementAt(1);
+		String collection = (String)q.elementAt(2);
+		String view = null;
+		if (q.size() == 4) {
+		    view = (String)q.elementAt(3);
+		}
+		if (view != null && view.equals("")) {
+		    view = null;
+		}
+		String engine;
+		String collection_home = COLLS_HOME+collection+File.separator;
+		
+		try{
+		    engine = (String)collH.get(collection);
+		}catch(Exception ex){engine = "MG";}
+		
+                System.out.println("Retrieving with " + engine);
+		
+		SearchInterface search_engine = null;
+		if( engine.equals("MG") ){
+		    search_engine = new MGSearcher(collection_home, collection, view);
+		} else if( engine.equals("LUCENE") ){
+		    search_engine = new LUCSearcher(collection_home, view);
+		} 
+		System.err.println("Searching for doc "+docNum);
+		Vector doc = search_engine.getDocContent(Integer.parseInt(docNum));
+		response.addElement(doc);			
+		sendObject(resp,response);
+		search_engine = null;
+	    }
+	} catch(Exception e){}
+    }
+    
+
+    public Vector connect(Vector q)
+    {
+	return null;
+    }
+
+
+    private Vector activateCollections()
+    {
+	System.err.println("activating collections");
+	Vector colls = new Vector();
+	collH = new Hashtable();
+	Document doc = null;
+	
+	File dir = new File(COLLS_HOME);
+	if( !dir.exists() ){
+	    System.err.println("collections directory, "+COLLS_HOME+", doesn't exist, can't activate collections");
+	    return null;
+	}
+	
+	File[] list = dir.listFiles();
+	
+	for( int i = 0; i < list.length; i++ ){
+	    File f = list[i];
+	    if( f.isDirectory()){
+		File config_file = new File(f, "config.xml");
+		if (!config_file.exists()) {
+		    System.err.println("Collection "+f.getName()+" has no config file, ignoring");
+		    // ignore this collection
+		    continue;
+		} else {
+		    System.err.println("Trying to init collection "+f.getName());
+		}
+		try {
+		    DocumentBuilderFactory doc_build_fac = DocumentBuilderFactory.newInstance();
+		    DocumentBuilder doc_build = doc_build_fac.newDocumentBuilder();
+		    doc = doc_build.parse(config_file);
+		} catch (Exception e) {
+		    System.err.println("Error parsing config file "+config_file.getPath()+", ignoring this collection");
+		    continue;
+		}
+		Element root = doc.getDocumentElement();
+		String name = root.getAttribute("name");
+		String desc = root.getAttribute("description");
+		String engine = root.getAttribute("engine");
+		int size = Integer.parseInt(root.getAttribute("size"));
+		String coll_id = f.getName();
+		Element view_elem = (Element)XMLUtil.getChildByTagName(root, "viewList");
+		String [] views = null;
+		if (view_elem != null) {
+		    NodeList view_list = view_elem.getElementsByTagName("view");
+		    int num_views = view_list.getLength();
+		    if (num_views > 0) {
+			views = new String[num_views];
+			for (int j=0; j<num_views; j++) {
+			    String v = ((Element)view_list.item(j)).getAttribute("name");
+			    views[j]=v;
+			}
+		    }
+		}
+		Collection c = new Collection(coll_id,name,desc,size,engine,views);
+		System.out.println("Hashing.." + coll_id);
+		colls.addElement(c);
+		collH.put(coll_id,engine);
+	    }			
+	}
+	doc = null;
+	return colls;
+    }
+    
+    private DataBlock generateDataBlock(String collection_home, String view, Vector docNums, Vector descriptions) {
+
+	if (docNums.size()==0) {
+	    return null;
+	}
+	DataBlock data = new DataBlock();
+
+	CKServer ck_server = new CKServer(collection_home, view);
+	
+	/**** set data fields one by one and pass one what ever gets assembled ****/
+		
+	try{
+	    
+	    ck_server.setDescriptions(descriptions);
+	    
+	    /**** get candidate keywords ****/
+	    
+	    String[] keywords = ck_server.computeKeywords(docNums);
+	    
+	    data.words = keywords;
+	    
+	    
+	    /**** get sparse document * keyword matrix ****/
+	    
+	    SparseMatrix matrix = ck_server.getSparseMatrix(docNums);
+	    
+	    data.matrix = matrix;
+	    
+	    
+	    /**** get document indices, this is a subset of the original ****/
+	    /**** those without keywords are excluded                    ****/
+	    
+	    int[] docs = ck_server.getHitDocuments();
+	    
+	    data.docs = docs;
+	    
+	    
+	    String[] desc = ck_server.getHitDescriptions();
+	    
+	    data.descriptions = desc;
+	    
+	    
+	    /**** get 10 or so clusters ****/
+	    
+	    Vector[] clusters = ck_server.getClusters();
+	    
+	    data.clusters = clusters;
+	    
+	    
+	    double[][] centroids = ck_server.getCentroids();
+	    
+	    
+	    /**** send their centroids through sammon mapping ****/
+	    
+	    Point2D[] sammon = ck_server.getSammonMap(centroids);
+	    
+	    System.out.println("Num descriptions: " + descriptions.size());
+	    System.out.println("Num docs: " + docs.length);
+	    System.out.println("Num desc: " + desc.length);
+	    
+	    data.sammon = sammon;
+	    
+	} catch (Exception e) {
+	    System.err.println("VisServlet: computing clustering Error: "+e);
+	    e.printStackTrace();
+	}
+	ck_server = null;
+	return data;
+    }
+}
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/browser/Browser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/browser/Browser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/browser/Browser.java	(revision 14150)
@@ -0,0 +1,125 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.browser;
+
+import javax.swing.JFrame;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.applet.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.net.URL;
+import java.io.IOException;
+import java.util.*;
+
+public class Browser extends JFrame
+{
+  JScrollPane scrollPane;
+  JEditorPane editorPane;
+  JPanel panel;
+  public Browser(URL url)
+  {
+    jb_init(url);
+  }
+  
+  public Browser(Vector document, long docNumber)
+  {
+    jb_init(document);
+  }
+
+  private void jb_init(Vector document)
+  {
+     WindowListener l = new WindowAdapter()
+    {
+      public void windowClosing(WindowEvent e)
+      {
+        setVisible(false);          // hide the Frame
+        dispose();
+      }
+    };
+    
+    panel = new JPanel();
+    setSize(500,400);
+    Container content = this.getContentPane();
+    panel.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+    panel.setLayout(new BorderLayout(10,10));
+    content.setLayout(new BorderLayout(10,10));
+    editorPane = new JEditorPane();
+    editorPane.setEditable(false);               
+    editorPane.setContentType("text/html");   
+    
+    String text = new String("");
+    
+    for( int i = 0; i < document.size(); i++ )
+    {
+        text += (String)document.elementAt(i) + "\n";
+    }
+    
+    editorPane.setText(text);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");
+    
+    /*editorPane = createEditorPane(url);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");*/
+    
+    setVisible(true);
+  }
+
+  private void jb_init(URL url)
+  {
+    WindowListener l = new WindowAdapter()
+    {
+      public void windowClosing(WindowEvent e)
+      {
+        setVisible(false);          // hide the Frame
+        dispose();
+      }
+    };
+    panel = new JPanel();
+    setSize(500,400);
+    Container content = this.getContentPane();
+    panel.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+    panel.setLayout(new BorderLayout(10,10));
+    content.setLayout(new BorderLayout(10,10));
+    editorPane = createEditorPane(url);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");
+    setVisible(true);
+  }
+
+  private JEditorPane createEditorPane(URL url)
+  {
+    JEditorPane edPane = new JEditorPane();
+    edPane.setEditable(false);
+    displayURL(url,edPane);
+    return edPane;
+  }
+
+  private void displayURL(URL url, JEditorPane edPane)
+  {
+    try
+    {
+      edPane.setPage(url);
+    }
+    catch(IOException e)
+    {
+      System.out.println("Bad URL: "+ url.toString());
+    }
+  }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/AveLink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/AveLink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/AveLink.java	(revision 14150)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class AveLink extends LinkFunc
+{
+    double link(double a, double b)
+	{
+  		return (a+b)/2;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Cluster.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Cluster.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Cluster.java	(revision 14150)
@@ -0,0 +1,120 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+public class Cluster
+{
+    public int _id;
+    public Cluster  _parent = null;
+    public Cluster  _child_1 = null;
+    public Cluster  _child_2 = null;
+    public Vector docVect;
+    public Vector docObjVect;
+    public int[] wordFreq;  	
+    
+    public double _children_sim;
+    
+    // probably the size of the cluster
+    public int _items;
+    
+    public int _split;
+    
+    
+    
+    // [] -> centroid for each vector element ?
+    public double[] _centroid=null;
+    
+    public Vector _iter_items = null;
+    public int _iter_items_num = 0;
+    
+    public int _x = 0;
+    public int _y = 0;
+    public double[] getCentroid(){return _centroid;}
+    
+    public Cluster(int id, Cluster parent, Cluster child_1, Cluster child_2, double children_sim, int items)
+    {
+	_id                    = id;
+	_children_sim          = children_sim;
+	_items                 = items;
+	_split                 = 0;
+	_parent                = parent;
+        _iter_items            = new Vector(0);
+        
+        // probably to get a balanced tree
+	if (child_1 != null && child_2 != null)
+	    {
+		if (child_2._items > child_1._items)
+		    {
+		        _child_1 = child_1;
+		        _child_2 = child_2;
+		    }
+    		else
+		    {
+		        _child_1 = child_2;
+		        _child_2 = child_1;
+		    }
+  	    }
+	else // if either or both are null
+	    {
+    		_child_1 = child_1;
+    		_child_2 = child_2;
+	    }
+    }
+    
+    public void setDocVect(Vector docs)
+    {
+	docVect = docs;
+    }
+    
+    public void setDocObjVect(Vector docObj)
+    {
+	docObjVect = docObj;
+    }
+    
+    public Vector getDocObjVect()
+    {
+	return docObjVect;
+    }
+    
+    public Vector getDocVect()
+    {
+	return docVect;
+    }
+    
+    public void setWordFreq(int[] freqs)
+    {
+	wordFreq = freqs;
+    }
+    
+    public int[] getWordFreq()
+    {
+	return wordFreq;
+    }
+    
+    // a nice recursive function to determine the depth of the hierarchy
+    public int depth()
+    {
+	if (_id != -1)
+	    return 0;
+	else
+    		return Math.max(1 + _child_1.depth(), 1 + _child_2.depth());
+	}
+
+    void addChildrenToOutputVector(int []coding, Vector output)
+	{
+  	  if (_id != -1)
+          {
+    	    Integer ii = new Integer(coding[_id]);
+            output.addElement(ii);
+
+          }
+  	  else
+    	  {
+      	    _child_1.addChildrenToOutputVector(coding,output);
+      	    _child_2.addChildrenToOutputVector(coding,output);
+    	  }
+    }
+
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/ClusterArray.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/ClusterArray.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/ClusterArray.java	(revision 14150)
@@ -0,0 +1,56 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+import vishnu.testvis.object.*;
+import vishnu.datablock.*;
+
+
+public class ClusterArray
+{
+    Vector contents;
+    
+    ClusterArray(DataManager dm, Vector indices)
+    {
+	int i;
+	
+	contents = new Vector();
+
+	// num_of_clusters = number of docs
+	for (i = 0; i < indices.size(); i++)
+	    {
+		int id = ((Integer)indices.elementAt(i)).intValue();
+		Cluster c = new Cluster(i, null, null, null, 0, 1);
+		Vector docIndexVector = new Vector();
+		docIndexVector.addElement(new Integer(id));
+		c.setDocVect(docIndexVector);
+		
+		Vector docObjVector = new Vector();
+		DocObj docObj = dm.getDocObjAt(id);
+		docObjVector.addElement(docObj);
+		c.setDocObjVect(docObjVector);    
+		
+		DataBlock data = dm.getDataBlock();
+		int[] wordFreqs = new int[data.words.length];
+		for(int d=0; d < data.words.length; d++) wordFreqs[d]=0;
+		
+		SparseNode row = data.matrix.getRow(id);
+		
+		while( row!= null)
+		    {
+			wordFreqs[row.getColumn()]++;
+			row = row.getNext();
+		    }
+		c.setWordFreq(wordFreqs);
+		
+    		contents.addElement(c);
+		
+		// i.e. id = i, similarity between kids = 0, #items = 1
+	    }
+    }
+    
+    
+}
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/ClusterObj.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/ClusterObj.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/ClusterObj.java	(revision 14150)
@@ -0,0 +1,171 @@
+package vishnu.testvis.dendro;
+
+import java.awt.*;
+import vishnu.testvis.object.*;
+import java.util.Vector;
+
+public class ClusterObj
+
+{
+    Cluster cluster;
+    Point position;
+    int radius;
+    boolean inSubCluster;
+    private boolean orderStatus;
+    private boolean keywordStatus;
+    private String[] freqWords;
+    
+    DocObj[] _docs;		// list of DocObj ref
+    String _label;		// (top 3-5? related words)
+    DataManager _data;
+    
+    public ClusterObj(Cluster c, Point P, int r, DataManager data)
+    {
+        cluster = c;
+        position = P;
+        radius = r;
+        inSubCluster = false;
+        _data = data;
+        orderStatus = false;
+        keywordStatus = false;
+    }
+    
+    
+    public void setPosition(Point p)
+    {
+        position = p;
+    }
+    
+    public void setOrderStatus(boolean b)
+    {
+        orderStatus = b;
+    }
+    
+    public boolean getOrderStatus()
+    {
+        return orderStatus;
+    }
+    
+    public void setFreqWords(String[] str)
+    {
+        freqWords = str;
+        keywordStatus = true;
+    }
+    
+    public String[] getFreqWords()
+    {
+        return freqWords;
+    }
+    
+    public boolean getKeywordStatus()
+    {
+        return keywordStatus;
+    }
+    
+    public void setIsSubset(boolean b)
+    {
+        inSubCluster = b;
+    }
+    
+    public boolean getIsSubset()
+    {
+        return inSubCluster;
+    }
+    
+    public boolean inCircle(int x, int y)
+    {
+        double dist_sqr = (x-position.x)*(x-position.x) + (y-position.y)*(y-position.y);
+        return (dist_sqr <= (4*radius*radius)) ? true : false;
+    }
+    
+    
+    public void setDocObjs(DocObj[] docArray)
+    {
+        _docs = docArray;
+    }
+    
+    
+    public int getX()
+    {
+        return position.x;
+    }
+    
+    public int getY()
+    {
+        return position.y;
+    }
+    
+    public int getRad()
+    {
+        return radius;
+    }
+   
+   
+    public ClusterObj(DocObj[] d, String t, DataManager dm)
+    {
+        _docs=d; _label=t;
+        _data = dm;
+    }
+  
+  
+  public int getDocsSize(){return _docs.length;}
+
+  public String getLabel(){return _label;}
+
+  public String getLabel1(){return getDocsSize() + " documents found";}
+  public String getLabel2(){return getLabel();}
+
+
+  public String getInfo(){
+    return getDocsSize() + " documents found\n" + getLabel();
+  }
+  
+
+
+  /**
+   * This returns a list of list of documents. The inner list of documents represents
+   * which documents have the corresponding related word
+   * pre: _docs can't be null
+   * @return a vector (same length as the related words) of
+   *         vectors (arbitrary length)
+   *
+   */
+  public Vector getHitDocFreq()
+  {
+    int s = _data.getWordCount();
+    
+    Vector rwVec = new Vector(s);
+    
+    for(int i=0;i<s;i++)
+    {
+      rwVec.addElement(new Vector(0));
+      for(int j=0;j<_docs.length;j++)
+      {
+        if ( _docs[j].getWeights(i) > 0 )
+          ((Vector) rwVec.elementAt(i)).addElement(_docs[j]);
+      }
+    }
+    return rwVec;
+  }
+  
+  
+  public int getDocSize()
+  {
+        return cluster._items;
+  }
+
+
+  public String formatDocs()
+  {
+    int l = getDocsSize();
+    String s = "<html><font face=Arial size=-1> "+ l +" documents found\n";
+    for(int i=0;i<l;i++)
+    {
+      s += _docs[i].format4Fields();
+    }
+    s += "</font></html>\n";
+    return s;
+  }
+
+        
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Clustering.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Clustering.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Clustering.java	(revision 14150)
@@ -0,0 +1,248 @@
+package vishnu.testvis.dendro;
+
+/* Related word hierarchical clustering using a given triangle similarity matrix */
+
+//import cluster.*;
+import java.util.*;
+import vishnu.testvis.object.*;
+
+public class Clustering
+{
+    public static final int COMPLETE_LINKAGE = 0;
+    public static final int AVERAGE_LINKAGE  = 1;
+    public static final int SINGLE_LINKAGE   = 2;
+
+    private Vector [] theResult;
+    private double [][] theCentroids;
+    private Cluster theRoot;
+    public DataManager dataManager;
+    private int method = 1;
+
+    public Vector [] getTheResults()
+    {
+        return theResult;
+    }
+    public double[][] getTheCentroids()
+    {
+        return theCentroids;
+    }
+    
+    public Cluster getRoot()
+    {
+        return theRoot;
+    }
+
+
+    public Clustering (DataManager dm, int linkage, float [][]m,
+		       int num_of_vectors,
+		       int arraySize, int min_c_items,
+		       int target_clusters, int limit)
+    {
+	
+        SimMatrix sim_matrix;
+	LinkFunc linkage_func=null;
+	dataManager = dm;
+	
+	// number of keywords
+	int VECTOR_SIZE = arraySize;
+        
+	Cluster toplevel_cluster;
+	TheInt clusters = new TheInt(0);
+	
+	Distance sim_func = new EuclideanDistance();
+	
+	double[] heights;
+	double [][] vectors = floatToDoubleMatrix(m);//intToDoubleMatrix(m);
+	//int [] coding = new int[m.length];
+        
+	// populate array with numbers from 0 to #docs-1
+	//for (int c=0;c<m.length;c++) coding[c]=c;
+	
+	int i, j;
+	
+	switch (linkage)
+	    {
+    	    case COMPLETE_LINKAGE:
+      		linkage_func = new MaxLink();
+      		break;
+    	    case AVERAGE_LINKAGE:
+      		linkage_func = new AveLink();
+      		break;
+    	    case SINGLE_LINKAGE:
+      		linkage_func = new MinLink();
+      		break;
+    	    default:
+      		System.out.println("clustering.c: main(): linkage type "+
+	                  linkage +" not defined.");
+      	        System.exit(0);
+    	}
+
+        long startTime = System.currentTimeMillis();
+        long readTime = System.currentTimeMillis();
+        
+        
+        // if fewer documents than upper cluster limit
+        // I think this is an error, should be vectors.length, viz. #documents
+        // which was never spotted as limit is always smaller
+        // than #keywords
+        // if(limit > vectors[0].length) limit = vectors[0].length;
+
+        // contains distance matrix between documents, built using
+        // the double vector[i][] list of keyword frequencies
+
+
+	Vector indices = null;
+	//randomly choose sqrt(k*n) documents
+
+	if( method == 0 )
+	    {
+
+		int sampleSize = 2*(int)Math.sqrt(vectors.length);
+		Vector sampleInt = new Vector();
+		Vector allDocs = new Vector();
+		
+		for( int s=0; s<vectors.length;s++ )
+		    {
+			allDocs.addElement(new Integer(s));
+		    }
+		
+		while( sampleInt.size() < sampleSize )
+		    {
+			int r = (int)(Math.random()*allDocs.size());
+			sampleInt.addElement(new Integer(r));
+			allDocs.removeElementAt(r);
+		    }
+		double[][] sample_matrix = new double[sampleSize][VECTOR_SIZE];
+		for(int s=0; s<sampleSize;s++)
+		    {
+			int docNum = ((Integer)sampleInt.elementAt(s)).intValue();
+			sample_matrix[s] = vectors[docNum];
+		    }
+		
+		indices = sampleInt;
+		sim_matrix = new SimMatrix(sample_matrix,limit,sim_func,VECTOR_SIZE);
+
+		long simMatixTime = System.currentTimeMillis();
+		
+		// an array of heights sorted by descending disim
+		// as many heights as there are docs
+		heights = new double[m.length];//[limit - 1];
+		
+		//int maxDoc = 0;
+		int maxDoc = indices.size();
+		
+		Hierarchical hierarchical = new Hierarchical(maxDoc);
+		theRoot = hierarchical.clustering(dm, sim_matrix, maxDoc, heights,linkage_func,indices);
+		
+		// collect all clusters
+		Vector content=new Vector();
+		getLeaves(theRoot,content);
+		
+		for( int it = 0; it < allDocs.size(); it++ )
+		    {
+			int docNum = ((Integer)allDocs.elementAt(it)).intValue();
+			
+			double min_dist = 10000;
+			int min_index = -1;
+
+			for( int c = 0; c < content.size(); c++ )
+			    {
+				
+				Cluster clust = (Cluster)content.elementAt(c);
+
+				int docNums = ((Integer)clust.getDocVect().elementAt(0)).intValue();
+
+				double sim = sim_func.get(vectors[docNums],vectors[docNum], VECTOR_SIZE);
+
+				if (sim < min_dist)
+				    {
+					min_dist = sim;
+					min_index = c;
+				    }
+			    }
+			
+			Cluster clust = (Cluster)content.elementAt(min_index);
+			clust._iter_items.addElement(new TheInt(docNum));
+			clust._iter_items_num++;
+			clust.docVect.addElement(new Integer(docNum));
+			long iterativeTime = System.currentTimeMillis();
+			
+		    }
+	    }			 
+	else
+	    {
+   		Vector v = new Vector();
+		for( int l = 0; l < vectors.length; l++ )
+		    {
+			v.addElement(new Integer(l));
+		    }
+
+		indices = v;
+		sim_matrix = new SimMatrix(vectors, limit, sim_func, VECTOR_SIZE);
+	        
+		long simMatixTime = System.currentTimeMillis();
+		
+		// an array of heights sorted by descending disim
+		// as many heights as there are docs
+		heights = new double[m.length];//[limit - 1];
+		
+		//int maxDoc = 0;
+		int maxDoc = indices.size();
+		
+		Hierarchical hierarchical = new Hierarchical(maxDoc);
+		theRoot = hierarchical.clustering(dm, sim_matrix, maxDoc, heights,linkage_func,indices);
+		
+		// collect all clusters
+		Vector content=new Vector();
+		getLeaves(theRoot,content);
+	    }
+	
+    }
+
+    static double[][] intToDoubleMatrix(int [][] m)
+    {
+        int h=m.length;
+        double [][] res = null;
+        if (h > 0)
+        {
+          int w = m[0].length;
+          res = new double[h][w];
+          for(int c=0;c<h;c++)
+            for(int d = 0; d<w; d++)
+              res[c][d]=(double)m[c][d];
+        }
+        return res;
+   }
+    static double[][] floatToDoubleMatrix(float [][] m)
+    {
+        int h=m.length;
+        double [][] res = null;
+        if (h > 0)
+        {
+          int w = m[0].length;
+          res = new double[h][w];
+          for(int c=0;c<h;c++)
+            for(int d = 0; d<w; d++)
+              res[c][d]=(double)m[c][d];
+        }
+        return res;
+   }
+
+
+    private void getLeaves(Cluster node, Vector content)
+    {
+	if( node._child_1 != null )
+	    {
+		getLeaves(node._child_1,content);
+		getLeaves(node._child_2,content);
+	    }
+	else
+	    {
+		content.addElement(node);
+		Vector v = node.docVect;
+		int i = ((Integer)v.elementAt(0)).intValue();
+	    }
+    }
+    
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java	(revision 14150)
@@ -0,0 +1,102 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+import vishnu.testvis.treemap.table.*;
+
+class DendroKeywordPanel extends JPanel
+{
+    RW_DataModel _dataModel;
+    JTable _table = new JTable();
+    JScrollPane theScrollPane;
+    JCheckBox checkBoxes[];
+    JLabel labels[];
+    Vishnu currentMain;
+    DendroPanel parentPane;
+
+    public DendroKeywordPanel(Vishnu currentMain, DendroPanel parent)
+    {
+        this.currentMain = currentMain;
+        this.parentPane = parent;
+        
+        theScrollPane = new JScrollPane();
+
+        setPreferredSize(new Dimension(200,900));
+        setBorder(BorderFactory.createLoweredBevelBorder());
+        setLayout(new BorderLayout(10,0));
+        Font theFont = new java.awt.Font("Monospaced", 0, 12);
+
+         _dataModel = new RW_DataModel();
+         _table.setModel(_dataModel);
+
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(100);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(50);
+        _table.setRowSelectionAllowed(false);
+        setupRenderer();
+
+        theScrollPane.getViewport().add(_table, null);
+        add(theScrollPane,"Center");
+
+        setVisible(true);
+
+        JButton refineButton = new JButton("Filter Documents");
+        refineButton.setBorder(BorderFactory.createRaisedBevelBorder());
+        refineButton.setMaximumSize(new Dimension(30,70));
+
+        MouseListener ml = new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+                 refineEngine();
+            }
+        };
+
+        refineButton.addMouseListener(ml);
+        refineButton.setFont(theFont);
+        add(refineButton,"South");
+    }
+
+    public void refineEngine()
+    {
+        // the filter words are available from within _dataModel, so nothing
+       // is passed on
+       parentPane.hLinkPanel.setText(_dataModel.getRefinedDocs());
+       String trec_id[] = _dataModel.getRefinedTrecIDs();
+       String filter_Words[] = _dataModel.getFilterWords();
+       String s = new String("");
+       for( int i = 0; i < filter_Words.length; i++ )
+       {
+            s += filter_Words[i] + " ";
+       }
+    }
+        
+
+    public void update(vishnu.testvis.dendro.ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+
+    public void clear()
+    {
+        _dataModel.clear();
+    }
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/DendroPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/DendroPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/DendroPanel.java	(revision 14150)
@@ -0,0 +1,61 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.sammon.*;
+import java.util.*;
+
+/** Panel containing the dendro map interface */
+public class DendroPanel extends JPanel
+{
+
+  public final static int LIST_ON_LEFT = 0;
+  public final static int LIST_ON_RIGHT = 1;
+  
+  public JPanel picturePanel;
+  public TreePanel treePanel;
+  public DendroKeywordPanel keywordPanel;
+  public HLinkPanel hLinkPanel;
+  public Stack dataStack;
+  public DataManager dataMan=null;
+  public Vishnu currentMain;
+  public JPanel topPanel = new JPanel();
+
+  public DendroPanel(Vishnu parent)
+  {
+    currentMain = parent;
+    setUpGui();
+  }
+  
+  public void setUpGui()
+  {
+        setLayout(new BorderLayout(10,10));
+        setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+        treePanel = new TreePanel(this,currentMain);
+        keywordPanel = new DendroKeywordPanel(currentMain,this);	
+	Dimension d = currentMain.getSize();
+        hLinkPanel = new HLinkPanel((int)(d.width)/4,d.height,currentMain);
+        picturePanel = new JPanel();
+        picturePanel.setLayout(new BorderLayout(0,3));
+              
+        picturePanel.add(treePanel,"Center");
+        add("East",  hLinkPanel.linkPane);
+        add("Center",   picturePanel);
+        add("West",   keywordPanel);
+        setVisible(true);
+  }
+  
+  public void update()
+  {
+    dataMan = currentMain.dataManager;
+    dataStack = new Stack(); 
+    treePanel.clear();
+    treePanel.update();
+    keywordPanel.clear();
+    hLinkPanel.clear();
+  }
+}
+  
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Distance.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Distance.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Distance.java	(revision 14150)
@@ -0,0 +1,10 @@
+package vishnu.testvis.dendro;
+
+// Abstract class
+
+public abstract class Distance
+{
+	public abstract double get(double[] v1, double[] v2, int size);
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java	(revision 14150)
@@ -0,0 +1,16 @@
+package vishnu.testvis.dendro;
+
+public class EuclideanDistance extends Distance
+{
+    public double get(double[] v1, double[] v2, int size)
+    {
+  	double sum = 0, x;
+   //System.out.println("Array length "+ v1.length);
+	for(int c=0;c<v1.length;c++)
+	{
+      	    x = v1[c] - v2[c];
+	    sum += x*x;
+	}
+  	return Math.sqrt(sum);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java	(revision 14150)
@@ -0,0 +1,80 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import vishnu.testvis.visual.*;
+
+class HLinkPanel extends JEditorPane {
+
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+
+    public int numDisplayedDocs = 15;
+
+    public HLinkPanel(int width,int length, Vishnu currentMain) {
+
+        linkPane = new JScrollPane(this);
+        this.currentMain=currentMain;
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        //linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener{
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            //System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            //System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+            {
+
+                String hpresult = he.getDescription();
+                int index = Integer.parseInt(hpresult);
+                System.out.println("hyperlink result: " + index);
+                if (index > 0)
+                    currentMain.dataManager.fetchDocNo(index);
+            }
+        }
+    }
+
+    /*public void update()
+    {
+        int count = currentMain.dataManager.getDocCount();
+        Vector subSet = new Vector(count);
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+            subSet.addElement(new Integer(i));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDocumentDescription(subSet);
+
+        setText(htmlDescription);
+
+    }*/
+
+    public void clear()
+    {
+        setText("");
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Hierarchical.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Hierarchical.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Hierarchical.java	(revision 14150)
@@ -0,0 +1,356 @@
+package vishnu.testvis.dendro;
+
+import vishnu.cluster.*;
+import java.util.*;
+import vishnu.testvis.object.*;
+
+public class Hierarchical
+{
+    public double[] minrow;
+    public int[]    colind;
+    //public SimMatrix smatrix;
+    public TriangleIndex triangle;
+    public final double weight = 3.2;
+    public DataManager dataManager;
+
+    public Hierarchical(int matrix_rows)
+    {
+	// create Triangle with "limit" arg = maximum #clusters
+	// an object to help with indexing the cells 
+	// in a triangular matrix
+	triangle = new TriangleIndex(matrix_rows);
+	
+    }
+    
+    // set each minrow[i] to the value of the minimum element of that row
+    void set_minrow (SimMatrix m, int row)
+    {
+	// triangular matrix without diagonal, hence zeroth row is all zero
+        if(row == 0) return;
+	
+	int i;
+	int index_start, index_end;
+        
+        // first column
+	index_start = triangle.index (row, 0);
+	// last column (row(!)-1 because as many rows as columns)
+	index_end   = triangle.index (row, row - 1); 
+	
+	minrow[row] = m.matrix[index_start];
+	colind[row] = index_start;
+	
+    	for (i = index_start + 1; i <= index_end; i++)
+	    {
+      		if (m.matrix[i] < minrow[row])
+		    {
+			minrow[row] = m.matrix[i];
+			colind[row] = i;
+		    }
+	    }
+    }
+    
+    // return the ROW in which the minimum disimilarity resides
+    int get_minrow (int last_row)
+    {
+	int i;
+	int row;
+	double value;
+	
+	row   = 1;
+	value = minrow[1];
+	
+	for (i = 2; i <= last_row; i++)
+	    {
+      		if (minrow[i] < value)
+		    {
+			row = i;
+			value = minrow[i];
+		    }
+	    }
+	
+	return row;
+    }
+    
+    // num_of_rows is number of documents, or subsample for Buckshot
+    // indices is a vector of the doc ids (made necessary by Buckshot, which uses a subsample of all docs)
+    Cluster clustering (DataManager dm, SimMatrix sim_matrix, int num_of_rows,
+			double[] heights, LinkFunc linkage_func, Vector indices)
+    {
+	Cluster child_1;
+	Cluster child_2;
+	Cluster parent=null;
+	ClusterArray cluster_ary;
+	
+	
+        int min_row, index, row, col, i;
+	double sim;
+
+        int last_row_index, sim_matrix_size;
+
+        sim_matrix_size = num_of_rows *(num_of_rows - 1) / 2;
+  		
+	// Place each of the documents in a cluster of each own 
+	// there are vector.length docs but only num_of_rows = limit places in the array
+	// may be two-stage clustering process, i.e. cluster subset of docs and then add the rest
+	//	cluster_ary = new ClusterArray (dm, num_of_rows);
+
+	cluster_ary = new ClusterArray(dm,indices);
+	last_row_index = num_of_rows - 1;
+	
+	// a variable that is a double and exists for each row - what can this be?
+	// the lowest dissimilarity score?
+	minrow = new double[num_of_rows];
+	
+	// a variable that is an int and exists for each row - what can that be?
+	// the document index of the document with that lowest score?
+	colind = new int[num_of_rows];
+	
+  	    for (i = 1; i < num_of_rows; i++)
+        {
+            set_minrow(sim_matrix, i);
+        }
+        
+        // do all the hierarchical clustering by running over all rows, updating the matrix 
+        // as a new cluster is formed each time
+  	    for (i = 0; i < num_of_rows - 1; i++)
+        {
+                // find the row with the lowest disimilarity
+      	        min_row = get_minrow(last_row_index);
+      	        sim = minrow[min_row];
+      	        
+      	        // the column with the lowest dis... (sim) in that row
+      	        index  = colind[min_row];
+
+                // array of heights, DESCENDING disim, i.e. sim goes last
+      	        heights[num_of_rows - 2 - i] = sim; 
+
+      	        row = triangle.i_ind[index];
+      	        col = triangle.j_ind[index];
+      	        
+      	        newMerge(row,col,sim_matrix,last_row_index,linkage_func,cluster_ary);
+      	            
+      	        sim_matrix_size -= last_row_index;
+      	        
+      	        child_1 = (Cluster)cluster_ary.contents.elementAt(row);
+      	        child_2 = (Cluster)cluster_ary.contents.elementAt(col);
+
+      	        parent = new Cluster (-1, null, child_1, child_2, sim, child_1._items + child_2._items);
+      	        Vector vDoc = new Vector();
+      	        vDoc.addAll(child_1.getDocVect());
+      	        vDoc.addAll(child_2.getDocVect());
+      	        parent.setDocVect(vDoc);
+      	        
+      	        Vector vDocObj = new Vector();
+      	        vDocObj.addAll(child_1.getDocObjVect());
+      	        vDocObj.addAll(child_2.getDocObjVect());
+      	        parent.setDocObjVect(vDocObj);
+      	        
+      	        int[] wordFreq1 = child_1.getWordFreq();
+      	        int[] wordFreq2 = child_2.getWordFreq();
+      	        for( int z = 0; z < wordFreq1.length; z++ )
+      	        {
+      	            wordFreq1[z] += wordFreq2[z];
+      	        }
+      	        parent.setWordFreq(wordFreq1);
+      	        
+      	        child_1._parent = parent;
+      	        child_2._parent = parent;
+      	        cluster_ary.contents.setElementAt(cluster_ary.contents.elementAt(last_row_index),row);
+      	        cluster_ary.contents.setElementAt(parent,col);
+      	        last_row_index--;
+         	        
+    	    }
+
+  	    return parent;
+
+        }
+        
+    void newMerge(int r, int c, SimMatrix m, int lastRowIndex, LinkFunc linkage_func, ClusterArray clusterArr)
+    {
+        int n = lastRowIndex+1;
+        double newMatrix[][] = new double[n][n];
+        for( int i = 0; i < n; i++ )
+        {
+            for( int j = 0; j < n; j++ )
+            {
+                if( i != j ) newMatrix[i][j] = (double)((int)(m.matrix[triangle.index(i,j)]*100))/100;
+                    else newMatrix[i][j] = 0;
+            }
+        }
+        
+        for( int i = 0; i < n; i++ )
+        {
+            int sizeA =((Cluster)clusterArr.contents.elementAt(c))._items + ((Cluster)clusterArr.contents.elementAt(r))._items;
+            int sizeB = ((Cluster)clusterArr.contents.elementAt(i))._items;
+            // large if cluster differ in size
+            double scale = weight * (double)((sizeA+sizeB)/2*(sizeA+sizeB)/2)/(sizeA*sizeA+sizeB*sizeB);
+            
+            newMatrix[c][i] = scale*linkage_func.link(newMatrix[c][i],newMatrix[r][i]);            
+            
+        }
+        
+        newMatrix[c][c] = 0;
+        newMatrix[c][r] = newMatrix[c][n-1];
+        for( int i = 0; i < n; i++ )
+        {
+            newMatrix[i][c] = newMatrix[c][i];
+        }
+        
+        
+        for( int i = 0; i < n; i++ )
+        {
+            if( i != c ) newMatrix[r][i] = newMatrix[n-1][i];
+        }
+        newMatrix[r][r] = 0;
+        for( int i = 0; i < n; i++ )
+        {
+            newMatrix[i][r] = newMatrix[r][i];
+        }
+        
+        int index = 0;
+        for( int i = 0; i < n; i++ )
+        {
+            for( int j = 0; j < n; j++ )
+            {
+                if( j < i )
+                {
+                    m.matrix[index] = newMatrix[i][j];
+                    index++;
+                }
+          
+            }
+          
+       }
+       
+       for (int i = 1; i < n; i++)
+       {
+            set_minrow(m, i);
+       }
+        
+     
+    }
+
+    // form a new cluster from the two most similar documents
+    void merge (int i, int j, SimMatrix m, int n, LinkFunc linkage_func)
+	{
+  /* i and j are the matrix elements with maximum similarity         */
+  /* matrix is the triangle similarity matrix with last row index n  */
+  /* linkage_func is a pointer to the linkage function               */
+
+  /* i > j */
+
+  	    int c, k, index1, index2;
+
+        // go over all rows
+  	    for (k = 0; k <= n; k++)
+    	{
+    	    // if k < j the affected elements are on row j
+      		if (k < j)
+		    {
+		        // index1 is the smaller of the two, as j < i
+	     	    index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+        	}
+      		else if (k > j && k < i)
+		    {
+	            // if j < k < i then only one element is updated per row
+	  	        index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+    	  	    if (index1 == colind[k])
+	    	    {
+	            // the previous minimum of the row has been changed so recaculate minimum of whole row
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind
+	      		    if (m.matrix[index1] < minrow[k])
+        			{
+		          	    minrow[k] = m.matrix[index1];
+		  	            colind[k] = index1;
+			        }
+	    	    }
+		    }
+      		else if (k > i && k < n)
+		    {
+	  	        index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	  
+          	    if (index1 == colind[k])
+	    	    {
+	                // the previous minimum of the row has been changed so recaculate minimum of whole row */
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      		    if (m.matrix[index1] < minrow[k])
+			        {
+	  		            minrow[k] = m.matrix[index1];
+                        colind[k] = index1;
+			        }
+	    	    }
+
+	  	        index1 = triangle.index (k, i);
+	  	        index2 = triangle.index (k, n);
+	  	        m.matrix[index1] = m.matrix[index2];
+	  	  	    if (index1 == colind[k])
+	    	    {
+	            // the previous minimum of the row has been changed so recaculate minimum of whole row 
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      		    if (m.matrix[index1] < minrow[k])
+			        {
+		  	            minrow[k] = m.matrix[index1];
+		  	            colind[k] = index1;
+			        }
+	    	    }
+		    }
+      	    else if (k == n  && i != n)
+		    {
+	  	        index1 = triangle.index (k, j);
+	  			index2 = triangle.index (k, i);
+	  			m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	  			if (index1 == colind[k])
+	    		{
+	                // the previous minimum of the row has been changed so recaculate minimum of whole row */
+	      			set_minrow(m, k);
+	    		}
+	  			else
+	    		{
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      			if (m.matrix[index1] < minrow[k])
+					{
+		  				minrow[k] = m.matrix[index1];
+		  				colind[k] = index1;
+					}
+	    		}
+
+	   			for (c = 0; c < i; c++)
+	    		{
+	      			index1 = triangle.index (c, i);
+	      			index2 = triangle.index (c, n);
+	      			m.matrix[index1] = m.matrix[index2];
+	      		}
+			}
+						
+    	}
+  		set_minrow(m, i);
+  		set_minrow(m, j);
+	}
+
+}
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Iterative.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Iterative.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/Iterative.java	(revision 14150)
@@ -0,0 +1,111 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+class Iterative
+{
+    Vector vector;
+    int VECTOR_SIZE;
+    
+    Iterative(int vsize)
+    {
+	VECTOR_SIZE = vsize;
+    }
+    
+    public void printVector(double[] a)
+    {
+        int i;
+
+	for (i = 0; i < VECTOR_SIZE; i++)
+    	    System.out.println("v[" + i + "] = " + a[i]);
+    }
+    
+    
+    void cluster_center (Cluster cluster, double [][] vectors, double[] centroid)
+    {
+        int i;
+	
+      	if (cluster._id == -1)
+	    {
+		cluster_center (cluster._child_1, vectors, centroid);
+		cluster_center (cluster._child_2, vectors, centroid);
+	    }
+        else
+	    {
+		double[] doubleArray = vectors[cluster._id];
+		for (i = 0; i < doubleArray.length; i++)
+	            centroid[i] += doubleArray[i];
+	    }
+    }
+    
+    void compute_cluster_centers (ClusterArray clusters, int num_of_clusters, double [][] vectors)
+    {
+	int i, j; double ssum;
+	Cluster clust;
+	
+	for (i = 0; i < num_of_clusters; i++)
+	    {
+		clust = (Cluster)clusters.contents.elementAt(i);
+	        clust._centroid = new double[VECTOR_SIZE];
+		cluster_center (clust, vectors, clust._centroid);
+		for (ssum = j = 0; j < VECTOR_SIZE; j++)
+		    ssum += clust._centroid[j] * clust._centroid[j];
+		if(ssum>0)                // normalise
+		    for(ssum = 1.0/Math.sqrt(ssum), j = 0; j < VECTOR_SIZE; j++)
+			clust._centroid[j] *= ssum;
+	    }
+    }
+    
+
+    void place_item (ClusterArray clusters,
+		     int num_of_clusters,
+		     double [][] vectors,
+		     int item_id,  Distance sim_func)
+    {
+	double max_sim, sim;
+	int max_sim_index;
+	
+	int i;
+	
+	max_sim_index = 0;
+        Cluster clust = (Cluster)clusters.contents.elementAt(0);
+	max_sim = sim_func.get(clust._centroid,
+			       vectors[item_id], VECTOR_SIZE);
+	
+  	    for (i = 1; i < num_of_clusters; i++)
+    	{
+            clust = (Cluster)clusters.contents.elementAt(i);
+      	    sim = sim_func.get(clust._centroid,
+			vectors[item_id], VECTOR_SIZE);
+
+            if (sim < max_sim)
+	        {
+	  	        max_sim_index = i;
+	  	        max_sim = sim;
+	        }
+    	}
+  	    
+  	    clust = (Cluster)clusters.contents.elementAt(max_sim_index);
+        clust._iter_items.addElement(new TheInt(item_id));
+	    clust._iter_items_num++;
+    }
+
+    // add all documents not yet added to the existing cluster hierarchy (which is based on the 
+    // limit first documents. Hence there are vector.length - limit docs to add
+    // consequently index runs from limit (start_index) to vector.length (end_index)
+    void clustering (ClusterArray clusters, TheInt num_of_clusters,
+                      double[][] vectors, int start_index, int end_index,
+                      Distance sim_func)
+    {
+        int i;
+    
+  	    if (start_index >= end_index)
+    	    	return;
+
+  	    compute_cluster_centers(clusters, num_of_clusters.val, vectors);
+
+  	    for (i = start_index; i < end_index; i++)
+      	    	place_item(clusters, num_of_clusters.val, vectors, i, sim_func);
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/LinkFunc.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/LinkFunc.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/LinkFunc.java	(revision 14150)
@@ -0,0 +1,8 @@
+package vishnu.testvis.dendro;
+
+//Abstract linkfunctions class
+
+public abstract class LinkFunc
+{
+	abstract double link(double a, double b);
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/MaxLink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/MaxLink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/MaxLink.java	(revision 14150)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class MaxLink extends LinkFunc
+{
+	public double link(double a, double b)
+	{
+  		return (a > b) ? a : b;      //*99+b: b*99+a)/100;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/MinLink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/MinLink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/MinLink.java	(revision 14150)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class MinLink extends LinkFunc
+{
+	public double link(double a, double b)
+	{
+  		return (a < b? a*999+b: b*999+a)/1000;
+	}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java	(revision 14150)
@@ -0,0 +1,76 @@
+package vishnu.testvis.dendro;
+
+import vishnu.testvis.object.*;
+
+public class QSortForDocObj
+{
+
+    private boolean pauseTrue(int lo, int hi) throws Exception {
+        return true;
+    }
+  
+   void QuickSort(DocObj a[], int lo0, int hi0) throws Exception
+   {
+      int lo = lo0;
+      int hi = hi0;
+      int mid;
+
+      if ( hi0 > lo0)
+      {
+
+           mid = a[ ( lo0 + hi0 ) / 2 ].getNumber();
+
+         // loop through the array until indices cross
+         while( lo <= hi )
+         {
+            /* find the first element that is greater than or equal to
+             * the partition element starting from the left Index.
+             */
+             while( ( lo < hi0 ) && pauseTrue(lo0, hi0) && ( a[lo].getNumber() < mid ))
+                 ++lo;
+
+            /* find an element that is smaller than or equal to
+
+             * the partition element starting from the right Index.
+             */
+             while( ( hi > lo0 ) && pauseTrue(lo0, hi0) && ( a[hi].getNumber() > mid ))
+                 --hi;
+
+            // if the indexes have not crossed, swap
+            if( lo <= hi )
+            {
+               swap(a, lo, hi);
+               ++lo;
+               --hi;
+            }
+         }
+
+         /* If the right index has not reached the left side of array
+          * must now sort the left partition.
+
+          */
+         if( lo0 < hi )
+            QuickSort( a, lo0, hi );
+
+         /* If the left index has not reached the right side of array
+          * must now sort the right partition.
+          */
+         if( lo < hi0 )
+            QuickSort( a, lo, hi0 );
+
+      }
+   }
+
+   private void swap(DocObj a[], int i, int j)
+   {
+      DocObj D;
+      D = a[i];
+      a[i] = a[j];
+      a[j] = D;
+   }
+
+   public void sort(DocObj a[]) throws Exception
+   {
+      QuickSort(a, 0, a.length - 1);
+   }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/SimMatrix.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/SimMatrix.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/SimMatrix.java	(revision 14150)
@@ -0,0 +1,46 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+// kainof DistanceMatrix
+public class SimMatrix
+{
+  public double[] matrix;
+
+  // "vectors" is the expanded SparseMatrix, i.e. doc x keywords
+  // VECTOR_SIZE = #keywords
+  public SimMatrix(double[][] vectors,
+      int limit, Distance simfunc, int VECTOR_SIZE)
+  {
+    int i,j;
+    double iarray[];
+    
+    // size = #docs
+    int size=vectors.length, index=0;
+    
+    // to exclude diagonal and conjugates
+    // - - - - -
+    //         |
+    // *       |
+    // * *     |
+    // * * *   |    
+    int realSize = (size * (size - 1)/2);
+    matrix=new double[realSize];
+    System.out.println("Similarity matrix size: " + size);
+    
+    for(i = 1; i < size; i++)
+    {
+        iarray=vectors[i];
+        for(j = 0; j < i; j++)
+        {
+            matrix[index] = simfunc.get(iarray,
+                            vectors[j],
+                            VECTOR_SIZE);
+	        index++;
+        }
+    }
+  }
+
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TheInt.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TheInt.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TheInt.java	(revision 14150)
@@ -0,0 +1,17 @@
+package vishnu.testvis.dendro;
+
+public class TheInt
+{
+	public int val;
+
+	TheInt(int x)
+	{
+		val = x;
+	}
+	
+	public void setInt(int i)
+	{
+	    val = i;
+	}
+	
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TheVectors.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TheVectors.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TheVectors.java	(revision 14150)
@@ -0,0 +1,79 @@
+package vishnu.testvis.dendro;
+
+import java.io.*;
+import java.util.*;
+
+public class TheVectors
+{
+    public Vector theData;
+    public int size;
+    public Vector coding;
+    public int zeros;
+
+
+    TheVectors(String file, int VECTOR_SIZE)
+    {
+        theData= new Vector(VECTOR_SIZE);
+	coding= new Vector(VECTOR_SIZE);
+        int index,code =0, zerolist=0;
+	double ssum,value;
+    	try
+	{
+      	    FileReader inFile = new FileReader(file);
+      	    BufferedReader inLines = new BufferedReader(inFile);
+	    size=0;
+            String line = null, tokenString;
+	    do
+	    {
+
+		line = inLines.readLine();
+		if (line!= null)
+		{
+
+		  double[] row= new double[VECTOR_SIZE];
+ 					StringTokenizer tokens = new StringTokenizer(line," ");
+                  ssum=0.0;
+   		  for (index=0; index<VECTOR_SIZE; index++)
+		  {
+                      if(tokens.hasMoreTokens())
+                      {
+                          tokenString = tokens.nextToken();
+           	          value = Double.parseDouble(tokenString);
+		          row[index]= value;
+		          ssum+= value*value;
+                      }
+                      else
+                      {
+                          row[index]=0.0;
+                      }
+	          }
+	          if(index==0) continue;
+                  ssum=Math.sqrt(ssum);
+	          if(ssum!=0.0)
+	          {
+	              for(int i=0;i<index;i++)
+		      row[i]/=ssum;
+		      theData.addElement(row);
+		      coding.addElement(new TheInt(code));
+                      size++;
+		  }
+		  else
+		  {
+		      if(zerolist==0)
+		      {
+			//System.out.print("0 ");
+	                  zerolist=1;
+		      }
+		      System.out.print(code + " ");
+		  }
+	      }
+	      code++;
+	  }while(line != null);
+	}
+	catch(IOException e)
+	{
+	    System.err.println("IO Exception in read_vectors");
+	    e.printStackTrace();
+        }
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TreePanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TreePanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TreePanel.java	(revision 14150)
@@ -0,0 +1,822 @@
+package vishnu.testvis.dendro;
+
+import java.awt.image.*;
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import java.util.Enumeration;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.sammon.*;
+import vishnu.datablock.*;
+
+
+public class TreePanel extends JPanel
+{
+    
+    DendroPanel parent;
+    Vishnu currentMain;
+    DataManager dataManager;
+    DataBlock data;
+    Image offScreen=null;
+    Dimension offScreenSize;
+    Graphics offGraphics;
+    Graphics2D offGraphics2D;
+    Vector clusterVect = null;
+    Cluster root = null;
+    Cluster selectedRoot = null;
+    ClusterObj selectedCluster = null;
+    ClusterObj oldCluster = null;
+    
+    int selectedClusterID = -1; 
+    int oldClusterID = -1;
+    JPopupMenu popup;
+    boolean constructed = false;
+    int selectedDepth = -1;
+    int initBranchLength = 85;
+
+    boolean firstTime = true;
+    Graphics2D big;
+    BufferedImage bi;
+    Rectangle area;
+    Dimension d = null;
+    
+    public TreePanel(DendroPanel parent, Vishnu frame)
+    {
+        currentMain = frame;
+        this.parent = parent;
+        dataManager = frame.dataManager;
+             
+        popup = new JPopupMenu();
+        popup.setVisible(false);
+        add(popup);
+        
+        this.addComponentListener( new ComponentAdapter()
+        {
+            public void componentResized(ComponentEvent evt)
+	    {
+		repaint();
+            }
+        }
+        );
+        
+        this.addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent mouseEvent)
+            {
+                int x = mouseEvent.getX();
+                int y = mouseEvent.getY();
+                
+                if( clusterVect != null )
+                {
+                    
+                    oldClusterID = selectedClusterID;
+                    oldCluster = selectedCluster;
+                    selectedClusterID = -1;
+                    selectedCluster = null;
+                    
+                    boolean notFound = true;
+                    int i = 0;
+                    
+                    while( notFound && (i < clusterVect.size()) )
+                    {
+                       ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                       
+                       if( co.inCircle(x,y) )
+                       {
+                            selectedCluster = co;
+                            selectedClusterID = i;
+                                        
+                            if( selectedClusterID != oldClusterID )
+                            {
+                                 repaint();
+                            }
+                            notFound = false;
+                       }
+                       i++;
+                    }
+                    
+                    if( (oldClusterID != selectedClusterID) && ( oldClusterID != -1 ) )
+                    {
+                        popup.setVisible(false);
+                        repaint();
+			//update();
+                    }
+                    
+                    if( selectedClusterID != -1 )
+                    {
+                        if( selectedClusterID != oldClusterID )
+                        {
+                            popup.removeAll();
+                            int size = selectedCluster.cluster._items;
+                            String str =  size + ((size > 1) ? " documents" : " document");
+                            JMenuItem menuItem = new JMenuItem(str);
+                            menuItem.addActionListener(new ActionListener()
+                            {
+                                public void actionPerformed(ActionEvent actionEvent)
+                                {
+                                    getDocObjs();
+                                    String[] drillWords = selectedCluster.getFreqWords();
+                                    String keyWordStr = new String(drillWords[0]+" "+drillWords[1]+" "+drillWords[2]);
+
+                                    updateHLinkPanel();                                
+                                    updateKeyWordTable();
+                                }
+                                
+                            });
+                            
+                            popup.add(menuItem);
+                            
+                            popup.add(new JPopupMenu.Separator());
+                            
+                            int mode0 = -1;
+                            
+                            // if the currentRoot is selected, dont show "drilldown"
+                            if( selectedClusterID == 0 )
+                            {
+                                // if we are at the top, dont show "undrill"
+                                if( ((Object)selectedCluster.cluster).equals((Object)root) )
+                                {
+                                    str = "Top of hierarchy";
+                                    mode0 = 2;
+                                }
+                                else
+                                {
+                                    str = "Sequential undrill";
+                                    mode0 = 1;
+                                }
+                            }
+                            // if somewhere else in the tree
+                            else
+                            {
+                                if( selectedCluster.cluster._items > 1 )
+                                {
+                                    str = "Drill down";
+                                    mode0 = 0;
+                                }
+                                else
+                                {
+                                    str = "Outermost twigs";
+                                    mode0 = 3;
+                                }
+                            }
+                            
+                            final int mode = mode0;
+                            
+                            menuItem = new JMenuItem(str);
+                            if( (mode != 0 ) && (mode != 1) )
+                                menuItem.setEnabled(false);
+                                
+                            menuItem.addActionListener(new ActionListener()
+                            {
+                                public void actionPerformed(ActionEvent evt)
+                                {
+                                    if( mode == 0 )
+                                    {
+                                        selectedRoot = selectedCluster.cluster;
+                                        String[] drillWords = selectedCluster.getFreqWords();
+                                        String keyWordStr = new String(drillWords[0]+" "+drillWords[1]+" "+drillWords[2]);
+                                        
+                                        // to establish new clusterobj vector
+                                        constructed = false;   
+                                        repaint();
+					//update();                                  
+                                    }
+                                    if( mode == 1 )
+                                    {
+                                        selectedRoot = selectedRoot._parent;
+                                        constructed = false;
+                                        repaint();
+					//update();
+                                    }
+                                }
+                            });
+                            popup.add(menuItem);
+                            
+                            if( mode == 1 )
+                            {
+                                menuItem = new JMenuItem("Total undrill");
+                                menuItem.addActionListener(new ActionListener()
+                                {
+                                    public void actionPerformed(ActionEvent evt)
+                                    {
+                                        selectedRoot = root;
+                                        constructed = false;
+                                        //update();
+					repaint();
+                                    }
+                                });
+                                popup.add(menuItem);
+                            }
+                            
+                            popup.add(new JPopupMenu.Separator());
+                            int[] wordFreq = selectedCluster.cluster.getWordFreq();
+                            String[] words = getMostFrequentWords(wordFreq);
+                            
+                            selectedCluster.setFreqWords(words);
+                            
+                            menuItem = new JMenuItem(words[0]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            menuItem = new JMenuItem(words[1]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            menuItem = new JMenuItem(words[2]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            popup.validate();
+                            popup.setVisible(true);
+                        }
+                        
+                        popup.show(mouseEvent.getComponent(), mouseEvent.getX(), mouseEvent.getY());
+                        mouseEvent.consume();
+                    }     
+                }  // of clustervect != null
+             } // of mouse moved
+        });
+        
+        setVisible(true);
+    }
+    
+    public void clear()
+    {
+        constructed = false;
+    }
+
+        
+    public String[] getMostFrequentWords(int[] wordFreq)
+    {
+        
+        
+        int max1Freq = 0;
+        int max1Index = 0;
+        int max2Freq = 0;
+        int max2Index = 0;
+        int max3Freq = 0;
+        int max3Index = 0;
+        
+        for( int i = 0; i < wordFreq.length; i++ )
+        {
+            if( wordFreq[i] > max3Freq )
+            {
+                if( wordFreq[i] > max2Freq )
+                {
+                    if( wordFreq[i] > max1Freq )
+                    {
+                        max3Freq = max2Freq;
+                        max3Index = max2Index;
+                        max2Freq = max1Freq;
+                        max2Index = max1Index;
+                        max1Freq = wordFreq[i];
+                        max1Index = i;
+                    }
+                    else
+                    {
+                        max3Freq = max2Freq;
+                        max3Index = max2Index;
+                        max2Freq = wordFreq[i];
+                        max2Index = i;
+                    }
+                }
+                else
+                {
+                    max3Freq = wordFreq[i];
+                    max3Index = i;
+                }
+            }
+                
+        }
+        
+        data = dataManager.getDataBlock();
+        String[] words = new String[3];
+        words[0] = data.words[max1Index];
+        words[1] = data.words[max2Index];
+        words[2] = data.words[max3Index];
+        return words;
+    }
+    
+    public void getDocObjs()
+    {
+       
+       Vector docVect = selectedCluster.cluster.getDocVect();
+       DocObj[] docs = new DocObj[docVect.size()];
+       
+       for( int i = 0; i < docVect.size(); i++ )
+       {
+            int index = ((Integer)docVect.elementAt(i)).intValue();
+            docs[i] = dataManager.getDocObjAt(index);
+       }    
+       
+       
+       if( !selectedCluster.getOrderStatus() )
+       {
+            QSortForDocObj qSort = new QSortForDocObj();
+            try
+            {   
+                 qSort.sort(docs);
+            }
+            catch (Exception e)
+            {}
+            
+            selectedCluster.setDocObjs(docs);
+            selectedCluster.setOrderStatus(true);
+       }
+       
+        
+    }
+    
+    public void updateHLinkPanel()
+    {
+       
+        String htmlDescription = selectedCluster.formatDocs();
+        parent.hLinkPanel.setText(htmlDescription);
+    }
+    
+    public void updateKeyWordTable()
+    {
+        parent.keywordPanel.update(selectedCluster);
+    }
+    
+    
+    public void setConstructed(boolean b)
+    {
+        constructed = b;
+    }
+    
+    
+    public void getData()
+    {
+        dataManager = currentMain.dataManager;
+        data = dataManager.getDataBlock();
+        SparseMatrix m = data.matrix;
+        
+        Vector idVect = new Vector();
+        Vector sammCluster =  dataManager.getSammonClusters();
+        
+        for( int i = 0; i < sammCluster.size(); i++ )
+        {
+            SammCluster cluster = (SammCluster)sammCluster.elementAt(i);
+            idVect.addAll(cluster.getIDs());
+        }
+        
+        for( int c = 0; c < idVect.size(); c++ )
+        {
+            Integer i = (Integer)idVect.elementAt(c);
+        }
+        
+        for (int c = 0; c< idVect.size();c++)
+        {
+          // the document id of the cth document
+          Integer i = (Integer)idVect.elementAt(c);
+        }
+        
+        //int [][] full = m.getExpanded(); 
+        float [][] full = m.getExpanded(); 
+        // the number of new clusters generated is either 20
+        // or if there are not many docs half the #docs
+               
+        int limit =  m.getRowCount();
+        
+        int clusterNum = 10;
+                      
+        Clustering clustering = new Clustering(dataManager,Clustering.COMPLETE_LINKAGE,
+					       full,m.getRowCount(),
+					       m.getColumnCount(),1,
+					       clusterNum,limit);
+          
+        root = clustering.getRoot();
+        selectedRoot = root;
+    }
+    
+    
+    public void update(Graphics g)
+    {
+	Graphics2D g2 = (Graphics2D)g;
+	
+	d = getSize();
+	int w = d.width;
+	int h = d.height;
+	
+	area = new Rectangle(d);
+	bi = (BufferedImage)createImage(w, h);
+	offGraphics2D = bi.createGraphics();
+	area.setLocation(w/2-50, h/2-25);
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	
+	offGraphics2D.setColor(Color.white);
+	offGraphics2D.fillRect(0,0,d.width,d.height);
+		
+	if( !constructed )
+	    {
+		clusterVect = new Vector();
+		selectedCluster = null;
+		selectedClusterID = -1;
+	    }
+	
+	int initRad = 0;  
+	initBranchLength = (int)(Math.min(d.width,d.height)*0.12);
+	
+	Cluster cc = selectedRoot;
+	Point center = new Point(d.width/2,d.height/2);
+	int radius = (int)(Math.log(cc._items))* 3;
+	if( radius < 2 ) radius = 2;
+	
+	// all the leaf cluster ids underneath selected node
+	// to be painted bright red
+	Vector allSubNodeIDs = new Vector();
+	
+	if( !constructed ){
+	    ClusterObj rootObj = new ClusterObj(cc,center,radius,dataManager);
+	    clusterVect.addElement(rootObj);            
+	}
+	else{
+	    ClusterObj c = (ClusterObj)clusterVect.elementAt(0);
+	    c.setPosition(center);
+	    clusterVect.setElementAt(c,0);
+	    c = (ClusterObj)clusterVect.elementAt(0);
+	}
+	
+	int depth = 1;
+	TheInt ii = new TheInt(1);
+	
+	boolean selected = false;
+	
+	// used to paint the tree different colour under selected node
+	selectedDepth = -1;
+
+	if( cc._items > 1 ){
+	    double alpha = (double)Math.PI/Math.pow(2,depth);
+	    double theta = 0;
+	    
+	    if( (selectedClusterID == 0) )
+                {
+                    selected = true;                    
+                    selectedDepth = 0;
+                    offGraphics2D.setColor(Color.blue);
+                }
+
+	    paintPoke(offGraphics2D,cc._child_1,center.x,center.y,center,depth,theta,-alpha,initRad, selected, allSubNodeIDs, ii);
+	    paintPoke(offGraphics2D,cc._child_2,center.x,center.y,center,depth,theta,alpha,initRad, selected, allSubNodeIDs, ii);
+	}
+	
+	for( int i = 0; i < clusterVect.size(); i++ )
+            {
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                
+                // if leaf draw reddish
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(new Color((float)1, (float)0.7, (float)0.7));
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else  // draw blueish
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float) 1));
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                               
+            }
+            
+            for( int i = 0; i < allSubNodeIDs.size(); i++ )
+            {
+                int value = ((Integer)allSubNodeIDs.elementAt(i)).intValue();
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(value);    
+                // if leaf
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(Color.red);
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(Color.blue);
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                
+                
+            }
+            
+            if( (selectedCluster != null)  && (selectedCluster.cluster._items > 1) )
+            {
+                offGraphics2D.setColor(Color.blue);
+                offGraphics2D.fillArc(selectedCluster.getX()-selectedCluster.getRad(),selectedCluster.getY()-selectedCluster.getRad(),2*selectedCluster.getRad(),2*selectedCluster.getRad(),0,360);
+		}
+	    
+	    g2.drawImage(bi, 0, 0, this);
+	constructed = true;
+    }
+
+           
+    public void update()
+    {
+	
+	if( !constructed )
+            {
+                clusterVect = new Vector();
+                selectedCluster = null;
+                selectedClusterID = -1;
+            }
+	
+	Dimension d = getSize();
+	offScreenSize = d;
+	if( d.width <= 0 ) return;
+
+	offScreen = createImage(d.width,d.height);
+	offGraphics = offScreen.getGraphics();
+            
+	Graphics2D offGraphics2D = (Graphics2D)offGraphics;
+	
+	offGraphics2D.setColor(Color.white);
+	offGraphics2D.fillRect(0,0,d.width,d.height);
+	offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	offGraphics2D.setStroke(myStroke);	        
+	
+	int initRad = 0;  
+
+	Cluster cc = selectedRoot;
+	Point center = new Point(d.width/2,d.height/2);
+	int radius = (int)(Math.log(cc._items))* 3;
+	if( radius < 2 ) radius = 2;
+	
+	// all the leaf cluster ids underneath selected node
+	// to be painted bright red
+	Vector allSubNodeIDs = new Vector();
+	
+	if( !constructed )
+            {
+                ClusterObj rootObj = new ClusterObj(cc,center,radius,dataManager);
+                clusterVect.addElement(rootObj);            
+            }
+	else
+            {
+                ClusterObj c = (ClusterObj)clusterVect.elementAt(0);
+                c.setPosition(center);
+                clusterVect.setElementAt(c,0);
+                c = (ClusterObj)clusterVect.elementAt(0);
+            }
+	
+	int depth = 1;
+	TheInt ii = new TheInt(1);
+	
+	boolean selected = false;
+	
+	// used to paint the tree different colour under selected node
+	selectedDepth = -1;
+	
+	if( cc._items > 1 ){
+	    double alpha = (double)Math.PI/Math.pow(2,depth);
+	    double theta = 0;
+	    
+	    if( (selectedClusterID == 0) ){
+		selected = true;                    
+		selectedDepth = 0;
+		offGraphics2D.setColor(Color.blue);
+	    }
+	    
+	    paintPoke(offGraphics2D,cc._child_1,center.x,center.y,center,depth,theta,-alpha,initRad, selected, allSubNodeIDs, ii);
+	    paintPoke(offGraphics2D,cc._child_2,center.x,center.y,center,depth,theta,alpha,initRad, selected, allSubNodeIDs, ii);
+	}
+	
+	for( int i = 0; i < clusterVect.size(); i++ )
+            {
+                
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                
+                // if leaf draw reddish
+                if( co.cluster._items == 1 )
+		    {
+			int r = co.getRad();
+                    offGraphics2D.setColor(new Color((float)1, (float)0.7, (float)0.7));
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else  // draw blueish
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float) 1));
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                               
+            }
+            
+            for( int i = 0; i < allSubNodeIDs.size(); i++ )
+            {
+                int value = ((Integer)allSubNodeIDs.elementAt(i)).intValue();
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(value);    
+                // if leaf
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(Color.red);
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(Color.blue);
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                
+                
+            }
+            
+            if( (selectedCluster != null)  && (selectedCluster.cluster._items > 1) )
+            {
+                offGraphics2D.setColor(Color.blue);
+                offGraphics2D.fillArc(selectedCluster.getX()-selectedCluster.getRad(),selectedCluster.getY()-selectedCluster.getRad(),2*selectedCluster.getRad(),2*selectedCluster.getRad(),0,360);
+            }
+
+           repaint();
+           constructed = true;
+    }
+    
+    // theta is the angle in the polar representation
+    public void paintPoke(Graphics2D g, Cluster child, int x, int y, Point center, int d, double theta, double alpha,int r, boolean selected, Vector allSubNodeIDs, TheInt ii)
+    {
+	// translate parent cluster position to center
+	int x1 = x - center.x;
+	int y1 = y - center.y;
+	
+	// rotate cluster about origin by alpha to obtain intermediate tree point
+	int x_inter = (int)(x1 * Math.cos(alpha) + y1 * Math.sin(alpha)) + center.x;
+	int y_inter = (int)(x1 * - Math.sin(alpha) + y1 * Math.cos(alpha)) + center.y;
+
+	// rotate cluster about origin by theta
+	int x2 = (int)(x1 * Math.cos(-theta) + y1 * Math.sin(-theta));
+	int y2 = (int)(x1 * - Math.sin(-theta) + y1 * Math.cos(-theta));
+	
+	// extend radius
+	int dRad = (initBranchLength-d*10);
+	if( dRad <= 0 ) dRad = 10;
+	int x3 = x2 + dRad;
+	int y3 = y2;
+
+	// rotate back by (theta +/- alpha)
+	int x4 = (int)(x3 * Math.cos(theta+alpha) + y3 * Math.sin(theta+alpha));
+	int y4 = (int)(x3 * -Math.sin(theta+alpha) + y3 * Math.sin(theta+alpha));
+
+	// finally, add the center back again
+	int x_new = x4 + center.x;
+	int y_new = y4 + center.y;
+		
+	int startAngle =  (int)(180*theta/Math.PI);
+	int rangeAngle = (int)(180*alpha/Math.PI);
+			
+	boolean hitSelectedNode = false;
+	if( selectedClusterID != -1 )
+	{
+	    Object obj1 = (Object)child;
+	    ClusterObj selObj = (ClusterObj)clusterVect.elementAt(selectedClusterID);
+	    Object obj2 = (Object)selObj.cluster;
+	    hitSelectedNode = obj1.equals(obj2);
+	}
+	
+	int currentClusterID = -1;
+    
+	if( selectedClusterID != -1 )
+	    {
+		int w = 0;
+		boolean found = false;
+		while( w < clusterVect.size() && !found )
+		    {
+			Object obj1 = (Object)child;
+			ClusterObj obj = (ClusterObj)clusterVect.elementAt(w);
+			Object obj2 = (Object)obj.cluster;
+			if( obj1.equals(obj2) )
+			    {
+				currentClusterID = w;
+				found = true;
+			    }
+			w++;
+		    }
+	    }
+	
+	if( (d <= selectedDepth) && (!hitSelectedNode) )
+	    {	        
+		g.setColor(new Color( (float) 0.7, (float) 0.7, (float) 1)); 
+		selected = false;
+		
+	    }
+	
+	g.drawArc(center.x-r,center.y-r,2*r,2*r,startAngle,rangeAngle);
+	g.drawLine(x_inter,y_inter, x_new, y_new);
+	
+	if( hitSelectedNode )
+	    {	 
+		selectedDepth = d;
+		selected = true;
+		g.setColor(Color.blue);
+		Vector docVector = child.getDocVect();
+	    }
+	
+	g.setColor(selected ? Color.blue : new Color((float) 0.7, (float) 0.7, (float) 1) );
+	
+	int radius = (int)(Math.log(child._items))* 3;
+	if( radius < 2 ) radius = 2;
+
+	if( child._items == 1 )
+	    {
+		Vector docVector = child.getDocVect();
+		radius = Math.max(5,(int)Math.sqrt(docVector.size())*2);
+	    }
+	
+  	d++;
+	r = (int)(Math.sqrt((x_new-center.x)*(x_new-center.x) + (y_new-center.y)*(y_new-center.y)));
+	
+	Point P = new Point(x_new,y_new);
+	if( !constructed )
+	    {	    
+		ClusterObj cObj = new ClusterObj(child,P,radius,dataManager);
+		clusterVect.addElement(cObj);            
+	    }
+	else
+	    {
+		ClusterObj c = (ClusterObj)clusterVect.elementAt(ii.val);
+		c.setPosition(P);
+		clusterVect.setElementAt(c,ii.val);
+	    }
+	ii.val++;
+	
+   	double new_alpha = (double)alpha/2;
+   	
+   	if( selected )
+   	    allSubNodeIDs.addElement(new Integer(currentClusterID) );
+	
+	if( child._items > 1 && d < 6 ){
+	    paintPoke(g,child._child_1,x_new,y_new,center,d,theta+alpha,new_alpha,r,selected,allSubNodeIDs,ii);
+	    paintPoke(g,child._child_2,x_new,y_new,center,d,theta+alpha,-new_alpha,r,selected, allSubNodeIDs,ii);
+	}
+
+	// we have reached the outermost twigs
+	else{
+	    int[] wordFreq = child.getWordFreq();
+	    String[] words = getMostFrequentWords(wordFreq);
+	    
+	    Font f = new Font("Courier",Font.PLAIN,16);
+	    FontMetrics fm = g.getFontMetrics(f);
+	    int ascent = fm.getAscent();
+	    int height = fm.getHeight();
+	    int wordLength = fm.stringWidth(words[0]);
+	    g.setFont(f);
+	    
+	    if( x_new > center.x ){
+		g.drawString(words[0],x_new + Math.max(radius,5) + 10,y_new + Math.max(radius,5));
+	    }
+	    else{
+		g.drawString(words[0],x_new - Math.max(radius,5) - 10 - wordLength,y_new + Math.max(radius,5));
+	    }
+	}
+    }
+    
+    
+    public void paintTree(Cluster cc, Graphics g, int x, int y, int dist)
+    {
+        int dy = 10;		
+        int rad = 2;
+        int dmt = 4;
+	g.setColor(Color.white);
+	g.fillArc(x-rad,y-rad,dmt,dmt,0,360);
+	g.setColor(Color.blue);
+	g.drawOval(x-rad,y-rad,dmt,dmt);
+	String desc = String.valueOf(cc._items);
+	g.drawString(desc,x-(int)rad/2 - (desc.length()-2)*3,y + (int)rad/2 - 2);
+	
+	if( (cc._items > 1) )
+	    {
+		g.drawLine(x,y+rad,x-dist,y-rad+dy);
+		g.drawLine(x,y+rad,x+dist,y-rad+dy);
+		if( cc._child_1 != null ) paintTree(cc._child_1, g, x - dist, y + dy, (int)dist/2);
+		if( cc._child_2 != null ) paintTree(cc._child_2, g, x +  dist, y + dy, (int)dist/2);
+	    }
+    }
+    
+    public void paint(Graphics g)
+    {
+       update(g);
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java	(revision 14150)
@@ -0,0 +1,85 @@
+package vishnu.testvis.dendro;
+
+public class TriangleIndex
+{
+	short [] i_ind;    /* from index to i */
+	short [] j_ind;    /* from index to j */
+
+
+
+	TriangleIndex(int matrix_rows)
+	{
+
+  		short i, j;
+        int theIndex;
+
+  		int num_of_elements = matrix_rows * (matrix_rows - 1) / 2;
+
+  		/* Allocate space for lookup tables */
+  		i_ind = new short[num_of_elements];
+  		j_ind = new short [num_of_elements];
+
+	  	for (i = 1; i < matrix_rows; i++)
+    	{
+      		for (j = 0; j < i; j++)
+			{
+	  			theIndex = TriangleIndex.index (i, j);
+				i_ind[theIndex] = i;
+	  			j_ind[theIndex] = j;
+			}
+    	}
+	}
+
+	int row_size (int size)
+	{
+  /* Don't forget that we actually start counting in the second row, since the first one is assumed to be all zeroes */
+  		int i = 1;
+  		int sum = 0;
+
+  		while (sum != size)
+    		sum += i++;
+  		return i;
+	}
+
+	static int index (int i, int j)
+	{
+  /* Assumes lower triangular matrix, i.e. i>j */
+
+  		int temp;
+
+  /* The matrix in this case is symmetric so (i, j) = (j, i) */
+  		if (i < j)
+    	{
+      		temp = i; i = j; j = temp;
+    	}
+
+  		if (i != j)
+    		return (i*i-i)/2 + j;
+    		//(i - 1) * (i - 2) / 2 + i - 1 + j;
+  		else         /* The elements on the diagonal are all 0 */
+    	{
+      		return -1; /* so just return the dummy value -1      */
+    	}
+	}
+	static void print (double[] matrix, int row_size)
+	{
+  		int i, j, theIndex;
+
+  		for (i = 0; i < row_size; i++)
+    	{
+      		for (j = 0; j < row_size; j++)
+			{
+	  			if (i <= j)
+	    			System.out.print(" ");
+	  			else
+	    		{
+	      			theIndex = index (i, j);
+	      			System.out.print(matrix[theIndex] + " ");
+	    		}
+			}
+      		System.out.println();
+    	}
+	}
+
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/evaluate/Logging.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/evaluate/Logging.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/evaluate/Logging.java	(revision 14150)
@@ -0,0 +1,107 @@
+
+/**
+ * Title:        Visualisation<p>
+ * Description:  An implementation of some search result visualisations
+ * <p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      <p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.evaluate;
+import vishnu.testvis.object.*;
+import vishnu.testvis.visual.*;
+import java.io.*;
+import java.util.*;
+import java.text.*;
+
+public class Logging
+{
+    private Vishnu displayFrame;
+    private String _id;
+    String filename;
+    PrintWriter outlog;
+    long startmilli;
+
+    public Logging(String id, String path, Vishnu d)
+    {
+        displayFrame = d;
+        DateFormat longTime = DateFormat.getDateTimeInstance(DateFormat.FULL,DateFormat.FULL);
+        DateFormat shortTime = DateFormat.getDateTimeInstance(DateFormat.SHORT,DateFormat.SHORT);
+        Date todayDate = new Date();
+        String node = replace_chars(
+            replace_chars(id + "_" + shortTime.format(new Date()) + ".xml"," ","_"),
+            "/:","-");
+        if (null==path)filename = node;
+        else filename = path + node;
+        System.out.println(filename);
+        //Properties p = d.getProperties();
+        _id = id;
+        long Time;
+        try
+        {
+            /*outlog =
+                new PrintWriter(
+                    new BufferedWriter(
+                        new FileWriter(filename)));*/
+            File target = new File(filename);
+		    FileWriter fw = new FileWriter(target);
+		    outlog = new PrintWriter(fw,true);
+		
+		    outlog.println("<LOG>");
+            startmilli=System.currentTimeMillis();
+            log("ID",_id);
+            log("DATE_AND_TIME",longTime.format(todayDate));
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    private String replace_chars(String target,String oldch, String newch)
+    {
+        StringTokenizer tokenizer = new StringTokenizer(target,oldch);
+        StringBuffer buff = new StringBuffer();
+
+        while(tokenizer.hasMoreElements())
+        {
+            if(buff.length()>0)
+                buff.append(newch);
+            buff.append(tokenizer.nextToken());
+        }
+        return buff.toString();
+    }
+    
+    public void log(String tag, String value)
+    {
+        String out ="<" + tag + " MILLISECS = \"" +
+            Long.toString(System.currentTimeMillis()-startmilli) + "\">" +  value + "</" + tag + ">";
+        outlog.println(out);
+    }
+    
+
+    public void log(String tag)
+    {
+        String out ="<" + tag + " MILLISECS = \"" +
+            Long.toString(System.currentTimeMillis()-startmilli) + "\"/>";
+        outlog.println(out);
+    }
+
+    public void endsection(String tag)
+    {
+        outlog.println("</"+tag+">");
+    }
+
+    public void startsection(String tag)
+    {
+        outlog.println("<"+tag+">");
+    }
+
+    public void logout()
+    {
+        log("LOGOUT",_id);
+        outlog.println("</LOG>");
+        outlog.close();
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/evaluate/Queries.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/evaluate/Queries.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/evaluate/Queries.java	(revision 14150)
@@ -0,0 +1,338 @@
+package vishnu.testvis.evaluate;
+import java.io.*; 
+import java.util.*;
+
+public class Queries
+{
+    Vector queryID;
+    Vector queryString;
+    Vector relevance[];
+
+    public Queries()
+    {
+        getQueries();
+        
+        // only of the queries just read
+        getRelevanceData();
+        //printRelevance();
+    }
+    
+    private void getRelevanceData()
+    {
+        
+        relevance = new Vector[queryID.size()];
+        for( int i = 0; i < queryID.size(); i++ )
+        {
+            relevance[i] = new Vector();            
+        }
+        
+        try
+        {
+		    FileReader file = new FileReader("qrels.dat");
+		    BufferedReader buff = new BufferedReader(file);
+		    
+		    String line = buff.readLine();
+		    //System.out.println(line);
+		    		    
+		    StringTokenizer st = new StringTokenizer(line);
+		    String id = st.nextToken();
+		    //System.out.println(id);
+		    int index = 0;
+		    if( queryID.contains(id) )
+		    {
+		        index = queryID.indexOf(id);
+		        //System.out.println("Contained at position " + index);
+		    }
+		    
+		    String docStr = st.nextToken();
+		    //System.out.println(docStr);
+		    relevance[index].addElement(docStr);
+		    boolean eof = false;
+		    			
+		    while( !eof ) 
+		    {
+		        line = new String();
+			    line = buff.readLine();	
+			    
+			    if( line == null || line == "" )
+			        eof = true;
+			    else
+			    {
+			        st = new StringTokenizer(line);
+	    	    	id = st.nextToken();
+		            if( queryID.contains(id) )
+		            index = queryID.indexOf(id);
+		            docStr = st.nextToken();
+		            relevance[index].addElement(docStr);
+		            st = new StringTokenizer(line);
+		        }
+	        }
+	        buff.close();	
+		}
+		catch( IOException e)
+		{
+			System.out.println("Error -- " + e.toString());
+		}    
+    }
+    
+    
+    private void getQueries()
+    {
+        queryID = new Vector();
+        queryString = new Vector();
+        
+        try
+		{
+		    FileReader file = new FileReader("queries.dat");
+		    BufferedReader buff = new BufferedReader(file);
+		    
+		    String line = buff.readLine();
+		    //System.out.println(line);
+		    		    
+		    StringTokenizer st = new StringTokenizer(line);
+		    
+		    String keywords = new String();
+		    
+		    while( st.hasMoreTokens() )
+		    {
+		        String strTemp = st.nextToken();
+		        
+		        // check whether token is last token
+		        // assume last token is the query number
+		        if( !st.hasMoreTokens() )
+		        {
+		            queryID.addElement(strTemp);
+		            keywords = keywords.trim();
+		            queryString.addElement(keywords);
+		        }
+		        else
+		        {
+		            keywords += " " + strTemp;
+		        }
+		    }
+		   	    
+            boolean eof = false;
+			
+		    while( !eof )
+		    {
+		        line = new String();
+			    line = buff.readLine();	
+			    //System.out.println(line);
+			    if( line == null || line == "" )
+			        eof = true;
+			    else
+			    {   
+	    		    keywords = new String();
+		    
+		            st = new StringTokenizer(line);
+		            while( st.hasMoreTokens() )
+		            {
+		                String strTemp = st.nextToken();
+		                
+		                // check whether token is last token
+		                // assume last token is the query number
+		                if( !st.hasMoreTokens() )
+		                {
+		                    queryID.addElement(strTemp);
+		                    queryString.addElement(keywords);
+		                    //System.out.println(keywords);
+		                }
+		                else
+		                {
+		                    keywords += " " + strTemp;
+		                }
+		            }
+        		}
+	        }
+	        buff.close();	
+		}
+		catch( IOException e)
+		{
+			System.out.println("Error -- " + e.toString());
+		}
+	}
+        
+    public Object[] getQueryObjectArray()
+    {
+        Object[] objArray = new Object[queryString.size()];
+	    for( int i = 0; i < queryString.size(); i++ )
+		{
+		    objArray[i] = queryString.elementAt(i);
+		}
+		return objArray;
+    }
+        
+    public Vector getQueryIDs()
+    {
+        return queryID;
+    }
+    
+    
+    public void printRelevance()
+    {
+        for( int i = 0; i < relevance.length; i++ )
+        {
+            for( int j = 0; j < relevance[i].size(); j++ )
+            {
+                System.out.print((String)relevance[i].elementAt(j));
+            }
+            System.out.println();
+            System.out.println();
+        }
+    }
+    
+    // returns vector with first element = total of relevant followed
+    // by precision at 10, 20...
+    public Vector getPerformance(String query, String rankedTrecIDs[])
+    {
+        int num = 0;
+        Vector precisionAt = new Vector();
+        int index = queryString.indexOf(query);
+        //System.out.println("Query number " + index + " selected");
+        if( index == -1 )
+            return precisionAt;  // return empty vector
+                
+        boolean relevant[] = new boolean[rankedTrecIDs.length];
+        //System.out.println("Size of ranked list: " + rankedTrecIDs.length);
+        
+        for( int i = 0; i < rankedTrecIDs.length; i++ )
+        {
+            // determine for each doc in the list whether it is relevant
+            if( relevance[index].contains(rankedTrecIDs[i]) )
+            {
+                relevant[i] = true;
+                num++;
+            }
+            else 
+                relevant[i] = false;
+                
+            // precision at 10,20,30,40,50    
+            if( (i+1) % 10 == 0 && (i+1)/10 < 6)
+            {
+                Integer prec = new Integer(Math.round(((float)num/(i+1)*100)));
+                
+                //System.out.println((double)num/(i+1));
+                precisionAt.addElement(prec);
+                
+            }    
+        }
+        
+        precisionAt.addElement(new Integer(relevance[index].size()));
+        precisionAt.addElement(new Integer(num));
+        //System.out.println(num + " relevant docs from a total of " + relevance[index].size() + " have been found");
+        
+        return precisionAt;
+  }
+    
+}
+
+/*		int totalRel = no_in_cat[categories.indexOf(queryId)]-1; // number of relevant
+		int relSoFar = 0;
+		
+		int pre_vs_rec[] = new int[11];
+			
+				
+		// trec method to compute average precision
+		// compute P for relevant images only
+		// P(r0) = max(P(r >= r0))
+
+		
+		// the following array stores pre/rec for every relevant, the first 
+		// entry is precision for the 0th item = 0.
+		// maximum size of array = items.length
+		// actual size of array = totalRel
+		
+		int recall = 0;
+		int precision = 0;
+		
+		
+		// begin of Rueger method
+		
+		relSoFar = totalRel;
+            
+	   	    // i  = 1 because target img is at zero!!!
+	   	    int maxPrecision = 0;
+	   	    int cutL = 10;
+		    for( int i = no_of_items -1; i > 0; i-- )
+			{
+				String listId = new
+			    String((String)cat_of_item.elementAt((int)list[i]));
+				if( listId.equals(queryId) )
+			    {
+			        recall = (int)(100*relSoFar)/totalRel;
+			        
+			        precision = (int)(100*relSoFar)/i;
+			        
+
+			        double cut = (double)(recall)/10;
+			        
+			        //if( r == 6 ) System.out.print(recall + " " + precision + ":");
+			        			        
+                    // if hit the next cut-off cutL, then set cut-off value to whatever 
+                    // is maximum
+			        if( cut == cutL )
+			        {
+			            //System.out.print("eq ");
+			            if( precision > maxPrecision )
+			            {
+			              //  System.out.print("eq-cm(" + precision + ") ");
+			                maxPrecision = precision;
+			            }
+                        
+                        //System.out.print("eq-ap(" + maxPrecision + ") ");
+			            pre_vs_rec[cutL] = maxPrecision;
+			            cutL--;
+			        }
+			        
+			        // if moved to the left of cut-off, the cut-off value (and those
+			        // lying between it and the last one set) is given the 
+			        // current maxPrecision
+			        else if( cut < cutL )   
+			        {
+			            for( int c = (int)cut+1; c < cutL+1; c++ )
+			            {
+			                pre_vs_rec[c] = maxPrecision;
+			            }
+			            
+			            
+			            if( precision > maxPrecision )
+			            {
+			                maxPrecision = precision;
+			            }
+			            
+			            cutL = (int)cut;
+    
+			        }
+		
+		            
+			        
+			        // just in case cut > cutL, we need still to update maxPrecision
+			        if( precision > maxPrecision )
+			        {
+			            maxPrecision = precision;
+			        }
+			        relSoFar--;
+			    
+			    }
+			    // if not relevant, nothing happens
+		    }
+		    
+		    for( int z = 0; z < cutL+1; z++ )
+		    {
+		        pre_vs_rec[z] = maxPrecision;
+		    }
+		    	    
+            // end of Rueger method
+		
+		// average precision for this query                                                     
+		double area = 0;
+		for(int j = 0; j < 10; j++ )
+		{
+			area += (pre_vs_rec[j]+pre_vs_rec[j+1])/2* 0.1;
+		}
+			
+		Performance p = new Performance(area, pre_vs_rec, queryId);
+		return p;
+  
+	}
+*/
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/CacheManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/CacheManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/CacheManager.java	(revision 14150)
@@ -0,0 +1,169 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.*;
+
+
+public class CacheManager
+{
+    
+    int cacheSize;
+    Vector file_names_dat;
+    Vector file_names_doc;
+        
+    public CacheManager()
+    {
+        //getCache();
+    }
+    
+    private void getCache()
+    {
+        Properties p = System.getProperties();
+        String currentDir = p.getProperty("user.dir");
+        File cache_dir = new File(currentDir);
+        file_names_dat = new Vector();
+        file_names_doc = new Vector();
+        
+        String fileArr[] = null;
+        if( cache_dir.exists() )
+        {
+        
+            fileArr = cache_dir.list(new FilenameFilter()
+            {
+                public boolean accept(File dir, String name)
+		        {
+		            return ( name.endsWith(".dat") || name.endsWith(".doc") );
+		        }
+	        });
+	    }
+	    else 
+	    {
+	        System.out.println("Directory does not exist");
+	        System.exit(0);
+	    }
+	    
+	    for( int i = 0; i < fileArr.length; i++ )
+	    {   
+	        if( fileArr[i].endsWith(".dat") )
+	            file_names_dat.addElement(fileArr[i]);
+	        if( fileArr[i].endsWith(".doc") )
+	            file_names_doc.addElement(fileArr[i]);
+	    }	    
+    }
+           
+    public void addDataItem(DataBlock data, String key)
+    {   
+        String fileName = new String(key +".dat");
+        file_names_dat.addElement(fileName);
+        
+        try
+        {
+            FileOutputStream ostream = new FileOutputStream(fileName);
+            ObjectOutputStream out = new ObjectOutputStream(ostream); 
+            out.writeObject(data);
+            ostream.close();                
+        }
+        catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+    }
+    
+     public void addDocItem(Vector doc, String key)
+    {   
+        String fileName = new String(key +".doc");
+        file_names_doc.addElement(fileName);
+        
+        try
+        {
+            FileOutputStream ostream = new FileOutputStream(fileName);
+            ObjectOutputStream out = new ObjectOutputStream(ostream); 
+            out.writeObject(doc);
+            ostream.close();                
+        }
+        catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+    }
+   
+        
+    public boolean containsData(String query)
+    {
+        String path = new String(query+".dat");
+        
+        if( file_names_dat.contains(path) )
+            return true;
+            else
+            return false;
+    }
+    
+    public boolean containsDoc(String docName)
+    {
+        String path = new String(docName+".doc");
+        
+        if( file_names_doc.contains(path) )
+            return true;
+            else
+            return false;
+    }
+  
+    
+    public Vector getDoc(String docName)
+    {
+        String filename = new String(docName + ".doc");
+        Vector doc = null;
+        
+        try
+        {
+            FileInputStream istream = new FileInputStream(filename);
+            ObjectInputStream in = new ObjectInputStream(istream);
+            doc = (Vector)in.readObject();        
+        }
+    	catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+        return doc;   
+    }
+    
+    
+ 
+    public DataBlock getData(String query)
+    {
+        String filename = new String(query + ".dat");
+        DataBlock data = null;
+        
+        try
+        {
+            FileInputStream istream = new FileInputStream(filename);
+            ObjectInputStream in = new ObjectInputStream(istream);
+            data = (DataBlock)in.readObject();        
+        }
+    	catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+        return data;   
+    }
+    
+    public int getSize()
+    {
+        return file_names_dat.size();
+    }
+    
+    public Vector listCache()
+    {
+        return file_names_dat;
+    }
+    
+    public void printMe()
+    {
+        for( int i = 0; i < file_names_dat.size(); i++ )
+        {
+            String object = (String)file_names_dat.elementAt(i);
+            System.out.println(object);
+        }
+    }
+  
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java	(revision 14150)
@@ -0,0 +1,69 @@
+/**
+ * @(#)ClusterGroupObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.object;
+
+public class ClusterGroupObj
+{
+    ClusterObj[] clusters;  //-- list of ClusterObj ref
+    String label;  //-- a general description for the group (take the highest related word)
+    private int groupSize;  //-- total number of docs in the group
+    public ClusterGroupObj(ClusterObj[] c, String ls)
+    {
+        //System.out.println("ClusterGroupObject constructor called label " + ls);
+        //System.out.flush();
+        clusters=c; label=ls;
+        groupSize=0;
+        for (int i=0;i<clusters.length;i++)
+      		groupSize+=clusters[i].getDocsSize();
+	}
+
+
+ public String getLabels()
+ {
+   return label;
+ }
+
+
+  public String getLabels_toString()
+  {
+    return label;
+  }
+/*
+  	String tmp="";
+  	for (int i=0;i<label.length;i++) tmp = tmp + " " + label[i];
+  	return tmp;
+  }
+*/
+  /**
+   * @return the total number of documents in the group
+   */
+  public int getGroupSize(){return groupSize;}
+
+  /**
+   * @return the number of clusters in the group
+   */
+  public int getClustersSize(){return clusters.length;}
+
+
+  /**
+   * @return the clusters list
+   */
+  public ClusterObj[] getClusters(){return clusters;}
+
+
+  public String getInfo(){
+  	return "No of clusters: "+ String.valueOf(clusters.length)+ " totally contains: "+
+  		String.valueOf(getGroupSize())+" docs\nGroup Label: "+getLabels_toString()+"\n";
+  }
+
+  public void showInfo(){
+    System.out.print(getInfo());
+    for (int i = 0; i<getClustersSize(); i++)
+      System.out.print( i+")  "+clusters[i].getInfo());
+  }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/ClusterObj.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/ClusterObj.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/ClusterObj.java	(revision 14150)
@@ -0,0 +1,77 @@
+/**
+ * @(#)ClusterObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.object;
+import vishnu.datablock.*;
+import java.util.Vector;
+
+public class ClusterObj
+{
+    DocObj[] _docs;		//-- list of DocObj ref
+    String _label;		//-- (top 3-5? related words)
+    Point2D [] _sammon;			//-- 2D mapping for the centroids
+    DataManager _data;
+    public ClusterObj(DocObj[] d, String t, Point2D [] sammon, DataManager dm)
+    {
+        _docs=d; _label=t;
+        _sammon = sammon;
+        _data = dm;
+    }
+  /**
+   * @return the number of documents in the cluster
+   */
+  public int getDocsSize(){return _docs.length;}
+
+  public String getLabel(){return _label;}
+
+  public String getLabel1(){return getDocsSize() + " documents found";}
+  public String getLabel2(){return getLabel();}
+
+
+  public String getInfo(){
+    return getDocsSize() + " documents found\n" + getLabel();
+  }
+
+
+  /**
+   * This returns a list of list of documents. The inner list of documents represents
+   * which documents have the corresponding related word
+   * pre: _docs can't be null
+   * @return a vector (same length as the related words) of
+   *         vectors (arbitrary length)
+   *
+   */
+  public Vector getHitDocFreq()
+  {
+    int s = _data.getWordCount();
+    Vector rwVec = new Vector(s);
+    for(int i=0;i<s;i++)
+    {
+      rwVec.addElement(new Vector(0));
+      for(int j=0;j<_docs.length;j++)
+      {
+        if ( _docs[j].getWeights(i) > 0 )
+          ((Vector) rwVec.elementAt(i)).addElement(_docs[j]);
+      }
+    }
+    return rwVec;
+  }
+
+
+  public String formatDocs()
+  {
+    int l = getDocsSize();
+    String s = "<html><font face=Arial size=-1> "+ l +" documents found\n";
+    for(int i=0;i<l;i++)
+    {
+      s += _docs[i].format4Fields();
+    }
+    s += "</font></html>\n";
+    return s;
+  }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/DataManager.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/DataManager.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/DataManager.java	(revision 14150)
@@ -0,0 +1,627 @@
+package vishnu.testvis.object;
+
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.dendro.*;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.io.*;
+import java.net.*;
+
+import vishnu.testvis.sammon.*;
+import vishnu.util.Sammon;
+import vishnu.cluster.Clustering;
+import vishnu.testvis.browser.*;
+
+public class DataManager
+{
+    private Vishnu displayFrame;
+    private DataBlock data = null;
+    private CacheManager cache = null;
+    public Vector collections;
+    public Collection currentCollection = null;
+    public String currentView = null;
+    /** Labels for the clusters needed by tree map,
+     *  generated in this class */
+    private String [] clusterLabels = null;
+    /** vertical array of vectors representing
+     *  the documents that word x appears in */
+    private int [] clustersBigWord;
+    //private Vector [] wordDocuments;
+    /** ClusterObjs as used in Tree map */
+    private DocObj [] docObjs;
+    private ClusterObj [] clusterObjs = null;
+    private ClusterGroupObj [] clusterGroups = null;
+    private int[][] sommClusterWords;
+    private Vector sammonClusters;
+
+    private String engine_type = "Engine";
+    
+    /** Set gsdl state */
+    public void setEngineType(String engine) {
+	engine_type = engine;
+    }
+    /** Generate labels for the clusters needed by tree map,
+     *  pre: data is initialised
+     */
+
+    private void setUpClusterLabels()
+    {
+        int [] wordFreqs = new int [data.words.length];
+        StringBuffer sbuff= new StringBuffer(20);
+        clustersBigWord = new int[data.clusters.length];
+        clusterLabels= new String[data.clusters.length];
+        sommClusterWords= new int [data.clusters.length][4];
+
+        for(int c=0;c<data.clusters.length;c++)
+        {
+            for(int d=0; d < data.words.length; d++) wordFreqs[d]=0;
+
+            for(int d=0; d < data.clusters[c].size(); d++)
+            {
+                Integer n = (Integer) data.clusters[c].elementAt(d);
+                int docNo = n.intValue();
+                SparseNode row = data.matrix.getRow(docNo);
+                while( row!= null)
+                {
+                    wordFreqs[row.getColumn()]++;
+                    row = row.getNext();
+                }
+            }
+
+            int col1 = -1, c1val = 0;
+            int col2 = -1, c2val = 0;
+            int col3 = -1, c3val = 0;
+            int col4 = -1, c4val = 0;
+            for(int d=0; d < data.words.length; d++)
+            {
+                if(c1val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=col2;
+                    c3val=c2val;
+                    col2=col1;
+                    c2val=c1val;
+                    col1=d;
+                    c1val=wordFreqs[d];
+                }
+                else if(c2val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=col2;
+                    c3val=c2val;
+                    col2=d;
+                    c2val=wordFreqs[d];
+                }
+                else if(c3val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=d;
+                    c3val=wordFreqs[d];
+                }
+                else if(c4val < wordFreqs[d])
+                {
+                    col4=d;
+                    c4val=wordFreqs[d];
+                }
+
+            } // for (each document in the cluster)
+           if(col1 > -1)
+           {
+                sbuff.append(data.words[col1]);
+                if(col2 > -1)
+                {
+                    sbuff.append(" ");
+                    sbuff.append(data.words[col2]);
+                    if(col3 > -1)
+                    {
+                        sbuff.append(" ");
+                        sbuff.append(data.words[col3]);
+                    }  // if
+                } // if
+                clusterLabels[c] = new String(sbuff);
+                sbuff.setLength(0);
+            } //if
+            else
+                clusterLabels[c] = new String("<empty>");
+
+            clustersBigWord[c] = col1;
+            sommClusterWords[c][0] = col1;
+            sommClusterWords[c][1] = col2;
+            sommClusterWords[c][2] = col3;
+            sommClusterWords[c][3] = col4;
+        } // for (each cluster)
+    } // end of setUpClusterLabels()
+    
+
+    private void setUpDocObjs()
+    {
+        docObjs = new DocObj[data.docs.length];
+        for(int c=0;c<data.docs.length;c++)
+            docObjs[c] = new DocObj(c, data.descriptions[c], data.matrix.getRow(c), data.docs[c]);
+    }
+    
+    public DocObj getDocObjAt(int index)
+    {
+        return docObjs[index];
+    }
+
+    public int getClusterGroupsSize()
+    {
+        return clusterGroups.length;
+    }
+
+    public ClusterGroupObj getClusterGroup(int i)
+    {
+        return clusterGroups[i];
+    }
+    private void setUpClusterObjs()
+    {
+        clusterObjs = new ClusterObj[data.clusters.length];
+        for(int c=0;c<data.clusters.length;c++)
+        {
+            DocObj [] dObjArray = new DocObj[data.clusters[c].size()];
+            for(int d=0; d<data.clusters[c].size();d++)
+            {
+                Integer n = (Integer)data.clusters[c].elementAt(d);
+                dObjArray[d] = docObjs[n.intValue()];
+            }
+            clusterObjs[c]= new ClusterObj(dObjArray,clusterLabels[c],data.sammon,this);
+        }
+        //ClusterObj(DocObj[] d, String t, Point2D sammon, DataManager dm)
+        // !!!!remember to comment out this loop when it works
+        //for(int c=0;c<data.clusters.length;c++)
+        //    System.out.println("In cluster " + c  + clusterObjs[c].getDocsSize() + " documents ");
+    }
+    /** This for tree map as it needs clusters of clusters
+     * this is a cheap and probably not very satisfactory solution
+     * as all I am doing is taking clusters with the common most significant
+     * word and putting them in groups.
+     */
+    private void setUpClusterGroupObjs()
+    {
+        int count = data.clusters.length;
+        int num=0, i,j;
+        int [] sizes = new int[count];
+        for(i=0;i < count;i++)
+            sizes[i]=1;
+
+        // contains #supercluster clusterList objects, each a list of clusters
+        Vector groupList = new Vector(count);
+        
+        // contains #supercluster BigWords for each supercluster
+        Vector wordList = new Vector(count);
+        for(i=0;i < count;i++)
+        {
+            if(sizes[i]>0)
+            {
+                Vector clusterList = new Vector(count);
+                clusterList.addElement(clusterObjs[i]);
+                int bigWord = clustersBigWord[i];
+                for (j = i+1; j < count; j++)
+                    if (clustersBigWord[j] == bigWord)
+                    {
+                        sizes[i]+=sizes[j];
+                        if(sizes[j]>0)
+                            clusterList.addElement(clusterObjs[j]);
+                        sizes[j]=0;
+                    }
+                groupList.addElement(clusterList);
+                wordList.addElement(new Integer(bigWord));
+            }
+        }
+
+        clusterGroups = new ClusterGroupObj[groupList.size()];
+
+        for(i=0;i < groupList.size();i++)
+        {
+            Vector clusterList = (Vector)groupList.elementAt(i);
+            if(clusterList == null)
+                System.out.println("Cluster list "+ i + " is null");
+            else
+            {
+                //ClusterObj [] clusterObjsArray = clusterList.toArray();
+                int groupSize = clusterList.size();
+                ClusterObj [] clusterObjsArray = new ClusterObj [groupSize];
+                for (j=0; j < groupSize;j++)
+                    clusterObjsArray[j] = (ClusterObj) clusterList.elementAt(j);
+                //System.out.println("Group " + i + " is " + clusterObjsArray.length + " long");
+                Integer bigWord = (Integer)wordList.elementAt(i);
+                int bw = bigWord.intValue();
+                String word;
+                if (bw <0)
+                {
+                    System.out.println("Group " + i + " is " + clusterObjsArray.length + " long");
+                    System.out.println("bigword is empty");
+                    word = "< Empty >";
+                }
+                else
+                    word = data.words[bigWord.intValue()];
+                clusterGroups[i] = new ClusterGroupObj(clusterObjsArray,word);
+            }
+        }
+    }
+    
+    
+    private void printData(String[] str, String filename)
+    {
+        try
+	    {
+	        filename += ".txt";
+            File target = new File(filename);
+		    FileWriter fw = new FileWriter(target);
+		    PrintWriter out = new PrintWriter(fw,true);
+		    String line = new String();
+		   			
+			    for( int j = 0; j < str.length; j++ )
+			    {
+			        line = str[j] + " ";
+			        out.println(j + " " + line);
+			        System.out.println(j + " " + line);
+		        }
+		    
+		    out.close();
+	    } catch (IOException e){ System.out.println("Error " + e.toString());
+	    } catch (SecurityException se) { System.out.println("Error " + 	se.toString());}
+	}
+	
+	public void setUpDendroClusters()
+	{
+	    displayFrame.dendroPanel.treePanel.getData();
+	}
+    
+       
+    
+    public int query(String queryStr)
+    {
+        if( !displayFrame.refine )
+        {
+            if( displayFrame.firstSearch )
+            {
+                displayFrame.firstSearch = false;
+            }
+        }
+                
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return 0;
+	}
+	theEngine.setDisplay(displayFrame);
+	//GSDLEngine theEngine = new GSDLEngine(displayFrame);
+	System.out.println("Searching for " + queryStr);
+	DataBlock db =  theEngine.getData(queryStr, currentCollection.dir, currentView);
+
+        if( db == null ) return 0;
+
+	data = db;
+	
+        int spaceIndex=0;
+        int count = 0;
+        int wordNum = 0;
+        int clusterNum=0;
+        StringBuffer qbuff = new StringBuffer();
+            
+        setUpClusterLabels();
+        setUpDocObjs();
+        setUpClusterObjs();
+        setUpClusterGroupObjs();
+        setUpSammClusters();
+        if( !displayFrame.refine ) setUpDendroClusters();
+        
+	count = this.data.docs.length;
+	wordNum = this.data.words.length;
+	clusterNum = this.data.clusters.length;
+	
+        if( !displayFrame.refine )
+	    {
+		displayFrame.searchStarted = true;
+	    }
+        
+	displayFrame.update();
+        return count;
+    }
+
+
+    public void fetch(int index)
+    {
+        if(data==null) return ;
+        if(index < 0 || index >= data.docs.length) return;
+        int docNum = data.docs[index];
+        fetchDocNo(docNum);
+    }
+
+
+    public void fetchDocNo(long docNum)
+    {
+        System.out.println("Fetch document: " + Long.toString(docNum));
+
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return;
+	}
+	theEngine.setDisplay(displayFrame);
+	//GSDLEngine engine = new GSDLEngine(displayFrame);
+	Vector document = theEngine.getDocument(Long.toString(docNum),currentCollection.dir, currentView);
+	
+	// Fetch theFetch = new Fetch(displayFrame);
+        // document = theFetch.getDocument(docNum);
+        
+	Browser browser;
+	if (document.size() > 1) {
+	    String url = (String)document.elementAt(1);
+	    
+	    if( url.indexOf("http") != -1 )
+		{
+		    try{
+			displayFrame.getAppletContext().showDocument(new URL(url),"_blank");
+			// http://icarus.umkc.edu/cgi-bin/ptext?doc=perseus:text:1999.01.0154"),"_blank");
+		    }catch(IOException ex){ex.printStackTrace();};
+		}     
+	    else browser = new Browser(document,docNum);
+	} else { 
+	    browser = new Browser(document,docNum);
+	}
+    }
+    
+
+    public String [] getClusterLabels()
+    {
+        return clusterLabels;
+    }
+
+
+    public String [] getDescriptions()
+    {
+        if(data != null)
+            return data.descriptions;
+        else return new String[0];
+    }
+    
+    
+    public String [] getTrecIDs()
+    {
+        if(data != null)
+        {
+            String trecID[] = new String[data.descriptions.length];
+            for( int i = 0; i < data.descriptions.length; i++ )
+            {
+                StringTokenizer st = new StringTokenizer(data.descriptions[i]);
+                trecID[i] = st.nextToken();             
+            }
+            return trecID;
+        }
+        else return new String[0];
+    }
+    
+       
+    public String getDescriptionStr(Vector subset)
+    {
+        String returnStr = new String("");
+        for(int c = 0; c < subset.size();c++)
+        {
+            Integer n = (Integer)(subset.elementAt(c));
+            returnStr = returnStr + data.descriptions[n.intValue()] + "\n";
+        }
+        return returnStr;
+    }
+    public int [] getDocs()
+    {
+        if(data != null)
+            return data.docs;
+        else return new int[0];
+    }
+
+    public String [] getWords()
+    {
+        if(data != null)
+        {
+            if(data.words==null)
+            {
+                System.out.println("No words in data!");
+                return new String[0];
+            }
+            return data.words;
+        }
+        else return new String[0];
+    }
+
+    public SparseMatrix getMatrix()
+    {
+        if(data != null)
+            return data.matrix;
+        else return null;
+    }
+
+    public Vector [] getClusters()
+    {
+        if(data != null)
+            return data.clusters;
+        else return null;
+    }
+    public Point2D [] getSammon()
+    {
+        if(data != null)
+            return data.sammon;
+        else return new Point2D[0];
+    }
+
+    public DataManager(Vishnu d) {
+	this(d, null);
+    }
+    public DataManager(Vishnu d, String engine)
+    {
+        displayFrame = d;
+	if (engine != null && !engine.equals("")) {
+	    engine_type = engine;
+	}
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return;
+	}
+
+	theEngine.setDisplay(displayFrame);
+	Vector q = new Vector();
+	q.addElement("initialise");
+	collections = theEngine.getCollections(q); 
+    }
+
+
+    // This constructor creates a smaller data manager from
+    // several clusters from a larger one
+    // it is intended to be used with sammon mapping drilling down
+    // The clustering code is lifted from the C version and could be
+    // improved
+    
+    public DataManager(DataBlock oldData, Vector idVect, Vishnu d)
+    {
+        displayFrame = d;
+        
+        // create new document/keyword matrix with only the 
+        // documents from the selected cluster, but keeping
+        // the same 100 keywords as columns
+        SparseMatrix m = new SparseMatrix(idVect.size(),
+                                          oldData.matrix.getColumnCount());
+
+        for (int c = 0; c< idVect.size();c++)
+        {
+          // the document id of the cth document
+          Integer i = (Integer)idVect.elementAt(c);
+          
+          // ptr to the first document of ith row (linked list)
+          SparseNode row = oldData.matrix.getRow(i.intValue());
+          // add the row to new lean matrix
+          while(row != null)
+          {
+              m.addDataItem(c,row.getColumn(),row.getValue());
+              row=row.getNext();
+          }
+        }
+        
+        float [][] full = m.getExpanded(); 
+        
+        // the number of new clusters generated is either 20
+        // or if there are not many docs half the #docs
+        int clusterNum =  m.getRowCount()>20?10:m.getRowCount()/2;
+        
+        // limit set to 20 if > 20 docs otherwise to 3/4 #docs
+        int limit =  m.getRowCount()>20?20:(m.getRowCount()/2)+
+                      (m.getRowCount()/4);
+                      
+        Clustering clustering = new Clustering(Clustering.COMPLETE_LINKAGE,
+					       full,/*m.getRowCount(),
+						      m.getColumnCount(),*/1,
+					       clusterNum,limit);
+                                                    
+        Vector [] clusters = clustering.getClusters();
+        double [][] centroids= clustering.getCentroids();
+
+        Sammon map = new Sammon(centroids);
+        Point2D [] sammon = map.getMapping();
+        String [] words = oldData.words;
+        int [] newDocs = new int[idVect.size()];
+        String [] newDesc = new String[idVect.size()];
+        for (int c=0;c<idVect.size();c++)
+        {
+          Integer i = (Integer) idVect.elementAt(c);
+          int index = i.intValue();
+          newDocs[c] = oldData.docs[index];
+          newDesc[c] = oldData.descriptions[index];
+        }
+        data = new DataBlock(newDocs,newDesc,words,m,clusters,sammon);
+        
+        setUpClusterLabels();
+        setUpDocObjs();
+        setUpClusterObjs();
+        setUpClusterGroupObjs();
+        setUpSammClusters();
+    }
+
+    public DataBlock getDataBlock()
+    {
+        return data;
+    }
+    
+    public int getWordCount()
+    {
+        if (data!=null)
+            return data.words.length;
+        else return 0;
+    }
+    public int getDocCount()
+    {
+        if (data!=null)
+            return data.docs.length;
+        return 0;
+    }
+
+    public Vector getSammonClusters()
+    {
+        if(data != null)
+            return sammonClusters;
+        else return null;
+    }
+
+    private void setUpSammClusters()
+    {
+        LengthTable [] table = new LengthTable [data.clusters.length];
+        
+        // create as many sammon clusters as there are clusters in the data object
+        // and store it in sammmonClusters
+        sammonClusters = new Vector(data.clusters.length);
+        
+        for (int c = 0; c < data.clusters.length; c++){
+	    table[c]= new LengthTable(c,data.clusters[c].size());
+        }
+        
+        
+        // sort by cluster size, I guess
+        QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+        try
+        {
+            qSort.sort(table);
+        }
+        catch (Exception e)
+        {}
+        
+        
+        for (int c = 0; c < data.clusters.length; c++)
+        {
+            int i = table[c].index;
+             SammCluster cluster = new SammCluster(data.clusters[i],
+                        data.sammon[i], sommClusterWords[i],
+                        clusterObjs[i]);
+            sammonClusters.addElement(cluster);
+        }
+    }
+
+    public void requestCollection()
+    {
+	// should reset the data
+	data = null;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/DocObj.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/DocObj.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/DocObj.java	(revision 14150)
@@ -0,0 +1,89 @@
+/**
+ * @(#)DocObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ * Adapted to use sparce matrix Matthew Carey 7/7/2000
+ */
+
+package vishnu.testvis.object;
+import vishnu.datablock.*;
+import java.util.*;
+public class DocObj
+{
+    private int _number;
+    private int _mgNumber;
+    private String _description;
+//--!!! we don't need to store the whole number, as the actual value is not very important
+//--!!! but we need to keep track the doc contains particular related words or not
+//--!!! therefore, one bit will do
+    private SparseNode _weights, _sparceNode;
+
+
+    public DocObj(int num, String s,SparseNode w, int mgn)
+    {
+        _number = num;
+        _description=s;
+        //_fields=f;
+        _weights=w;
+        _mgNumber = mgn;
+    }
+
+  //public void setWeights(int[] w){_weights=w;}
+    public int getNumber()
+    {
+        return _number;
+    }
+    public String getDescription()
+    {
+        return _description;
+    }
+
+    public SparseNode getWeights()
+    {
+        return _weights;
+    }
+    public float getWeights(int i)
+    {
+        SparseNode test = _weights;
+        while(test != null)
+        {
+            if(test.getColumn() == i) return test.getValue();
+            test = test.getNext();
+        }
+        return 0;
+    }
+    static public String getANumberOfWords(String in, int num)
+    {
+        StringTokenizer st = new StringTokenizer(in);
+        StringBuffer buff = new StringBuffer();
+        for(int c=0;c<num && st.hasMoreTokens() ;c++)
+        {
+            if (c>0) buff.append(" ");
+            buff.append(st.nextToken());
+        }
+        String retString = new String(buff);
+        return retString;
+    }
+    
+    public String getTrecID()
+    {
+        StringTokenizer st = new StringTokenizer(_description);
+        return st.nextToken();             
+    }
+
+    public String format4Fields()
+    {
+        String s = "";
+        s += "<dl>\n";
+        s += "  <dt> <a HREF=\"" + _mgNumber +"\">"+ "Document: " + _mgNumber + "</a>\n";
+        //s += "  <dd> <b>"+ getANumberOfWords(_description,5) +"</b><br> \n";
+	String modDesc = _description.replaceAll("<"," ");
+	modDesc = modDesc.replaceAll(">"," ");
+        s += "    <dd> <font color=gray> " + modDesc + " </font>\n";
+        s += "</dl>\n";
+	System.out.println(s);
+        return s;
+  }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/Engine.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/Engine.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/Engine.java	(revision 14150)
@@ -0,0 +1,118 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+
+public class Engine
+{
+    Vishnu display;
+    String servlet;
+
+    public Engine() {
+    }
+
+    public Engine(Vishnu display)
+    {
+	this.setDisplay(display);
+	
+    }
+    
+    public void setDisplay(Vishnu display) {
+	this.display = display;
+	servlet = display.server_address;
+	System.out.println("servlet= "+servlet);
+
+    }
+    
+    public Vector getCollections(Vector q)
+    {
+	Vector c = null;// = new Vector();
+	try{
+	    URLConnection con = getConnection();
+            System.out.println("Sending request for collections");
+	    sendObject(con,q);
+	    c = (Vector)receiveObject(con);
+	} catch( Exception ex) {System.out.println(ex);}
+	return c;
+    }
+    
+    public DataBlock getData(String query, String collection, String view)
+    {
+        DataBlock data = null;
+        Vector q = new Vector();
+	q.addElement("search");
+        q.addElement(query);
+	q.addElement(collection);
+	q.addElement(view);
+	
+
+        try{
+            URLConnection con = getConnection();
+            sendObject(con,q);
+	    Vector v = (Vector)receiveObject(con);
+	    if( v == null || v.size()==0 ) {
+		System.out.println("Engine.getData: Null or empty result");
+	    } else {
+		data = (DataBlock)v.elementAt(0);
+	    }
+        } catch( Exception ex)  {
+	    System.out.println("Engine.getData: "+ex);
+	}
+	
+	return data;
+    }
+
+    
+    public Vector getDocument(String id, String collection, String view)
+    {
+	Vector q = new Vector();
+	Vector resp = new Vector();
+	q.addElement("fetch");
+	q.addElement(id);
+	q.addElement(collection);
+	q.addElement(view);
+
+	try{
+	    URLConnection con = getConnection();
+	    sendObject(con,q);
+	    resp = (Vector)receiveObject(con);
+	} catch(Exception ex) {ex.printStackTrace();}
+	return (Vector)resp.elementAt(0);
+    }
+
+
+    private URLConnection getConnection() throws IOException
+    {
+        URL u = new URL(servlet);
+        URLConnection con = u.openConnection();
+        con.setDoInput(true);
+        con.setDoOutput(true);
+        con.setUseCaches(false);
+	con.setRequestProperty("Content-type","application/octet-stream");
+        con.setAllowUserInteraction(false);
+        return con;
+    }
+    
+    
+    private void sendObject(URLConnection con, Object obj) throws IOException {
+        ObjectOutputStream out = new  ObjectOutputStream(con.getOutputStream());
+        if( obj != null )
+	    {
+                out.writeObject(obj);
+	    }
+        out.close();
+    }
+    
+    
+    private Object receiveObject(URLConnection con) throws Exception
+    {
+	Object obj;
+	InputStream is = con.getInputStream();
+	ObjectInputStream in = new ObjectInputStream(con.getInputStream() );
+	obj = in.readObject();
+	in.close();
+	return obj;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/GSDLEngine.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/GSDLEngine.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/GSDLEngine.java	(revision 14150)
@@ -0,0 +1,107 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+import vishnu.server.Packaging;
+import vishnu.util.XMLUtil;
+import vishnu.util.Base64;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+
+public class GSDLEngine extends Engine
+{
+    public GSDLEngine() {
+    }
+    
+    public GSDLEngine(Vishnu display)
+    {
+	this.setDisplay(display);
+    }
+        
+    public Vector getCollections(Vector q) {
+	return null;
+    }
+    
+    public DataBlock getData(String query_term, String collection, String view) {
+
+	
+	String query = servlet+"&type=search&q="+query_term;
+	if (view != null) {
+	    query += "&view="+view;
+	}
+	System.out.println("sending query: " + query_term);
+	DataBlock db = null;
+	try {
+	    URL cgi = new URL(query);
+	    DataInputStream in = new DataInputStream(cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    String results = XMLUtil.getNodeText(data_elem);
+	    
+	    // unserialize the data block
+	    db = (DataBlock) Base64.decodeToObject(results);
+	    //byte [] bytes = results.getBytes("UTF-8");
+// 	    byte [] bytes = Base64.decode(results);
+// 	    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+// 	    ObjectInputStream ois = new ObjectInputStream(bais);
+// 	    db = (DataBlock)ois.readObject();
+//	    ois.close();
+// 	    Element cluster_results = (Element)XMLUtil.getChildByTagName(data_elem, "cluster");
+// 	    String results = XMLUtil.getNodeText(cluster_results);
+// 	    //System.out.println("recived results:" + results);
+// 	    Element description_results = (Element)XMLUtil.getChildByTagName(data_elem, "descriptions");
+// 	    NodeList descs = description_results.getElementsByTagName("desc");
+// 	    Vector desc_vector = new Vector();
+// 	    for (int i=0; i<descs.getLength(); i++) {
+// 		String d = XMLUtil.getNodeText((Element)descs.item(i));
+// 		desc_vector.add(d);
+// 	    }
+// 	    Packaging p = new Packaging();
+// 	    db = p.getDataPackage(results, desc_vector);
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to cgi: " + e);
+	    e.printStackTrace();
+	    return null;
+	}
+	return db;
+
+    }
+
+    public Vector getDocument(String id, String collection, String view) {
+	String query = servlet+"&type=fetch&d="+id;
+	if (view != null) {
+	    query += "&view="+view;
+	}	
+	System.out.println("sending fetch: " + id);
+	Vector doc;
+	try {
+	    URL cgi = new URL(query);
+	    DataInputStream in = new DataInputStream(cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    String results = XMLUtil.getNodeText(data_elem);
+	    System.out.println("recived results:" + results);
+	    // ugly hack. make this nicer!!
+	    doc = new Vector(Arrays.asList(results.split("\n")));
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to cgi: " + e);
+	    e.printStackTrace();
+	    return null;
+	}
+	return doc;
+		
+    }
+    
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/LengthTable.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/LengthTable.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/object/LengthTable.java	(revision 14150)
@@ -0,0 +1,21 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.object;
+
+public class LengthTable
+{
+    public int index;
+    public int length;
+    public LengthTable(int ind, int len)
+    {
+        index= ind;
+        length = len;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java	(revision 14150)
@@ -0,0 +1,450 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import java.util.Enumeration;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.awt.image.*;
+
+class ClusterPanel extends JPanel
+{
+    Image offScreen=null;
+    Dimension offScreenSize;
+    Graphics2D offGraphics;
+    private SammCluster currentCluster;
+    private final int RADIUS = 30;
+    private SammCluster drawnCluster;
+    private GraphicalPane parentSammon;
+
+    boolean firstTime = true;
+    Graphics2D big;
+    BufferedImage bi;
+    Rectangle area;
+    Dimension d = null;
+    
+    int locationID = -1;
+    JPopupMenu[] popups;
+    JPopupMenu popup;
+    Vishnu _currentMain;
+    DataManager dm; 
+
+    public ClusterPanel(Vishnu currentMain, GraphicalPane parent)
+    {
+        this.parentSammon = parent;
+        _currentMain=currentMain;
+	dm = _currentMain.dataManager;
+        popup = new JPopupMenu();
+        popup.setVisible(false);
+        add(popup);
+
+        setOpaque(true);
+        setPreferredSize(new Dimension(600,600));
+        setBorder(BorderFactory.createCompoundBorder(
+            BorderFactory.createEmptyBorder(20,20,20,20),
+            BorderFactory.createLoweredBevelBorder()));
+            
+        addComponentListener(
+            new ComponentAdapter()
+        {
+            public void componentResized(ComponentEvent evt)
+            {
+                if( parentSammon.dataMan != null && parentSammon.dataMan.getDocCount()>0){
+		    repaint();
+                }
+
+            }
+        }
+        );
+
+        addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent mouseEvent)
+            {
+                int docsInSelectedClusters=0;
+                
+                if( parentSammon.dataMan == null) return;
+                int oldLocationID = locationID;
+                
+                Vector clusters = parentSammon.dataMan.getSammonClusters();
+                if (clusters == null) return;
+                locationID = -1;
+                for (int i = clusters.size()-1 ; i >= 0 ; i--)
+                {
+
+                    SammCluster cluster = (SammCluster) clusters.elementAt(i);
+                    int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+                    int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) +RADIUS;
+
+                    if (inCircle(mouseEvent,x,y,cluster.getDisplayedRadius()))
+                    {
+                        locationID = i;
+                    }
+                    if(cluster.getSelected())
+                      docsInSelectedClusters+=cluster.documentIDs.size();
+                }
+
+                if ((oldLocationID != locationID) && (oldLocationID != -1))
+                    popup.setVisible(false);
+
+
+                if (locationID != -1)
+                {
+		    if (locationID != oldLocationID)
+                    {
+               
+                        popup.removeAll();
+                             
+                        String[] allWords = parentSammon.dataMan.getWords();
+                        
+                        currentCluster = (SammCluster) clusters.elementAt(locationID);
+                        String str =  currentCluster.getSize() + " documents";
+                        JMenuItem menuItem = new JMenuItem(str);
+                        popup.add(menuItem);
+
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent actionEvent)
+                            {
+                                drawnCluster = currentCluster;
+                                updateHLinkPanel();
+                                repaint();
+				//update
+                            }
+                        });
+
+                        popup.add(new JPopupMenu.Separator());
+                        menuItem = new JMenuItem("Drill down");
+
+                        if(!currentCluster.getSelected())
+                          docsInSelectedClusters+=currentCluster.documentIDs.size();
+                                                   
+                        if(docsInSelectedClusters<7)menuItem.setEnabled(false);
+                            else menuItem.setEnabled(true);
+                        
+                        // for drilldown
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent ActionEvent)
+                            {
+                                Vector idVector;
+                                SammCluster thisCluster = getCurrentCluster();
+                                                                
+                                int theIndex = thisCluster.getWords()[0];
+                                String drillWords = new String(parentSammon.dataMan.getWords()[theIndex]);
+                                for( int i = 1; i < thisCluster.getWords().length; i++ )
+                                {
+                                    theIndex = thisCluster.getWords()[i];
+                                    drillWords += " " + parentSammon.dataMan.getWords()[theIndex];
+                                }
+				
+				Vector clusterPl = parentSammon.dataMan.getSammonClusters();
+
+                                Vector idVect = (Vector)thisCluster.getIDs().clone();
+                                
+                                thisCluster.setSelected(true);
+                                for (int i = clusterPl.size()-1; i >= 0; i--)
+                                {
+
+                                  SammCluster cluster = (SammCluster) clusterPl.elementAt(i);
+                                  if (!cluster.equals(thisCluster) && cluster.getSelected())
+                                    idVect.addAll(cluster.getIDs());
+                                }
+                                if(idVect.size()>6)
+                                {
+                                
+                                  parentSammon.drillDown(idVect);
+                                  
+                                  currentCluster=null;
+                                  drawnCluster=null;
+				  //update();
+				  repaint();
+                                }
+                                
+                            }
+                        });
+                        popup.add(menuItem);
+                        if(!currentCluster.getSelected())
+                          menuItem = new JMenuItem("Select");
+                        else
+                          menuItem = new JMenuItem("Deselect");
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent ActionEvent)
+                            {
+                                SammCluster cluster = getCurrentCluster();
+                                cluster.setSelected(!cluster.getSelected());
+                                //update();
+				repaint();
+                                
+                            }
+                        });
+
+                        popup.add(menuItem);
+                        int[] words = currentCluster.getWords();
+                        popup.add(new JPopupMenu.Separator());
+                        for(int c=0;c<4;c++)
+                        {
+                          int index = words[c];
+                          if (index>= 0)menuItem = new JMenuItem(allWords[index]);
+                          popup.add(menuItem);
+                          menuItem.setEnabled(false);
+                        }
+
+                        popup.validate();
+                        popup.setVisible(true);
+			
+                    }
+                    popup.show(mouseEvent.getComponent(), mouseEvent.getX(), mouseEvent.getY());
+                    mouseEvent.consume();
+                }
+		
+            }
+	    });
+	
+        setVisible(true);
+    }
+
+    
+    public SammCluster getCurrentCluster() {return currentCluster;}
+    
+    
+    public void paint(Graphics g)
+    {
+	update(g);
+    }
+    
+
+    public boolean inCircle(MouseEvent mouseEvent, int x, int y, int radius)
+    {
+        int dx2 = (mouseEvent.getX()-x)*(mouseEvent.getX()-x);
+        int dy2 = (mouseEvent.getY()-y)*(mouseEvent.getY()-y);
+
+        return (dx2 + dy2 < radius*radius);
+    }
+    
+    
+    private void drawFlag(Graphics g,int x, int y)
+    {
+	g.drawLine(x,y,x,y-12);
+	int [] xs = new int[3];
+	int [] ys = new int[3];
+	xs[0]=x;xs[1]= x+10; xs[2]=x;
+	ys [0]=y-12; ys[1]=y-7;ys[2]=y-2;
+	g.fillPolygon(xs,ys,3);
+    }
+
+    
+    public void update(Graphics g)
+    {
+	Graphics2D g2 = (Graphics2D)g;
+	Dimension d = getSize();
+	offScreenSize = d;
+	
+	int w = d.width;
+	int h = d.height;
+	
+	area = new Rectangle(d);
+	bi = (BufferedImage)createImage(w, h);
+	offGraphics = bi.createGraphics();
+	area.setLocation(w/2-50, h/2-25);
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	
+	offGraphics.setColor(Color.white);
+	/*offGraphics.clearRect(0, 0, area.width, area.height);
+	offGraphics.draw(area);
+	offGraphics.fill(area);*/
+	offGraphics.fillRect(0,0,d.width,d.height);
+	
+        if( parentSammon.dataMan.getDocCount()<1) return;
+	
+        Vector clusters = parentSammon.dataMan.getSammonClusters();
+	if (clusters.size()==0) return;
+	double minNumDocs = ((SammCluster) clusters.elementAt(0)).getSize();
+        double maxNumDocs = ((SammCluster) clusters.elementAt(clusters.size()-1)).getSize();
+        double deltaNumDocs = maxNumDocs - minNumDocs;
+	
+        for (int i = clusters.size()-1; i >= 0; i--) {
+	    
+            SammCluster cluster = (SammCluster) clusters.elementAt(i);
+            int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+            int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+	    
+            // Value between 0 and 1 representing the cluster (0 is the smaller cluster and 1 is the bigger one
+            double clusterWeight = (cluster.getSize() - minNumDocs)/ deltaNumDocs;
+
+            offGraphics.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	    
+            cluster.setDisplayedRadius(RADIUS-(int)(RADIUS*(1-clusterWeight)/2));
+	    myStroke = new BasicStroke(3,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+            offGraphics.setStroke(myStroke);	        
+            offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            offGraphics.setColor(new Color((float)0,(float)0, (float)0.7));
+            offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            if (cluster.getSelected())
+		drawFlag(offGraphics,x,y-cluster.getDisplayedRadius());
+	    
+            if (cluster.equals(this.drawnCluster))
+		{
+		    
+		    offGraphics.setColor(new Color((float)0.3, (float)0.3, (float)1));
+		    offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		    
+		    offGraphics.setColor(new Color((float)0, (float)0, (float)0.7));  
+		    offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		}
+	    
+        }
+	
+        int i = clusters.size();
+        Font normal = new Font("Courier",Font.PLAIN,16);
+        Font bold = new Font("Courier",Font.BOLD,16);
+        for (Enumeration enumer = clusters.elements() ; enumer.hasMoreElements();)
+	    {
+		
+		SammCluster cluster = (SammCluster) enumer.nextElement();
+		int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+		int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+		offGraphics.setFont(bold);
+		offGraphics.setColor(new Color((float)0, (float)0,(float)0.4));
+		
+		FontMetrics fm = offGraphics.getFontMetrics();
+		int theIndex = cluster.getWords()[0];
+		
+		String theWord = parentSammon.dataMan.getWords()[theIndex];
+		
+		int width = fm.stringWidth(theWord);
+		offGraphics.drawString(theWord,x-(width/2),y+5);
+		i--;
+	    }
+	
+	g2.drawImage(bi,0,0,this);
+    }
+    
+    
+    public void update()
+    {
+	/*        Dimension d = getSize();
+        offScreenSize = d;
+        offScreen = createImage(d.width,d.height);
+        offGraphics = (Graphics2D)offScreen.getGraphics();
+	
+        offGraphics.setColor(Color.white);
+        offGraphics.fillRect(0,0,d.width,d.height);
+	
+        if( parentSammon.dataMan.getDocCount()<1) return;
+	
+        Vector clusters = parentSammon.dataMan.getSammonClusters();
+	
+	double minNumDocs = ((SammCluster) clusters.elementAt(0)).getSize();
+        double maxNumDocs = ((SammCluster) clusters.elementAt(clusters.size()-1)).getSize();
+        double deltaNumDocs = maxNumDocs - minNumDocs;
+	
+        for (int i = clusters.size()-1; i >= 0; i--) {
+	    
+            SammCluster cluster = (SammCluster) clusters.elementAt(i);
+            int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+            int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+	    
+            // Value between 0 and 1 representing the cluster (0 is the smaller cluster and 1 is the bigger one
+            double clusterWeight = (cluster.getSize() - minNumDocs)/ deltaNumDocs;
+
+            //offGraphics.setColor(new Color((float) (1 - 0.7*clusterWeight),(float) 0, (float) 0));
+            offGraphics.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	    
+            cluster.setDisplayedRadius(RADIUS-(int)(RADIUS*(1-clusterWeight)/2));
+            BasicStroke myStroke = new BasicStroke(3,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+            offGraphics.setStroke(myStroke);	        
+            offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            offGraphics.setColor(new Color((float)0,(float)0, (float)0.7));
+            offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            if (cluster.getSelected())
+		drawFlag(offGraphics,x,y-cluster.getDisplayedRadius());
+	    
+            if (cluster.equals(this.drawnCluster))
+		{
+		    
+		    offGraphics.setColor(new Color((float)0.3, (float)0.3, (float)1));
+		    offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		    
+		    offGraphics.setColor(new Color((float)0, (float)0, (float)0.7));  
+		    offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		}
+	    
+        }
+	
+        int i = clusters.size();
+        Font normal = new Font("Courier",Font.PLAIN,16);
+        Font bold = new Font("Courier",Font.BOLD,16);
+        for (Enumeration enumer = clusters.elements() ; enumer.hasMoreElements();)
+	    {
+		
+		SammCluster cluster = (SammCluster) enumer.nextElement();
+		int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+		int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+		offGraphics.setFont(bold);
+		offGraphics.setColor(new Color((float)0, (float)0,(float)0.4));
+		
+		FontMetrics fm = offGraphics.getFontMetrics();
+		int theIndex = cluster.getWords()[0];
+		
+		String theWord = parentSammon.dataMan.getWords()[theIndex];
+		
+		int width = fm.stringWidth(theWord);
+		offGraphics.drawString(theWord,x-(width/2),y+5);
+		i--;
+		}*/
+        repaint();
+    }
+
+
+    private void updateHLinkPanel()
+    {
+        if( parentSammon.dataMan==null) return;
+        
+        if (locationID != -1)
+	    {
+		
+		Vector clusters = parentSammon.dataMan.getSammonClusters();
+		
+		SammCluster selectedCluster = (SammCluster) clusters.elementAt(locationID);
+		ClusterObj cObj = selectedCluster.getClusterObj();
+		String htmlDescription = cObj.formatDocs();
+		
+		parentSammon.hLinkPanel.setText(htmlDescription);
+		parentSammon.keywordPanel.update(cObj);
+	    }
+    }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java	(revision 14150)
@@ -0,0 +1,148 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+
+/** Panel containing the sammon map interface */
+public class GraphicalPane extends JPanel
+{
+    public JPanel picturePanel;
+    public ClusterPanel clusterPanel;
+    public SammKeywordPanel keywordPanel;
+    public HLinkPanel hLinkPanel;
+    public Stack dataStack;
+    
+    public DataManager dataMan=null;
+    public Vishnu currentMain;
+    public JLabel pictureLabel = new JLabel("Level 0");
+    public JButton pictureButton = new JButton("Back");
+    public JPanel topPanel = new JPanel();
+    private int largeFont = 16;
+    private int smallFont = 14;
+    private int level = 0;
+    
+    public GraphicalPane(Vishnu currentMain)
+    {
+	this.currentMain = currentMain;
+	dataMan = currentMain.dataManager;
+	
+	dataStack=new Stack();
+	level=0;
+	setName("Sammon Map");
+	setLayout(new BorderLayout(10,10));
+	setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+	clusterPanel = new ClusterPanel(currentMain,this);
+	keywordPanel = new SammKeywordPanel(currentMain,this);
+	Dimension d = currentMain.getSize();
+	hLinkPanel = new HLinkPanel((int)(d.width)/4,d.height,currentMain);
+	picturePanel = new JPanel();
+	picturePanel.setLayout(new BorderLayout(0,3));
+	topPanel.setLayout(new FlowLayout(FlowLayout.LEFT,10,0));
+	topPanel.setBorder(BorderFactory.createEmptyBorder(0,0,0,0));
+	pictureLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	pictureButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	topPanel.add(pictureLabel);
+	topPanel.add(pictureButton);
+	pictureButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    unDrill();
+		}
+	    });
+	
+	pictureButton.setEnabled(false);
+	picturePanel.add(topPanel,"North");
+
+	picturePanel.add(clusterPanel,"Center");
+	add("East", hLinkPanel.linkPane);
+	add("Center", picturePanel);
+	add("West", keywordPanel);
+	setVisible(true);
+    }
+    
+    public void drillDown(Vector idVect)
+    {
+	System.out.println("Drilling");
+
+	// save old data manager and create new one with old data
+	// and vector of document ids of the selected cluster
+	dataStack.push(dataMan);
+	dataMan = new DataManager(dataMan.getDataBlock(), idVect, currentMain);
+	level++;
+	pictureButton.setEnabled(true);
+	keywordPanel.clear();
+	hLinkPanel.clear();
+	pictureLabel.setText("Level "+ level);
+    }
+    
+    public void unDrill()
+    {
+	if(!dataStack.empty()){
+		dataMan = (DataManager)dataStack.pop();
+		hLinkPanel.clear();
+		keywordPanel.clear();
+		level--;
+		clusterPanel.repaint();
+		pictureLabel.setText("Level "+ level);
+	    }
+	else level = 0;
+	if(level==0)
+	    pictureButton.setEnabled(false);
+    }
+  
+    public void update()
+    {
+	dataMan = currentMain.dataManager;
+	dataStack=new Stack(); // dispose of old stack - create new empty one
+	level=0;
+	clusterPanel.repaint();
+	keywordPanel.clear();
+	hLinkPanel.clear();
+    }
+  
+    
+    public void addNotify()
+    {
+        Dimension d = getSize();
+	//Record the size of the window prior to calling parents addNotify.
+        super.addNotify();
+	
+        if (fComponentsAdjusted)
+	    return;
+	
+	// Adjust components according to the insets
+        setSize(getInsets().left + getInsets().right + d.width,
+		getInsets().top + getInsets().bottom + d.height);
+        Component components[] = getComponents();
+        for (int i = 0; i < components.length; i++)
+	    {
+		Point p = components[i].getLocation();
+		p.translate(getInsets().left, getInsets().top);
+		components[i].setLocation(p);
+	    }
+        fComponentsAdjusted = true;
+    }
+    
+    /** Used for addNotify check. */
+    protected boolean fComponentsAdjusted = false;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java	(revision 14150)
@@ -0,0 +1,80 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import vishnu.testvis.visual.*;
+
+class HLinkPanel extends JEditorPane {
+
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+
+    public int numDisplayedDocs = 15;
+
+    public HLinkPanel(int width,int length, Vishnu currentMain) {
+
+        linkPane = new JScrollPane(this);
+        this.currentMain=currentMain;
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener{
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            //System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            //System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+            {
+
+                String hpresult = he.getDescription();
+                int index = Integer.parseInt(hpresult);
+                System.out.println("hyperlink result: " + index);
+                if (index > 0)
+                    currentMain.dataManager.fetchDocNo(index);
+            }
+        }
+    }
+
+    /*public void update()
+    {
+        int count = currentMain.dataManager.getDocCount();
+        Vector subSet = new Vector(count);
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+            subSet.addElement(new Integer(i));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDocumentDescription(subSet);
+
+        setText(htmlDescription);
+
+    }*/
+
+    public void clear()
+    {
+        setText("");
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java	(revision 14150)
@@ -0,0 +1,142 @@
+/*
+ * @(#)QSortAlgorithm.java        1.6 96/12/06
+ *
+ * Copyright (c) 1994-1996 Sun Microsystems, Inc. All Rights Reserved.
+ *
+ * Sun grants you ("Licensee") a non-exclusive, royalty free, license to use,
+ * modify and redistribute this software in source and binary code form,
+ * provided that i) this copyright notice and license appear on all copies of
+ * the software; and ii) Licensee does not utilize the software in a manner
+ * which is disparaging to Sun.
+ *
+
+ * This software is provided "AS IS," without a warranty of any kind. ALL
+ * EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, INCLUDING ANY
+ * IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE OR
+ * NON-INFRINGEMENT, ARE HEREBY EXCLUDED. SUN AND ITS LICENSORS SHALL NOT BE
+ * LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING
+ * OR DISTRIBUTING THE SOFTWARE OR ITS DERIVATIVES. IN NO EVENT WILL SUN OR ITS
+ * LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT OR DATA, OR FOR DIRECT,
+
+ * INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER
+ * CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF THE USE OF
+ * OR INABILITY TO USE SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ *
+ * This software is not designed or intended for use in on-line control of
+ * aircraft, air traffic, aircraft navigation or aircraft communications; or in
+ * the design, construction, operation or maintenance of any nuclear
+
+ * facility. Licensee represents and warrants that it will not use or
+ * redistribute the Software for such purposes.
+ */
+
+/**
+ * A quick sort demonstration algorithm
+ * SortAlgorithm.java
+ *
+ * @author James Gosling
+ * @author Kevin A. Smith
+ * @version         @(#)QSortAlgorithm.java        1.3, 29 Feb 1996
+ */
+
+package vishnu.testvis.sammon;
+
+import vishnu.testvis.object.*;
+
+public class QSortAlgorithmTableLength
+{
+
+    /**
+     * A version of pause() that makes it easier to ensure that we pause
+     * exactly the right number of times.
+
+     */
+    private boolean pauseTrue(int lo, int hi) throws Exception {
+        return true;
+    }
+
+   /** This is a generic version of C.A.R Hoare's Quick Sort
+    * algorithm.  This will handle arrays that are already
+    * sorted, and arrays with duplicate keys.<BR>
+    *
+    * If you think of a one dimensional array as going from
+    * the lowest index on the left to the highest index on the right
+    * then the parameters to this function are lowest index or
+
+    * left and highest index or right.  The first time you call
+    * this function it will be with the parameters 0, a.length - 1.
+    *
+    * @param a       an integer array
+    * @param lo0     left boundary of array partition
+    * @param hi0     right boundary of array partition
+    */
+   void QuickSort(LengthTable a[], int lo0, int hi0) throws Exception
+   {
+      int lo = lo0;
+      int hi = hi0;
+      int mid;
+
+      if ( hi0 > lo0)
+      {
+
+         /* Arbitrarily establishing partition element as the midpoint of
+
+          * the array.
+          */
+         mid = a[ ( lo0 + hi0 ) / 2 ].length;
+
+         // loop through the array until indices cross
+         while( lo <= hi )
+         {
+            /* find the first element that is greater than or equal to
+             * the partition element starting from the left Index.
+             */
+             while( ( lo < hi0 ) && pauseTrue(lo0, hi0) && ( a[lo].length < mid ))
+                 ++lo;
+
+            /* find an element that is smaller than or equal to
+
+             * the partition element starting from the right Index.
+             */
+             while( ( hi > lo0 ) && pauseTrue(lo0, hi0) && ( a[hi].length > mid ))
+                 --hi;
+
+            // if the indexes have not crossed, swap
+            if( lo <= hi )
+            {
+               swap(a, lo, hi);
+               ++lo;
+               --hi;
+            }
+         }
+
+         /* If the right index has not reached the left side of array
+          * must now sort the left partition.
+
+          */
+         if( lo0 < hi )
+            QuickSort( a, lo0, hi );
+
+         /* If the left index has not reached the right side of array
+          * must now sort the right partition.
+          */
+         if( lo < hi0 )
+            QuickSort( a, lo, hi0 );
+
+      }
+   }
+
+   private void swap(LengthTable a[], int i, int j)
+   {
+      LengthTable T;
+      T = a[i];
+      a[i] = a[j];
+      a[j] = T;
+   }
+
+   public void sort(LengthTable a[]) throws Exception
+   {
+      QuickSort(a, 0, a.length - 1);
+   }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/SammCluster.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/SammCluster.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/SammCluster.java	(revision 14150)
@@ -0,0 +1,86 @@
+package vishnu.testvis.sammon;
+
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.object.ClusterObj;
+
+public class SammCluster {
+
+    // The co-ordinates of the centroid in the keyword space
+    //public Vector centroid  = null;
+
+    Point2D sammon;
+
+    // The documents ID contained in this cluster
+    public Vector documentIDs = new Vector();
+    public int[] sigWords;
+    private boolean selected =false;
+    private ClusterObj cObj;
+    public int getSize() { return documentIDs.size();}
+
+    // When looking for a particular cluster in a vector
+    /*public boolean equals(Object obj) {
+
+      System.out.println("Equals " + obj.getClass());
+
+        if (obj instanceof MyInteger)
+        return (documentIDs.size() == ((MyInteger) obj).value);
+
+          return super.equals(obj);
+
+            }
+    */
+    // Co-ordinates of the cluster centroid in the 2 dimension space
+    private double x;
+
+    private double y;
+
+    // Displayed size of the cluster
+    private int displayedRadius;
+
+    public void setDisplayedRadius(int value) {displayedRadius = value;}
+
+    public int getDisplayedRadius() {return displayedRadius;}
+
+
+    //private int numDocs;
+
+    // TEMPORARY: For random generator
+
+    public SammCluster(Vector theDocs, Point2D sammon,
+          int [] sigWords, ClusterObj c)
+    {
+        documentIDs = theDocs;
+        this.sammon = sammon;
+        this.sigWords=sigWords;
+        this.cObj=c;
+    }
+
+    public boolean getSelected() {return selected;}
+    public void setSelected(boolean state){selected=state;}
+
+    double getX() {return sammon.x;}
+
+    double getY() {return sammon.y;}
+
+    public ClusterObj getClusterObj(){ return cObj;}
+
+    //int getNumDocs() {return numDocs;}
+
+    public Vector getIDs() {return documentIDs;}
+
+    void addDocumentID(Integer docID) {
+        documentIDs.addElement(docID);
+    }
+
+   // void setClusterCentroid(Vector centroid) {
+   //     this.centroid = centroid;
+
+    int [] getWords() { return sigWords;}
+
+    void dump()
+    {
+        //System.out.println("Centroid : " + centroid.toString());
+        //System.out.println("Doc IDs: " + documentIDs.toString());
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java	(revision 14150)
@@ -0,0 +1,148 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+import vishnu.testvis.treemap.table.*;
+
+class SammKeywordPanel extends JPanel
+{
+    RW_DataModel _dataModel;
+    //SammRWTable _dataModel;
+    JTable _table = new JTable();
+    JScrollPane theScrollPane;
+    JCheckBox checkBoxes[];
+    JLabel labels[];
+    Vishnu currentMain;
+    GraphicalPane parentPane;
+
+    public SammKeywordPanel(Vishnu currentMain, GraphicalPane parent)
+    {
+        this.currentMain = currentMain;
+        this.parentPane = parent;
+        
+        theScrollPane = new JScrollPane();
+
+         //setBackground(Color.white);
+        setPreferredSize(new Dimension(200,900));
+        setBorder(BorderFactory.createLoweredBevelBorder());
+        setLayout(new BorderLayout(10,0));
+        Font theFont = new java.awt.Font("Monospaced", 0, 12);
+
+        //JLabel title = new JLabel("Related keywords");
+        //title.setFont(theFont);
+        //add(title,"North");
+
+        _dataModel = new RW_DataModel();
+        //_dataModel = new SammRWTable();
+        _table.setModel(_dataModel);
+
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(100);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(50);
+        _table.setRowSelectionAllowed(false);
+        setupRenderer();
+
+        theScrollPane.getViewport().add(_table, null);
+        add(theScrollPane,"Center");
+
+        setVisible(true);
+
+        JButton refineButton = new JButton("Filter Documents");
+        refineButton.setBorder(BorderFactory.createRaisedBevelBorder());
+        refineButton.setMaximumSize(new Dimension(30,70));
+
+        MouseListener ml = new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+                // what does this do Shalini?
+                // Call the refine engine
+                refineEngine();
+            }
+        };
+
+        refineButton.addMouseListener(ml);
+        refineButton.setFont(theFont);
+        add(refineButton,"South");
+
+
+    }
+    public void refineEngine()
+    {
+       currentMain.log.startsection("FILTERING");
+       
+       parentPane.hLinkPanel.setText(_dataModel.getRefinedDocs());
+       String trec_id[] = _dataModel.getRefinedTrecIDs();
+       String filter_Words[] = _dataModel.getFilterWords();
+       String s = new String("");
+       for( int i = 0; i < filter_Words.length; i++ )
+       {
+            s += filter_Words[i] + " ";
+       }
+      
+       currentMain.log.log("FILTER_WORDS",s);
+       currentMain.log.log("FILTERED_DOCS",Integer.toString(trec_id.length));
+       
+       Vector performance = currentMain.queries.getPerformance(currentMain.query, trec_id);
+       
+       currentMain.log.startsection("PERFORMANCE");
+
+       if( performance.size() > 0 )
+       {
+            Integer relvTotal = (Integer)performance.elementAt(performance.size()-2);
+            Integer relvRetr = (Integer)performance.elementAt(performance.size()-1);
+            currentMain.log.log("RELEVANT_TOTAL",Integer.toString(relvTotal.intValue()));
+            currentMain.log.log("RELEVANT_RETRIEVED",Integer.toString(relvRetr.intValue()));
+            for( int i = 0; i < performance.size()-2; i++ )
+            {
+                Integer perf = (Integer)performance.elementAt(i);
+                String str = "PRECISION_" + Integer.toString((i+1)*10);
+                currentMain.log.log(str,Integer.toString(perf.intValue()));
+            }
+       }
+       else
+       {
+            currentMain.log.log("INFO","No relevance data available");
+       }
+     
+                   
+       currentMain.log.endsection("PERFORMANCE");
+       currentMain.log.endsection("FILTERING");
+    }
+        
+    // have dendroclusterobj/sammonclusterobj etc as subclasses
+    // of some abstract clusterObj superclass with some 
+    // core fields (docs, hitdocFreq etc)
+    public void update(ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+    
+    public void update2(vishnu.testvis.dendro.ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+
+    public void clear()
+    {
+        _dataModel.clear();
+    }
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java	(revision 14150)
@@ -0,0 +1,183 @@
+/**
+ * @(#)ClusterGroupRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+package vishnu.testvis.treemap.rectangle;
+
+import vishnu.testvis.object.*;
+import java.awt.*;
+import java.awt.geom.AffineTransform;
+
+public class ClusterGroupRect extends ZRect
+{
+  static final private Color __COLOUR=Color.black;
+  private ClusterGroupObj _data;
+
+  public ClusterGroupRect(float x1, float y1, float x2, float y2,
+    Color c, ClusterGroupObj o)
+  {
+    super(x1,y1,x2,y2,c, o.getGroupSize());
+    _data = o;
+  }
+
+
+  public ClusterGroupRect(float x1, float y1, float x2, float y2,ClusterGroupObj o)
+  {
+      super(x1,y1,x2,y2,__COLOUR, o.getGroupSize());
+      _data = o;
+  }
+
+
+
+  public void paint(Graphics oldg)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    centerText2(g,__COLOUR);
+    g.setColor(__COLOUR);
+    g.draw(getBounds2D());
+  }
+
+
+  public void paint(Graphics oldg, float f)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    centerText2(g,__COLOUR);
+    g.setColor(__COLOUR);
+    paintBorder(oldg, f);
+  }
+
+
+
+
+
+  /**
+   * !!! bug:- should check the boundary of the box and the length of the strings
+   * display at most two labels on the centre of the rectangle
+   * @param oldg is the display area
+   * @param c is the colour
+   */
+  private void centerText(Graphics2D g, Color c)
+  {
+    String labels = _data.getLabels();
+
+    if (labels!=null){
+// Font f = new Font("Courier",Font.BOLD,12);
+      Font f = g.getFont();
+      FontMetrics fm = g.getFontMetrics(f);
+      float ascent = (float) fm.getAscent();
+      float height = (float) fm.getHeight();
+      float x=getUpperLeftX(), y=getUpperLeftY(),w=(float)getWidth(), h=(float)getHeight();
+      float width0=0F, width1=0F, x0=0F, x1=0F, y0=0F, y1=0F;
+
+        g.setColor(c);
+        width0 = (float) fm.stringWidth(labels);
+        x0 = x + (w-width0)/2F;
+
+// g.setFont(f);
+//   AffineTransform transform
+//     = AffineTransform.getRotateInstance(-90.0*Math.PI/180.0,100.0,50.0);
+        g.drawString(labels,x0,y0);
+        /*
+        if (labels.length==1){
+          y0 = y + (h-height)/2F + ascent;
+          g.drawString(labels[0],x0,y0);
+        }
+        else
+        {
+          width1 = (float) fm.stringWidth(labels[1]);
+          x1 = x + (w-width1)/2F;
+          y0 = y + (h-(height*2F))/2F + ascent;
+          y1 = y0 + (height*1F);
+          g.drawString(labels[0],x0,y0);
+          g.drawString(labels[1],x1,y1);
+        }
+        */
+    }
+  }  //-- centerText
+
+
+
+AffineTransform transform;
+
+  //-- pre: _data.getLabels() has at least 2 elements
+  //-- post: draw two labels to the graphic
+  private void centerText2(Graphics2D g, Color c)
+  {
+    String labels = _data.getLabels();
+    if (labels!=null)
+    {
+      Font f;
+      FontMetrics fm;
+      int ascent, height;
+      int x=(int) getUpperLeftX(), y=(int)getUpperLeftY(),
+          w=(int) getWidth(),      h=(int)getHeight();
+      int width0=0, width1=0, x0=0, x1=0, y0=0, y1=0;
+
+      g.setColor(c);
+
+      for (int fs=12; fs>=6; fs-=2)
+	  {
+        f = new Font("Dialog",Font.BOLD,fs);    // <--- Font for the labels
+
+        fm = g.getFontMetrics(f);
+        ascent = fm.getAscent();
+        height = fm.getHeight();
+        width0 = fm.stringWidth(labels);
+        /*
+        width0 = fm.stringWidth(labels[0]);
+        width1 = fm.stringWidth(labels[1]);
+        */
+        //-- paint horizontally
+        /*if (Math.max(width0,width1)<w  &&  2*height<h)*/
+        if (width0<w  &&  height<h)
+        {
+          x0 = x + (w-width0)/2;
+          //x1 = x + (w-width1)/2;
+          y0 = y + (h-(height)) + ascent;
+          //y1 = y0 + (height*1);
+          g.setFont(f);
+          g.drawString(labels,x0,y0);
+          //g.drawString(labels[1],x1,y1);
+ //System.err.println(labels[0] +", "+labels[1]+ " painted in font size: "+fs +"    horizontal!");
+
+          break;
+        }
+        //-- paint vertically
+/*
+        else if (Math.max(width0,width1)<h  &&  2*height<w)
+        {
+          g.setFont(f);
+
+          x0 = x + (w-width0)/2;
+          x1 = x + (w-width1)/2;
+          y0 = y + (h-(height*2))/2 + ascent;
+          y1 = y0 + (height*1);
+          g.setFont(f);
+		  transform
+     		= AffineTransform.getRotateInstance(-90.0*Math.PI/180.0,
+                (double)(x+w)/2.0,(double)(y+h)/2.0);
+		 g.setTransform(transform);
+         System.err.println(transform.toString());
+
+         g.drawString(labels[0],x0,y0);
+         g.drawString(labels[1],x1,y1);
+         System.err.println(labels[0] +", "+labels[1]+ " painted in font size: "+fs +"    vertical!");
+         System.err.println("x: "+x+"  y: "+y+"  w: "+w+"  h: "+h );
+
+         transform
+            = AffineTransform.getRotateInstance(360.0*Math.PI/180.0,
+               (double)(x+w)/2.0,(double)(y+h)/2.0);
+         g.setTransform(transform);
+         System.err.println(transform.toString());
+
+         break;
+        }
+*/
+
+
+      }  // for
+
+    }  // if
+  }  // centerText2
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java	(revision 14150)
@@ -0,0 +1,163 @@
+/**
+ * @(#)ClusterRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import vishnu.testvis.object.ClusterObj;
+import java.awt.*;
+
+
+public class ClusterRect extends ZRect
+{
+    private ClusterObj _data;
+	public ClusterRect(float x1, float y1, float x2, float y2, Color c, ClusterObj o)
+    {
+		super(x1,y1,x2,y2,c,o.getDocsSize());
+        _data = o;
+	}
+
+  public void paint(Graphics oldg){
+    Graphics2D g = (Graphics2D) oldg;
+    g.setColor(getColor());
+    g.fill(getBounds2D());
+    g.setColor(Color.white);
+    g.draw(getBounds2D());
+  }
+
+  public ClusterObj getClusterObj(){return _data;}
+
+  public String getInfo(){return _data.getInfo();}
+
+
+  //------------------------------------------------------------------
+  private int _xRef=0, _yRef=0;   // used in paintLabels and createTipBox
+
+  /**
+   * paint the cluster's labels on the graphical pane
+   * @param oldg is the graphics to be pained
+   * @param w is the max width of the graphic
+   * @param h is the max heigth of the graphic
+   */
+  public void paintLabels(Graphics oldg, float w, float h){
+  Color tipBoxColor = new Color(255,255,180);
+  Graphics2D g = (Graphics2D) oldg;
+  Font f = new Font("TimesRoman",Font.PLAIN,12);
+  FontMetrics fm = g.getFontMetrics(f);
+  int ascent = fm.getAscent();
+  int height = fm.getHeight();
+  int boxWidth = 4 + Math.max(fm.stringWidth(_data.getLabel1()),
+                              fm.stringWidth(_data.getLabel2()) );
+  int boxHeight = 0 + height * 2;
+  Polygon poly;
+
+    g.setFont(f);
+
+    //-- setup the position of the tipbox
+    poly = createTipBox(boxWidth, boxHeight, w, h);
+
+    //-- paint the box
+    g.setColor(tipBoxColor);
+    g.fillPolygon(poly);
+    g.setColor(Color.black);
+    g.drawPolygon(poly);
+    //-- paint the text
+    g.drawString(_data.getLabel1(), _xRef+3F,
+                _yRef + (boxHeight-(height*2F))/2F + ascent);
+    g.drawString(_data.getLabel2(), _xRef+3F,
+                _yRef + (boxHeight-(height*2F))/2F + ascent + height);
+
+  }
+
+
+  private Polygon createTipBox(int boxW, int boxH, float maxW, float maxH){
+  int[] xs = new int[8];
+  int[] ys = new int[8];
+  int x=0, y=0,
+    arrowH=10,
+    arrowW=8,
+    arrowOffset=8,
+    gap=5; //-- gap is the space between the tipbox and the rectangle
+  float position=0.2F;  //-- set the box position
+
+    try{
+
+      //-- try RHS
+      x = (int) getLowerRightX() + gap;
+      y = (int) (getUpperLeftY() + getHeight()*position);
+      if ((x+boxW<=maxW) && (y+boxH<=maxH)){
+        xs[0]=xs[6]=xs[4]=xs[3]=xs[7] = x;
+        xs[1]=xs[2] = x + boxW;
+        xs[5] = x - arrowH;
+        ys[5]=ys[0]=ys[1]=ys[7] = y;
+        ys[3]=ys[2] = y + boxH;
+        ys[6] = y + arrowOffset;
+        ys[4] = y + arrowOffset + arrowW;
+        throw new Exception();
+      }
+
+      //-- try LHS
+      x = (int) getUpperLeftX() - gap - boxW;
+      if ((x>=0) && (y+boxH<=maxH)){
+        xs[0]=xs[7]=xs[6] = x;
+        xs[1]=xs[2]=xs[4]=xs[5] = x + boxW;
+        xs[3] = x + boxW + arrowH;
+        ys[7]=ys[0]=ys[1]=ys[3] = y;
+        ys[6]=ys[5] = y + boxH;
+        ys[2] = y + arrowOffset;
+        ys[4] = y + arrowOffset + arrowW;
+        throw new Exception();
+      }
+
+      //-- try top
+      x = (int) (getUpperLeftX() + getWidth()*position);
+      y = (int) getUpperLeftY() - boxH - gap;
+      if ((y>=0) && (x+boxW<=maxW)){
+        xs[0]=xs[7]=xs[6]=xs[4] = x;
+        xs[1]=xs[2] = x + boxW;
+        xs[5] = x + arrowOffset;
+        xs[3] = x + arrowOffset + arrowW;
+        ys[7]=ys[0]=ys[1] = y;
+        ys[6]=ys[5]=ys[3]=ys[2] = y + boxH;
+        ys[4] = y + boxH + arrowH;
+        throw new Exception();
+      }
+
+      //-- try bottom
+      y = (int) getLowerRightY() + gap;
+      if ((y+boxH<=maxH) && (x+boxW<=maxW)){
+        xs[7]=xs[0]=xs[6]=xs[2] = x;
+        xs[4]=xs[5] = x + boxW;
+        xs[1] = x + arrowOffset;
+        xs[3] = x + arrowOffset + arrowW;
+        ys[7]=ys[0]=ys[1]=ys[3]=ys[4]= y;
+        ys[6]=ys[5] = y + boxH;
+        ys[2] = y - arrowH;
+        throw new Exception();
+      }
+
+      //-- try inside the rectangle
+      x = (int) getUpperLeftX() + 5;
+      y = (int) getUpperLeftY() + 5 + 5;
+      xs[7]=xs[0]=xs[6]=xs[2] = x;
+      xs[4]=xs[5] = x + boxW;
+      xs[1] = x + 3;
+      xs[3] = x + 3 + 5;
+      ys[7]= ys[0]=ys[1]=ys[3]=ys[4] = y;
+      ys[6]= ys[5]   = y + boxH;
+      ys[2] = y - 5;
+
+    }
+    catch(Exception e){
+    }
+
+    _xRef = x;
+    _yRef = y;
+    return (new  Polygon(xs,ys,xs.length));
+  }
+
+
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java	(revision 14150)
@@ -0,0 +1,119 @@
+/**
+ * @(#)Rect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import java.awt.geom.Rectangle2D;
+import java.awt.*;
+
+/**
+ * The Rect class contains direction and colour, the direction is calculated
+ * based on the initial points.
+ */
+abstract class Rect extends Rectangle2D.Float{
+//-- the rectangle will move to this direction during zooming
+final static protected int __NORTH=0, __EAST=1, __SOUTH=2, __WEST=3;
+private int _direction;
+//-- any two points to create a rectangle
+private float _left_x, _left_y, _right_x, _right_y;
+//-- color of the rectangle
+private Color _colour;
+
+  public Rect(float x1, float y1, float x2, float y2, Color c){
+    setup(x1, y1, x2, y2);
+    _colour = c;
+  }
+
+
+  public void showInfo(){
+    System.err.println( "x: "+getX() +"  y: "+getY()+"  w: "+getWidth()+"  h: "+getHeight());
+    System.err.println(   "    direction: "+getDirection_toString()+"  colour: "+
+        _colour.toString());
+  }
+
+
+  /**
+   * setup the direction and the corresponding coordination
+   */
+  private void setup(float x1, float y1, float x2, float y2){
+    _left_x=x1; _left_y=y1;
+    _right_x=x2; _right_y=y2;
+    //-- set up the direction
+    if (x1<x2)
+      if (y1<y2) _direction = __WEST;
+      else if (y1>y2) _direction = __SOUTH;
+      else System.err.println("Error - Rectangle box cannot be zero");
+    else if (x1>x2)
+      if (y1<y2) _direction = __NORTH;
+      else if (y1>y2) _direction = __EAST;
+      else System.err.println("Error - Rectangle box cannot be zero");
+    else
+      System.err.println("Error - Rectangle box cannot be zero");
+    //-- setup the corresponding coordination of the rectangle obj
+    switch (_direction) {
+      case __NORTH:
+        super.setRect(x2,y1,x1-x2,y2-y1);
+        break;
+      case __EAST:
+        super.setRect(x2,y2,x1-x2,y1-y2);
+        break;
+      case __SOUTH:
+        super.setRect(x1,y2,x2-x1,y1-y2);
+        break;
+      case __WEST:
+        super.setRect(x1,y1,x2-x1,y2-y1);
+        break;
+    }
+  }  //-- setup
+
+
+  public void paint(Graphics oldg){
+    Graphics2D g = (Graphics2D) oldg;
+    g.setColor(getColor());
+    g.fill( getBounds2D() );
+    g.setColor(Color.black);
+    g.draw( getBounds2D() );
+  }
+
+  public void paintBorder(Graphics oldg, float w){
+    Graphics2D g = (Graphics2D) oldg;
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getUpperLeftY(),
+       w, (float) getHeight()));
+    g.fill(new Rectangle2D.Float(getLowerRightX()-w,getUpperLeftY(),
+       w, (float) getHeight()));
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getUpperLeftY(),
+       (float) getWidth(),w));
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getLowerRightY()-w,
+       (float) getWidth(),w));
+  }
+
+  public boolean isNorth(){return _direction==__NORTH;}
+  public boolean isEast(){return _direction==__EAST;}
+  public boolean isSouth(){return _direction==__SOUTH;}
+  public boolean isWest(){return _direction==__WEST;}
+
+  public Color getColor(){return _colour;}
+  public float getUpperLeftX(){return ((float) getX());}
+  public float getUpperLeftY(){return ((float) getY());}
+  public float getLowerRightX(){return ((float) (getX()+getWidth()));}
+  public float getLowerRightY(){return ((float) (getY()+getHeight()));}
+  public int getDirection(){return _direction;}
+  public int getOppositeDirection(){return (_direction+2)%4;}
+
+  /**
+   * @return the direction of the rectangule as a string
+   */
+  public String getDirection_toString(){
+    switch (_direction){
+      case __NORTH: return "NORTH";
+      case __EAST: return "EAST";
+      case __SOUTH: return "SOUTH";
+      default: return "WEST";
+    }
+  }
+
+
+}  //-- Rect
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java	(revision 14150)
@@ -0,0 +1,157 @@
+/**
+ * @(#)ZRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import java.awt.*;
+
+public class ZRect extends Rect{
+//-- no of doc that the rectangle represented
+private int _capacity=0;
+
+//-- the new points that the rectangle will be translated (java coordination)
+protected float _t_left_x=0f, _t_left_y=0f, _t_right_x=0f, _t_right_y=0f;
+//-- if the rectangle has been choosen, set it to true
+private boolean _chosen=false;
+
+  public ZRect(float x1, float y1, float x2, float y2, Color c){
+    super(x1, y1, x2, y2, c);
+  }
+
+
+  public ZRect(float x1, float y1, float x2, float y2, Color c, int s){
+    super(x1, y1, x2, y2, c);
+    _capacity = s;
+  }
+
+  public int getCapacity(){return _capacity;}
+  public void setCapacity(int i){_capacity=i;}
+
+  public boolean isChosen(){return _chosen;}
+  public void setChosen(){_chosen=true;}
+  public void resetChosen(){_chosen=false;}
+
+  /**
+   * set the new coordinate, that the rectangle will move to
+   * pre: the coordinate is based on Java graphic coordination
+   *      x1,y1 is the left upper corner and
+   *      x2,y2 is the right lower corner
+   */
+  public void translateTo(float x1, float y1, float x2, float y2){
+    _t_left_x=x1; _t_left_y=y1;
+    _t_right_x=x2; _t_right_y=y2;
+  }
+
+
+
+  private static boolean almostEqual(float a, float b){
+  float tolerant = 0.00001F;
+    return  (Math.abs(a-b) <= tolerant);
+  }
+
+
+  /**
+   * @param x is the input value
+   * @param x1 is the orginal x-coordinate of the rectangle
+   * @param y1 is the orginal y-coordinate of the rectangle
+   * @param x2 is the translated x-coordinate
+   * @param y2 is the translated y-coordinate
+   * @return y, based on x
+   */
+  private static float newY(
+    float x,
+    float x1, float y1, float x2, float y2,
+    int totalFrame, int currentFrame){
+//    return (x1==x2)? y2: ((x*(y1-y2) + x1*y2 - x2*y1) / (x1-x2));
+    if (almostEqual(x1,x2))
+      return ((y2-y1)/(float) totalFrame) * (float) (currentFrame+1) + y1;
+    else
+      return ((x*(y1-y2) + x1*y2 - x2*y1) / (x1-x2));
+  }
+
+  /**
+   * @param y is the input value
+   * @param x1 is the orginal x-coordinate of the rectangle
+   * @param y1 is the orginal y-coordinate of the rectangle
+   * @param x2 is the translated x-coordinate
+   * @param y2 is the translated y-coordinate
+   * @return x, based on y
+   */
+  private static float newX(
+    float y,
+    float x1, float y1, float x2, float y2,
+    int totalFrame, int currentFrame){
+//   return (y1==y2)? x2: ((y*(x1-x2) - x1*y2 + x2*y1)/(y1-y2));
+    if (almostEqual(y1,y2))
+      return ((x2-x1)/(float) totalFrame) * (float) (currentFrame+1) + x1;
+    else
+      return ((y*(x1-x2) - x1*y2 + x2*y1)/(y1-y2));
+  }
+
+
+  /**
+   * pre: translateTo is called
+   * @param how many animated pictures u want to create
+   * @retrun a list of animation picture (rectangles with diff positions)
+   */
+  public ZRect[] zooming(int noOfFrame){
+  ZRect[] vec = new ZRect[noOfFrame];
+  float dist1, dist2, t1, t2;
+
+    switch (getDirection()){
+      case __NORTH: case __SOUTH:
+        dist1 = (_t_left_y - getUpperLeftY())/(float)noOfFrame;
+        dist2 = (_t_right_y - getLowerRightY())/(float)noOfFrame;
+        t1 = getUpperLeftY();
+        t2 = getLowerRightY();
+        //-- create the animated pictures, except the last image
+        for(int i=0;i<noOfFrame-1;i++){
+          t1 += dist1;
+          t2 += dist2;
+          vec[i] = new ZRect(
+            newX(t1,getUpperLeftX(),getUpperLeftY(),_t_left_x,_t_left_y,noOfFrame,i),
+            t1,
+            newX(t2,getLowerRightX(),getLowerRightY(),_t_right_x,_t_right_y,noOfFrame,i),
+            t2,
+            getColor());
+        }
+        vec[vec.length-1] =
+          new ZRect(_t_left_x, _t_left_y, _t_right_x, _t_right_y, getColor());
+
+        break;
+      default:
+        //-- for WEST and EAST
+        dist1 = (_t_left_x - getUpperLeftX())/(float)noOfFrame;
+        dist2 = (_t_right_x - getLowerRightX())/(float)noOfFrame;
+        t1 = getUpperLeftX();
+        t2 = getLowerRightX();
+
+        //-- create the animated pictures, except the last image
+        for(int i=0;i<noOfFrame-1;i++){
+          t1 += dist1;
+          t2 += dist2;
+          vec[i] = new ZRect(
+            t1,
+            newY(t1,getUpperLeftX(),getUpperLeftY(),_t_left_x,_t_left_y,noOfFrame,i),
+            t2,
+            newY(t2,getLowerRightX(),getLowerRightY(),_t_right_x,_t_right_y,noOfFrame,i),
+            getColor());
+        }
+        /**
+         * create the final picture here, the reason we don't do in the above
+         * for loop is because we avoid the 'tolerant' error from the floating
+         * point computation.
+         */
+        vec[vec.length-1] =
+          new ZRect(_t_left_x, _t_left_y, _t_right_x, _t_right_y, getColor());
+
+
+    }  //-- switch
+
+    return vec;
+  }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java	(revision 14150)
@@ -0,0 +1,56 @@
+/**
+ * @(#)BarRenderer.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.table;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.table.TableCellRenderer;
+
+//!!! the % in getString and getColor may WRONG !!!
+
+public class BarRenderer extends JProgressBar implements TableCellRenderer
+{
+
+  public BarRenderer(){
+    super(JProgressBar.HORIZONTAL);
+    setBorderPainted(false);
+    this.setFont(new Font("Arial",Font.PLAIN,9));
+  }
+
+
+  public BarRenderer(int min, int max){
+    super(min,max);
+    setBorderPainted(false);
+    this.setFont(new Font("Arial",Font.PLAIN,9));
+  }
+
+  public String getString(){
+    if (getValue()==0) return "";
+    else return String.valueOf(getValue())+"%";
+  }
+
+  public Component getTableCellRendererComponent(JTable table, Object value,
+    boolean isSelected, boolean hasFocus, int row, int column){
+  int n = 0;
+    if (value instanceof Number){
+      n = ((Number) value).intValue();
+    } else{
+      System.err.println("Error - item in bar render is not a number");
+    }
+    setValue(n);
+    setForeground(getColor(n));
+    return this;
+  }
+
+  private Color getColor(int value){
+    if (value < 40) return Color.green;
+    else if (value <80) return Color.yellow;
+    else
+      return Color.red;
+  }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java	(revision 14150)
@@ -0,0 +1,289 @@
+/**
+ * @(#)RW_DataModel.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.table;
+
+import vishnu.testvis.object.ClusterObj;
+import vishnu.testvis.object.DocObj;
+import javax.swing.table.DefaultTableModel;
+import java.util.*;
+import vishnu.testvis.sammon.QSortAlgorithmTableLength;
+import vishnu.testvis.object.LengthTable;
+
+/**
+ * This table contains 3 columns:
+ *   - a tick box
+ *   - a related word
+ *   - and the hit document frequency
+ */
+public class RW_DataModel extends DefaultTableModel{
+static final private String[] __COLNAMES={"","Related Word","Hit Doc Freq"};
+private String[] _related_words;
+private ClusterObj _cObj;
+private Vector _colnames = new Vector(__COLNAMES.length);
+
+private Vector _docs = null;  // this is created inside 'setData'
+
+
+  public RW_DataModel()
+  {
+    //-- setup the column's names
+    for (int i=0;i<__COLNAMES.length;i++)
+      _colnames.addElement(__COLNAMES[i]);
+    clear();
+  }
+
+
+  //-- clear all data variables in this data model
+  public void clear()
+  {
+    _related_words = null;
+    _cObj = null;
+    _docs = null;
+    setDataVector(new Vector(0), _colnames);
+  }
+
+
+  //--------------------------------------------------------------------------
+  /**
+   * Set up the cluster data in the data model
+   * pre: the related-words are new to the data model
+   * @param rw is a list of related words
+   * @param c is the data of the cluster
+   */
+  public void setData(String[] rw, ClusterObj c)
+  {
+    Vector dataVec = new Vector(0);
+    Vector hdfVec;
+    Vector docs = new Vector();
+    _related_words = rw;
+    _cObj = c;
+
+    hdfVec = _cObj.getHitDocFreq();
+    _docs = new Vector();
+// invarient: _hdfVec.size() == rw.length
+
+    //-- for each related word do
+    for(int i=0;i<hdfVec.size();i++){
+      int freq = ((Vector)hdfVec.elementAt(i)).size();
+      if (freq>0)
+      {
+        Vector  entry;
+        entry = new Vector(__COLNAMES.length+1);
+        entry.addElement(new Boolean(false));
+        entry.addElement(_related_words[i]);
+        entry.addElement(new Integer(freq*100/_cObj.getDocsSize()));
+        dataVec.addElement(entry);
+
+        /**
+         * This extra cell is for internal used only.
+         * each entry consists of a vector of doc, those documents contain the
+         * related-word (with weight>0)
+         *
+         * This is used in the filtering routine
+         * so if I sort the above I must likewise sort this one.
+         */
+         docs.addElement(hdfVec.elementAt(i));
+      }
+    }  // for
+    LengthTable [] table = new LengthTable[dataVec.size()];
+    for (int i=0;i<dataVec.size();i++)
+    {
+      Vector ent =  (Vector)dataVec.elementAt(i);
+      Integer in = (Integer)ent.elementAt(2);
+      table[i]=new LengthTable(i,in.intValue());
+    }
+    QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+    try
+    {
+      qSort.sort(table);
+    }
+    catch (Exception e)
+    {}
+
+    Vector newDataVec= new Vector(dataVec.size());
+    for (int i=dataVec.size()-1;i>=0;i--)
+    {
+      int index = table[i].index;
+      newDataVec.addElement(dataVec.elementAt(index));
+      _docs.addElement(docs.elementAt(index));
+
+    }
+
+    setDataVector(newDataVec, _colnames);
+  } // setData
+  
+  // overloaded, this one for dendro
+  // we pass on freq vector instead of obj
+  public void setData(String[] rw, vishnu.testvis.dendro.ClusterObj _cObj)
+  {
+    Vector dataVec = new Vector(0);
+    Vector hdfVec;
+    Vector docs = new Vector();
+    _related_words = rw;
+
+    hdfVec = _cObj.getHitDocFreq();
+    _docs = new Vector();
+// invarient: _hdfVec.size() == rw.length
+
+    //-- for each related word do
+    for(int i=0;i<hdfVec.size();i++){
+      int freq = ((Vector)hdfVec.elementAt(i)).size();
+      if (freq>0)
+      {
+        Vector  entry;
+        entry = new Vector(__COLNAMES.length+1);
+        entry.addElement(new Boolean(false));
+        entry.addElement(_related_words[i]);
+        entry.addElement(new Integer(freq*100/_cObj.getDocsSize()));
+        dataVec.addElement(entry);
+
+        /**
+         * This extra cell is for internal used only.
+         * each entry consists of a vector of doc, those documents contain the
+         * related-word (with weight>0)
+         *
+         * This is used in the filtering routine
+         * so if I sort the above I must likewise sort this one.
+         */
+         docs.addElement(hdfVec.elementAt(i));
+      }
+    }  // for
+    LengthTable [] table = new LengthTable[dataVec.size()];
+    for (int i=0;i<dataVec.size();i++)
+    {
+      Vector ent =  (Vector)dataVec.elementAt(i);
+      Integer in = (Integer)ent.elementAt(2);
+      table[i]=new LengthTable(i,in.intValue());
+    }
+    QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+    try
+    {
+      qSort.sort(table);
+    }
+    catch (Exception e)
+    {}
+
+    Vector newDataVec= new Vector(dataVec.size());
+    for (int i=dataVec.size()-1;i>=0;i--)
+    {
+      int index = table[i].index;
+      newDataVec.addElement(dataVec.elementAt(index));
+      _docs.addElement(docs.elementAt(index));
+
+    }
+
+    setDataVector(newDataVec, _colnames);
+  } // setData
+
+
+  public String[] getRefinedTrecIDs()
+  {
+      Vector sVec = new Vector();
+      HashSet set = new HashSet();  //-- this can avoid duplicated items !
+      boolean isNewItem;
+      
+      for(int i=0;i<getDataVector().size();i++)
+      {
+            if (((Boolean) getValueAt(i,0)).booleanValue())
+            {
+                
+                Vector docs = (Vector) _docs.elementAt(i);
+
+                for (int j=0;j<docs.size();j++)
+                {
+                    isNewItem = set.add((DocObj) docs.elementAt(j));
+                    if (isNewItem)
+                    {
+                        sVec.addElement(((DocObj)docs.elementAt(j)).getTrecID());
+                        //System.out.println(((DocObj) docs.elementAt(j)).getTrecID());
+                    }
+                }
+            }
+      }
+      
+      String[] sStr = new String[sVec.size()];
+      for( int i = 0; i < sVec.size(); i++ )
+      {
+          sStr[i] = new String((String)sVec.elementAt(i));
+      }
+      return sStr;
+  }
+  
+  public String[] getFilterWords()
+  {
+        Vector sVect = new Vector();
+        
+        for(int i=0;i<getDataVector().size();i++)
+        {
+            if (((Boolean) getValueAt(i,0)).booleanValue())
+            {
+                sVect.addElement((String)getValueAt(i,1));
+            }   
+        }
+        
+        String sStr[] = new String[sVect.size()];
+        
+        for( int i = 0; i < sVect.size(); i++ )
+        {
+            sStr[i] = (String)sVect.elementAt(i);    
+        }
+        return sStr;
+  }
+
+  public String getRefinedDocs(){
+  String s = "";
+  HashSet set = new HashSet();  //-- this can avoid duplicated items !
+  boolean isNewItem;
+    for(int i=0;i<getDataVector().size();i++){
+
+      if (((Boolean) getValueAt(i,0)).booleanValue()) {
+       Vector docs = (Vector) _docs.elementAt(i);
+
+        for (int j=0;j<docs.size();j++){
+          isNewItem = set.add((DocObj) docs.elementAt(j));
+          if (isNewItem)
+          {
+            s += ((DocObj) docs.elementAt(j)).format4Fields();
+            //System.out.println(((DocObj) docs.elementAt(j)).format4Fields());
+          }
+            
+        }
+
+      }
+
+    }
+    s = "<html><font face=Arial size=+10> " + set.size()  + " documents found\n" + s +
+        "</font></html>\n";
+    return s;
+  }
+
+
+
+
+  //--------------------------------------------------------------------------
+  public Class getColumnClass(int col)
+  {
+    switch (col)
+    {
+      case 0: return Boolean.class;
+      case 1: return String.class;
+      case 2: return Integer.class;
+      default: return Object.class;
+    }
+  }
+
+  public int getColumnCount(){return __COLNAMES.length;}
+
+  public boolean isCellEditable(int row, int col){
+    return ((col==0) ? true : false);
+  }
+  public void setValueAt(Object value, int row, int col){
+    super.setValueAt(value,row,col);
+  }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java	(revision 14150)
@@ -0,0 +1,189 @@
+/**
+ * @(#)NavigatorPanel.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.visual.Vishnu;
+import vishnu.testvis.object.ClusterObj;
+import vishnu.testvis.object.DataManager;
+import vishnu.testvis.treemap.table.*;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import java.util.Vector;
+
+
+public class NavigatorPanel extends JComponent
+{
+    private JTable _table;
+    private JButton _showAllButton, _refineButton;
+    private JEditorPane _display;
+    private TMNavigator _outerFrame;
+    private ClusterObj _cObj;
+//public String _codeBase;
+
+    private RW_DataModel _dataModel;  //-- this is created within this class
+
+
+    public NavigatorPanel(JTable t, /*JButton b1,*/ JButton b2,
+        JEditorPane d, TMNavigator f)
+    {
+        _table = t;
+        /*_showAllButton = b1;
+        _showAllButton.setActionCommand("showAllDoc");
+        _showAllButton.setToolTipText("show ALL documents within the selected cluster");
+        _showAllButton.addActionListener(new ButtonHandler());*/
+
+        _refineButton = b2;
+        _refineButton.setActionCommand("refineDoc");
+        _refineButton.setToolTipText("show documents with any of the selected related-words");
+        _refineButton.addActionListener(new ButtonHandler());
+
+        _display = d;
+        _outerFrame = f;
+
+        _display.addHyperlinkListener(new HyperlinkListener() {
+                public void hyperlinkUpdate(HyperlinkEvent e)
+                {
+	            // Handle clicks; ignore mouseovers and other link-related events
+	                if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED)
+                    {
+	            // Get the HREF of the link and display it.
+                        String hpresult = e.getDescription();
+                        int index = Integer.parseInt(hpresult);
+                        System.out.println("hyperlink result: " + index);
+                        if (index > 0)
+                            _outerFrame._displayFrame.dataManager.fetchDocNo(index);
+	                }
+                }
+            });
+        _dataModel = new RW_DataModel();
+        _table.setModel(_dataModel);
+
+
+
+        _cObj = null;
+        unableButtons();
+        setupRenderer();
+    }
+
+    private void unableButtons()
+    {
+        //_showAllButton.setEnabled(false);
+        _refineButton.setEnabled(false);
+    }
+    private void enableButtons()
+    {
+        //_showAllButton.setEnabled(true);
+        _refineButton.setEnabled(true);
+    }
+
+    public void clear()
+    {
+        _cObj = null;
+        _dataModel.clear();
+        _display.setText("");
+        unableButtons();
+    }
+
+
+
+    public void setData(String[] rw, ClusterObj c)
+    {
+        _cObj = c;
+        _dataModel.setData(rw, _cObj);
+        enableButtons();
+        setupRenderer();
+    }
+
+
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+    
+    public void showDocuments()
+    {
+        if( _cObj != null )
+        {
+            _display.setText(_cObj.formatDocs());
+        }
+    }
+     
+
+    class ButtonHandler implements ActionListener
+    {
+        public void actionPerformed(ActionEvent event)
+        {
+            if (_cObj!=null)
+            {
+                String action = event.getActionCommand();
+                if (action.equals("showAllDoc"))
+                {
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    _display.setText(_cObj.formatDocs());
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+                else if (action.equals("refineDoc"))
+                {
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    _display.setText(   _dataModel.getRefinedDocs()   );
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                    
+                    _outerFrame._displayFrame.log.startsection("FILTERING");
+       
+                    String trec_id[] = _dataModel.getRefinedTrecIDs();
+                    String filter_Words[] = _dataModel.getFilterWords();
+                    String s = new String("");
+                    for( int i = 0; i < filter_Words.length; i++ )
+                    {
+                            s += filter_Words[i] + " ";
+                    }
+      
+                   _outerFrame._displayFrame.log.log("FILTER_WORDS",s);
+                   _outerFrame._displayFrame.log.log("FILTERED_DOCS",Integer.toString(trec_id.length));
+       
+                   Vector performance = _outerFrame._displayFrame.queries.getPerformance(_outerFrame._displayFrame.query, trec_id);
+                   
+                   _outerFrame._displayFrame.log.startsection("PERFORMANCE");
+                    
+                   if( performance.size() > 0 )
+                   {
+                        Integer relvTotal = (Integer)performance.elementAt(performance.size()-2);
+                        Integer relvRetr = (Integer)performance.elementAt(performance.size()-1);
+                        _outerFrame._displayFrame.log.log("RELEVANT_TOTAL",Integer.toString(relvTotal.intValue()));
+                        _outerFrame._displayFrame.log.log("RELEVANT_RETRIEVED",Integer.toString(relvRetr.intValue()));
+                        for( int i = 0; i < performance.size()-2; i++ )
+                        {
+                            Integer perf = (Integer)performance.elementAt(i);
+                            String str = "PRECISION_" + Integer.toString((i+1)*10);
+                            _outerFrame._displayFrame.log.log(str,Integer.toString(perf.intValue()));
+                        }
+                   }
+                   else
+                   {
+                        _outerFrame._displayFrame.log.log("INFO","No relevance data available");
+                   }
+                
+                    _outerFrame._displayFrame.log.endsection("PERFORMANCE");
+                    _outerFrame._displayFrame.log.endsection("FILTERING");
+                    
+                }
+            }
+        }
+    }  //-- end ButtonHandler
+
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java	(revision 14150)
@@ -0,0 +1,159 @@
+/**
+ * @(#)Navigator.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.visual.Vishnu;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.border.BevelBorder;
+
+
+public class TMNavigator extends JPanel
+{
+    final private int _TREEMAPWIDTH=300, _TREEMAPHEIGHT=350;
+    public final static int LIST_ON_LEFT = 0;
+    public final static int LIST_ON_RIGHT = 1;
+    private int listPos;
+
+//    JTextField _searchTF = new JTextField();
+//    JButton _searchButton = new JButton("Search");
+    public Vishnu _displayFrame;
+    JScrollPane _displayJSP = new JScrollPane();
+    JEditorPane _displayJEP = new JEditorPane();
+    JTable _table = new JTable();
+    JButton _showAllButton = new JButton("Show All Documents");
+    JButton _refineButton = new JButton("Filter");
+    NavigatorPanel navigator;
+//  = new NavigatorPanel(_table, _showAllButton, _refineButton, _displayJEP, this,_codeBase);
+
+    JButton _backButton = new JButton("<<   Back");
+    TreemapPanel _treemapPanel;
+//  = new TreemapPanel(_TREEMAPWIDTH,_TREEMAPHEIGHT, _backButton, navigator, _codeBase);
+
+
+
+	public TMNavigator(Vishnu displayFrame, int listPos)
+    {
+   	    _displayFrame = displayFrame;
+   	    this.listPos = listPos;
+   	    
+        navigator = new NavigatorPanel(_table, /*_showAllButton*/ _refineButton, _displayJEP, this);
+        _treemapPanel = new TreemapPanel(_TREEMAPWIDTH,_TREEMAPHEIGHT, _backButton, navigator, _displayFrame);
+        
+        buildGUI();
+        
+	    setSize(_TREEMAPWIDTH*2+40,_TREEMAPHEIGHT+340);
+        //setVisible(false);
+	    //setupEventHandler();
+	}
+
+    public void setVisible(boolean f)
+    {
+        super.setVisible(f);
+        _treemapPanel.setVisible(f);
+    }
+
+    void buildGUI()
+    {
+        JPanel tmpPanel;
+        JScrollPane tmpJSP;
+
+        //this.setLayout(new FlowLayout(FlowLayout.LEFT,10,10));
+        this.setLayout(null);
+//        _searchTF.setFont(new Font("Courier",Font.BOLD,16));
+//        _searchTF.setBounds(10,10,400,30);
+//        add(_searchTF,null);
+
+//        _searchButton.setActionCommand("search");
+//        _searchButton.setBounds(10+400+10,10,100,30);
+//        add(_searchButton,null);
+        _backButton.setBorder(BorderFactory.createEtchedBorder());
+        _backButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _backButton.setBounds(10+_TREEMAPWIDTH+10,10+_TREEMAPHEIGHT+10,200,30);
+        add(_backButton,null);
+
+        if( listPos == 0 ) // list on left
+        {
+            tmpPanel = newJPanel(_treemapPanel,Color.gray,
+            10,10,_TREEMAPWIDTH+3,_TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        if( listPos == 1 ) // list on right
+        {
+            tmpPanel = newJPanel(_treemapPanel,Color.gray,
+            10+_TREEMAPWIDTH+10,10,_TREEMAPWIDTH+3,_TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+
+
+        _displayJEP.setEditorKit(new HTMLEditorKit());
+        _displayJEP.setEditable(false);
+        Dimension sSize=this.getToolkit().getScreenSize();
+        tmpJSP = new JScrollPane();
+        tmpJSP.getViewport().add(_displayJEP);
+        tmpPanel = newJPanel(tmpJSP, Color.white,
+            10, 10+_TREEMAPHEIGHT+10+30+10, _TREEMAPWIDTH*2+10,
+            sSize.height>700?200:85);
+        add(tmpPanel,null);
+
+        //-- manually creating the columns
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(_TREEMAPWIDTH*2/3);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(_TREEMAPWIDTH/3);
+        _table.setRowSelectionAllowed(false);
+        tmpJSP = new JScrollPane();
+        tmpJSP.getViewport().add(_table);
+        
+        if( listPos == 0 )
+        {
+            tmpPanel = newJPanel(tmpJSP, Color.white,
+            10+_TREEMAPWIDTH+10, 10, _TREEMAPWIDTH, _TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        if( listPos == 1 )
+        {
+            tmpPanel = newJPanel(tmpJSP, Color.white,
+            10, 10, _TREEMAPWIDTH, _TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        
+        
+        _showAllButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _showAllButton.setBounds(10+_TREEMAPWIDTH+10,10+_TREEMAPHEIGHT+10,200,30);
+        _showAllButton.setBorder(BorderFactory.createEtchedBorder());
+        //add(_showAllButton,null);
+
+        _refineButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _refineButton.setBounds(10+_TREEMAPWIDTH+10+200,10+_TREEMAPHEIGHT+10,100,30);
+        _refineButton.setBorder(BorderFactory.createEtchedBorder());
+        add(_refineButton,null);
+
+    }
+
+
+    private JPanel newJPanel(Component c, Color bg,int x, int y, int w, int h)
+    {
+        JPanel p = new JPanel();
+        p.setBorder(new BevelBorder(BevelBorder.LOWERED));
+        p.setLayout(new BorderLayout());
+        p.setBounds(x,y,w,h);
+        p.setBackground(bg);
+        p.add(c,BorderLayout.CENTER);
+        return p;
+  }
+
+    public void update()
+    {
+        _treemapPanel.newSearch();
+    }
+
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java	(revision 14150)
@@ -0,0 +1,539 @@
+/**
+* name      TreemapPanel.java 
+* author    Peter Au
+* modified  Daniel Heesch (29.04.02)
+*/
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.object.DataManager;
+import vishnu.testvis.treemap.rectangle.ClusterRect;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.Stack;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import java.util.*;
+
+public class TreemapPanel extends Canvas
+{
+
+    private int _width, _height;
+    private JButton _popUpButton;
+    private NavigatorPanel _navigator;
+    private String _newTerms;
+    Vishnu _displayFrame;
+    Image _offScreenImage=null;
+    boolean _init_done=false, _drillDownFlag=false, _newSearchFlag=false, _popUpFlag=false;
+    Stack _visualObjs;
+    
+    JPopupMenu[] popups;
+    PopupMenu popup;
+
+    public TreemapPanel(int w, int h, JButton bb, NavigatorPanel n, Vishnu displayFrame)
+    {
+        _displayFrame = displayFrame;
+        _width=w; _height=h;
+        _popUpButton = bb;
+        _navigator = n;
+        _displayFrame = displayFrame;
+        setSize(_width,_height);
+        _visualObjs = new Stack();
+        addMouseListener(new MouseHandler());
+        _popUpButton.setActionCommand("popUp");
+        _popUpButton.setEnabled(false);
+        _popUpButton.setToolTipText("Go back to the previous level");
+        _popUpButton.addActionListener(new ButtonHandler());
+                   
+        popup = new PopupMenu();
+        
+        add(popup);
+
+    }
+
+    class MouseHandler extends MouseAdapter{}
+
+    class ButtonHandler implements ActionListener
+    {
+        public void actionPerformed(ActionEvent event)
+        {
+            String action = event.getActionCommand();
+            String st;
+            if (action.equals("popUp")){
+                processPopUp();
+            }
+        }
+    }
+    
+    public void update(Graphics oldg)
+    {
+        Graphics2D g = (Graphics2D) oldg;
+        Visualiser currentVObj;
+        
+        if (_newSearchFlag)
+        {
+            _visualObjs.push(new Visualiser(_displayFrame.dataManager, 0F,0F, _width, _height));
+            currentVObj = ((Visualiser) _visualObjs.peek());
+            currentVObj.createTreemapView(createImage(_width, _height));
+            
+            //-- draw the treemap view on the offscreen-buffer
+            currentVObj.drawTreemapView(_offScreenImage, this);
+            
+            _newSearchFlag = false;
+        }
+        else if (_drillDownFlag)
+        {
+            _displayFrame.refine = true;
+            currentVObj = (Visualiser) _visualObjs.peek();
+            
+            //-- create a list of image buffer for the animated pictures
+            Image iBuffer[]=new Image[Visualiser.__FRAME];
+            
+            for(int i=0;i<iBuffer.length;i++)
+                iBuffer[i]=createImage(_width,_height);
+
+            //-- prepare the zooming pictures of the current visaul object
+            currentVObj.prepare_zooming(iBuffer, this);
+
+            int s = _visualObjs.size();
+            float x = Visualiser.__borderwidth*s,
+                y = Visualiser.__borderheight*s;
+         
+            DataManager dm = new DataManager(_displayFrame);
+            dm.query(_newTerms);
+            _visualObjs.push(new Visualiser(dm,x, y, _width-x*2, _height-y*2));
+
+                /**
+                * create a treemap view for the new data
+                * before doing so, we have to copy the last zooming picture to the treemap
+                * this is because the zooming picture contains the borders
+                */
+                
+            Image tmpImg = createImage(_width, _height);
+            Graphics2D tmpGph = (Graphics2D) tmpImg.getGraphics();
+            tmpGph.drawImage(currentVObj.getLastZoomingImage(),0,0,this);
+            
+            //-- now create the treemap view
+            ((Visualiser) _visualObjs.peek()).createTreemapView(tmpImg);
+
+            //-- draw the treemap view (top element of the stack) on the offscreen buffer
+            ((Visualiser) _visualObjs.peek()).drawTreemapView(_offScreenImage, this);
+
+            //-- when everyting is done, we then show the animated pictures
+            //-- we use the 2nd element of the stack, not the top one
+            currentVObj.show_zoomIn(g,this);
+
+            _drillDownFlag = false;
+            setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+        }
+        else if (_popUpFlag)
+        {
+
+            currentVObj = (Visualiser) _visualObjs.peek();
+            //-- redraw the treemap view on the offscreen buffer, as the previous
+            //-- level is gone
+            currentVObj.drawTreemapView(_offScreenImage, this);
+            //-- show the animated pictures
+            currentVObj.show_zoomOut(g,this);
+            _popUpFlag = false;
+        }
+        paint(g);
+    }
+
+
+    public void clearDisplay()
+    {
+        _navigator.clear();
+    }
+
+
+    public void processPopUp()
+    {
+        if (_visualObjs.size() > 1)
+        {
+            _displayFrame.log.log("UNREFINE_QUERY");
+            //-- get rid of the top obj in the stack
+            _visualObjs.pop();
+            //-- reset the necessary instance variable back to the default values
+            ((Visualiser) _visualObjs.peek()).setDefault();
+            //-- clear the table and display
+            clearDisplay();
+            if (_visualObjs.size()==1) _popUpButton.setEnabled(false);
+            _popUpFlag = true;
+            repaint();
+        }
+    }
+
+    public void newSearch()
+    {
+        clearDisplay();
+        _popUpButton.setEnabled(false);
+        _visualObjs = new Stack();
+        _newSearchFlag = true;
+        repaint();
+    }
+
+
+    private void processDrillDown(int x, int y){
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+        ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+        //-- chose which cluster to drill down
+        if (r!=null){
+            //-- do sth on 'r'??? send it back to the server and do the clustering again
+            _popUpButton.setEnabled(true);
+            clearDisplay();
+            _drillDownFlag = true;
+            _newTerms = r.getClusterObj().getLabel();
+            _navigator.clear();  //-- clear the table and the display
+            repaint();
+
+        }
+        else
+            setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+
+
+
+    private void processInspect(int x, int y){
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+        ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(x,y);
+        if (r!=null){
+            //-- draw a corner on the selected cluster
+            ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                this, x, y, _width, _height);
+            //-- setup the data model in the table
+            _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                r.getClusterObj());
+                
+            // and show all documents!
+            _navigator.showDocuments();            
+                
+            repaint();
+        }
+        setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+    
+    
+    protected void processMouseEvent(MouseEvent e)
+    {
+        if (e.getID()==MouseEvent.MOUSE_CLICKED)
+        {       
+            
+        if (!_visualObjs.empty())
+        {
+            Font weak = new Font("Arial",Font.PLAIN,14);
+            Font bold = new Font("Arial",Font.BOLD,14);
+            
+            final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+            ClusterObj cObj = r.getClusterObj();
+            int numDocs = cObj.getDocsSize();
+            
+            boolean img_hasChanged;
+                    
+            popup.removeAll();
+            MenuItem menuItem = new MenuItem(numDocs + " documents");
+            menuItem.setFont(bold);
+            
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                       
+                    if (r!=null)
+                    {
+                        
+                        //-- draw a corner on the selected cluster
+                //        ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                //            this, x, y, _width, _height);
+                        //-- setup the data model in the table
+                        
+                        String str = r.getClusterObj().getLabel();
+                        _displayFrame.log.log("SELECTED_CLUSTER",str);
+                        _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                        r.getClusterObj());
+                
+                        // and show all documents!
+                        _navigator.showDocuments();            
+                
+                        repaint();
+                    }
+         
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+                  
+            menuItem = new MenuItem("Refined Query");
+            menuItem.setFont(bold);
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    
+                    // all this does is return the rectangle, on which drill down
+                    // is to be performed
+                    //ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+                    //-- chose which cluster to drill down
+                    if (r!=null)
+                    {
+                        //-- do sth on 'r'??? send it back to the server and do the clustering again
+                        _popUpButton.setEnabled(true);
+                        clearDisplay();
+                        _drillDownFlag = true;
+                        _newTerms = r.getClusterObj().getLabel();
+                        _navigator.clear();  //-- clear the table and the display
+                        repaint();
+                    }
+                    else
+                    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+            String label = cObj.getLabel();
+            
+            StringTokenizer st = new StringTokenizer(label);
+                       
+            while( st.hasMoreTokens() )
+            {
+                menuItem = new MenuItem(st.nextToken());
+                menuItem.setFont(bold);
+                menuItem.setEnabled(false);
+                popup.add(menuItem);
+            }
+                        
+            popup.show(this,e.getX(),e.getY());
+            
+            /*img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+                _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+            System.out.println("Changed?: " + img_hasChanged);
+            if (img_hasChanged)*/  repaint();
+        }
+        
+        }  // if mouseclick
+    }
+
+
+    //------------------------------------------------------------------
+
+    public void paint(Graphics oldg)
+    {
+        //System.out.println("Tree map paint called");
+        Graphics2D tmpGph, g = (Graphics2D) oldg;
+        Image tmpImg;
+        /**
+        * create off-scrren images for double-buffering
+        * There is a known deficiency bug (Java itself) with createImage in java applications.
+        * Workaround: to create necessary image buffers inside paint!
+        */
+        if (!_init_done){
+            _offScreenImage = createImage(_width, _height);
+            _init_done = true;
+        }
+
+        /*if(this.isVisible())*/ g.drawImage(_offScreenImage,0,0,this);
+
+    }  //-- paint
+
+
+
+}  //-- TreemapPanel
+
+
+        /*
+        addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent e)
+            {
+                if (!_visualObjs.empty())
+                {
+                    final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+                    ClusterObj cObj = r.getClusterObj();
+                    int numDocs = cObj.getDocsSize();
+                    System.out.print("Mouse moved to ");           
+                    System.out.println(e.getX() + " " + e.getY());
+                    
+                    boolean img_hasChanged;
+            
+                    popup.removeAll();
+                    MenuItem menuItem = new MenuItem(numDocs + " documents");
+                    menuItem.addActionListener(new ActionListener()
+                    {
+                        public void actionPerformed(ActionEvent actionEvent)
+                        {
+                            System.out.println("Show documents");
+                    
+                            if (r!=null)
+                            {
+                                //-- draw a corner on the selected cluster
+                                //((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                                // this, x, y, _width, _height);
+                                //-- setup the data model in the table
+                                _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                                r.getClusterObj());
+                
+                                // and show all documents!
+                                _navigator.showDocuments();            
+                
+                                 repaint();
+                            }
+         
+                        }
+                    });
+            
+                    popup.add(menuItem);
+                
+                    menuItem = new MenuItem("Drill down");
+                    menuItem.addActionListener(new ActionListener()
+                    {
+                        public void actionPerformed(ActionEvent actionEvent)
+                        {
+                            System.out.println("Drill down clicked");
+                        }
+                    });
+            
+                    popup.add(menuItem);
+            
+                    popup.show(this,e.getX(),e.getY());
+            
+                    img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+                    _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+                    System.out.println("Changed?: " + img_hasChanged);
+                    if (img_hasChanged)  repaint();
+                }
+            }
+        });*/
+        
+        /*  protected void processMouseEvent(MouseEvent e)
+    {
+
+        if (!_visualObjs.empty())
+        {
+            //-- if the mouse points outside the obj, then copy the original treemap view
+            //-- without the tipbox to the screen
+            if (e.getID()==MouseEvent.MOUSE_EXITED){
+                //-- get the most update view of the treemap
+                ((Visualiser) _visualObjs.peek()).drawTreemapView(_offScreenImage, this);
+                repaint();
+            }
+            else if (e.getID()==MouseEvent.MOUSE_CLICKED){
+
+                //-- if the user pressed the first mouse button
+                if ((e.getModifiers() & InputEvent.BUTTON1_MASK) == InputEvent.BUTTON1_MASK)
+                {
+                    System.out.println("Mouse clicked");
+                    processInspect(e.getX(),e.getY());
+                    
+                }
+                //-- if the user pressed the third mouse button
+                else if ((e.getModifiers() & InputEvent.BUTTON3_MASK) == InputEvent.BUTTON3_MASK)
+                    processDrillDown(e.getX(),e.getY());
+
+            }
+        }
+
+    }*/
+
+    //------------------------------------------------------------------
+/*    protected void processMouseMotionEvent(MouseEvent e)
+    {
+        System.out.println("Outer Mouse in motion");
+        if (!_visualObjs.empty())
+        {
+            Font weak = new Font("Arial",Font.PLAIN,14);
+            Font bold = new Font("Arial",Font.BOLD,14);
+            
+            final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+            ClusterObj cObj = r.getClusterObj();
+            int numDocs = cObj.getDocsSize();
+            
+            System.out.println("Inner Mouse in motion");
+            System.out.println(e.getX() + " " + e.getY());
+            //System.out.println("Mouse in motion");
+            boolean img_hasChanged;
+                    
+            popup.removeAll();
+            MenuItem menuItem = new MenuItem(numDocs + " documents");
+            menuItem.setFont(bold);
+            
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    System.out.println("Show documents");
+                    
+                    if (r!=null)
+                    {
+                        //-- draw a corner on the selected cluster
+                //        ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                //            this, x, y, _width, _height);
+                        //-- setup the data model in the table
+                        _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                        r.getClusterObj());
+                
+                        // and show all documents!
+                        _navigator.showDocuments();            
+                
+                        repaint();
+                    }
+         
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+                  
+            menuItem = new MenuItem("Refined Query");
+            menuItem.setFont(bold);
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    
+                    // all this does is return the rectangle, on which drill down
+                    // is to be performed
+                    //ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+                    //-- chose which cluster to drill down
+                    if (r!=null)
+                    {
+                        //-- do sth on 'r'??? send it back to the server and do the clustering again
+                        _popUpButton.setEnabled(true);
+                        clearDisplay();
+                        _drillDownFlag = true;
+                        _newTerms = r.getClusterObj().getLabel();
+                        _navigator.clear();  //-- clear the table and the display
+                        repaint();
+                    }
+                    else
+                    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+            String label = cObj.getLabel();
+            
+            StringTokenizer st = new StringTokenizer(label);
+                       
+            while( st.hasMoreTokens() )
+            {
+                menuItem = new MenuItem(st.nextToken());
+                menuItem.setFont(bold);
+                menuItem.setEnabled(false);
+                popup.add(menuItem);
+            }
+                        
+            popup.show(this,e.getX(),e.getY());
+            
+            //img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+            //   _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+            //System.out.println("Changed?: " + img_hasChanged);
+            //if (img_hasChanged)
+            repaint();
+        }
+    }*/
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java	(revision 14150)
@@ -0,0 +1,671 @@
+/**
+ * @(#)Visualiser.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.object.*;
+import vishnu.testvis.treemap.rectangle.*;
+import java.lang.Exception;
+import java.awt.*;
+import java.awt.image.ImageObserver;
+import java.awt.geom.GeneralPath;
+
+public class Visualiser extends Canvas implements Runnable
+{
+
+  static private final int __FRAMEDELAY = 100;
+
+  static final int __FRAME=10;
+  static float __borderwidth=5F, __borderheight=5F;
+//-- the following variables specify the actual drawing area
+  final private float _upperX, _upperY, _width, _height;
+
+  private DataManager _data;
+  private Image _treemapView = null;  //-- static treemap view
+  private Image[] _zoomingImages;
+
+//-- rectangles represent the cluster-groups and the clusters
+  private ClusterGroupRect[] _gpRects = null;
+  private ClusterRect[][] _cRects = null;
+
+//-- the rectangle the mouse pointed to, used for tipbox
+  private ClusterRect _previousRect = null;
+
+//-- to identify which rectangle has a corner-paint
+  private ClusterRect _cornerRect = null;
+
+
+  /**
+   * Constructor: visualise the data into the specific size of pane
+   * @param d is the data obj
+   * @param x is the left upper x-coordinate of the display pane
+   * @param y is the left upper y-coordinate of the display pane
+   * @param w is the width of the display pane
+   * @param h is the height of the display pane
+   */
+  public Visualiser(DataManager d, float x, float y, float w, float h)
+  {
+    int noOfGroup;
+    _data = d;
+    _upperX=x; _upperY=y; _width=w; _height=h;
+
+    //-- allocate memory for _cRects and _gpRects
+    noOfGroup = _data.getClusterGroupsSize();
+    _gpRects = new ClusterGroupRect[noOfGroup];
+    _cRects = new ClusterRect[noOfGroup][];
+    for (int i=0; i<noOfGroup; i++)
+      _cRects[i] = new ClusterRect [_data.getClusterGroup(i).getClustersSize()];
+    // create rectangles for the corresponding groups and clusters
+    createRects();
+  }
+
+
+  static public void setBorder(float w, float h)
+  {
+    __borderwidth = w;
+    __borderheight = h;
+  }
+
+
+  public DataManager getDataManager(){return _data;}
+
+  //---------------------------------------------------------------------
+  /**
+   * set different variables back to default values
+   */
+  public void setDefault()
+  {
+    _previousRect = null;
+    _cornerRect = null;
+    try
+    {
+      for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].isChosen())
+          for(int j=0;j<_cRects[i].length;j++)
+            if (_cRects[i][j].isChosen())
+	    {
+              _gpRects[i].resetChosen();
+              _cRects[i][j].resetChosen();
+              throw new Exception();
+            }  //-- if
+    }
+    catch(Exception e)
+    {
+    }  //-- catch
+  }
+
+
+  //---------------------------------------------------------------------
+  /**
+   * @return a snapshot of the tree map
+   */
+  public Image getTreemapView(){
+    return _treemapView;
+  }
+
+
+  /**
+   * create a treemap view of the obj, only call once
+   * @param img
+   */
+  public void createTreemapView(Image img)
+  {
+  	final float GP_BORDERWIDTH = 1F; // <-- set the width of the border of each group
+    _treemapView = img;
+    Graphics2D g = (Graphics2D) _treemapView.getGraphics();
+    for (int i=0;i<_gpRects.length; i++)
+	{
+      for(int j=0;j<_cRects[i].length;j++)
+	  {
+        _cRects[i][j].paint(g);
+      }
+      _gpRects[i].paint(g);
+    } //-- for
+    //-- paint the border of each group
+    for (int i=0;i<_gpRects.length; i++)
+       _gpRects[i].paintBorder(g,GP_BORDERWIDTH);
+    // paint the outer border
+    ZRect fooRect = new ZRect(_upperX,_upperY,_upperX+_width,_upperY+_height,null);
+    g.setColor(Color.black);
+    fooRect.paintBorder(g, GP_BORDERWIDTH);
+
+  }  //-- create_pict
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * draw the treemap view on the image, together with the corner triangle
+   */
+  public void drawTreemapView(Image img, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) img.getGraphics();
+    g.drawImage(_treemapView,0,0,observer);
+    drawCorner(g);
+  }
+
+
+  /**
+   * draw a small triangle on the top-right corner of the _cornerRect and make
+   * sure that the tipbox is drawn as well
+   */
+  public void drawCorner(ClusterRect r, Image offScreenImg, ImageObserver observer,
+    int x, int y, float w, float h)
+  {
+    _cornerRect = r;
+    _previousRect = null; //-- force the tipbox to be repainted
+    drawTipBox(offScreenImg, observer, x, y, w, h);
+  }
+
+
+  /**
+   * draw a small triangle on the top-right corner of the _cornerRect
+   */
+  private void drawCorner(Graphics2D g)
+  {
+    if (_cornerRect!=null)
+    {
+      GeneralPath p;
+      float x=_cornerRect.getLowerRightX(), y=_cornerRect.getUpperLeftY(), w=8F;
+      p = new GeneralPath(GeneralPath.WIND_NON_ZERO);
+      if (Math.min(_cornerRect.getWidth(),_cornerRect.getHeight())<w)
+        w = (float) Math.min(_cornerRect.getWidth(),_cornerRect.getHeight());
+      p.moveTo(x-0.7F,   y+0.9F);
+      p.lineTo(x-0.7F-w, y+0.9F);
+      p.lineTo(x-0.7F,   y+0.9F+w);
+      p.closePath();
+      g.setColor(Color.black);
+      g.fill(p);
+    }
+  }
+
+
+  /**
+   * signal the caller (by returning ture) to update the screen, a tipbox
+   * with the label of the cluster is drawn on the offscreen-image accordingly
+   * @param offScreenImg is the image where the tip box to be drawn on
+   * @param observer
+   * @param x is the horizontal x position of the mouse event
+   * @param y is the vertical y position  of the mouse event
+   * @param w is the max width of the image to be used
+   * @param h is the max height of the image to be used
+   * @return true if the tipbox is drawn on the offScreenImg, otherwise false
+   */
+  public boolean drawTipBox(Image offScreenImg, ImageObserver observer,
+    int x, int y, float w, float h)
+  {
+    Graphics2D g = (Graphics2D) offScreenImg.getGraphics();
+    ClusterRect tmpRect=null;
+
+    //-- find out which rectangle the mouse points to
+    try{
+      for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].contains(x,y))
+          for(int j=0;j<_cRects[i].length;j++)
+            if (_cRects[i][j].contains(x,y))
+            {
+              tmpRect = _cRects[i][j];
+              throw new Exception();
+            }  //-- if
+
+      //-- the mouse is not pointed to any of the rectangles
+      if (_previousRect==null) return false;
+      else
+      {
+        //-- the previous pict has a tip box => draw the orginal pict
+        _previousRect = null;
+        drawTreemapView(offScreenImg, observer);
+        return true;
+      }
+    }
+    catch(Exception e){
+      //-- the mouse is pointed to a cluster's rectangle
+
+      if (_previousRect==tmpRect)
+      {
+        //-- if the tip box is already on the graphic, we don't need to update
+        return false;
+      }
+      else
+      {
+        _previousRect = tmpRect;
+        drawTreemapView(offScreenImg, observer);
+        if (_previousRect!=null)
+        {
+          _previousRect.paintLabels(g, w, h);
+        }
+        return true;
+      }
+
+    }  //-- catch
+  }  //-- drawTipBox
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * find out which rectangle the mouse point to (for inspect)
+   * @param x is the x-cordinate of the mouse position
+   * @param y is the y-cordinate of the mouse position
+   * @return a cluster rectangle that the mouse pointed to
+   */
+  public ClusterRect inspect(int x, int y)
+  {
+    return findClusterRect(x, y, false);
+  }
+
+  /**
+   * find out which rectangle the mouse point to (for drill down)
+   * @param x is the x-cordinate of the mouse position
+   * @param y is the y-cordinate of the mouse position
+   * @return a cluster rectangle that the mouse pointed to
+   */
+  public ClusterRect drillDown(int x, int y)
+  {
+    return findClusterRect(x, y, true);
+  }
+
+  /**
+   * @param x is the horizontal x position of the mouse event
+   * @param y is the vertical y position  of the mouse event
+   * @param setChosenFlag
+   * @return the cluster rectangle that the mouse points to, and if the
+   *   'setChosenFlag' is true, then set up the corresponding flag in
+   *   _gpRect and _cRect
+   */
+  private ClusterRect findClusterRect(int x, int y, boolean setChosenFlag)
+  {
+    //-- find out which rectangle the mouse points to
+    for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].contains(x,y))
+            for(int j=0;j<_cRects[i].length;j++)
+                if (_cRects[i][j].contains(x,y))
+                {
+                    if (setChosenFlag)
+                    {
+                        _gpRects[i].setChosen();
+                        _cRects[i][j].setChosen();
+                    }
+                    return _cRects[i][j];
+                }  //-- if
+    return null;
+  }
+
+
+  //---------------------------------------------------------------------
+  /**
+   * !!! BUG - Color is not fixed !!!
+   * initialise _gpRects and _cRects
+   * create_sub_rects is called
+   */
+  public void createRects()
+  {
+    float area = _width*_height;  //-- the area of the display
+    int noOfGroup = _data.getClusterGroupsSize();
+    float totalDocs = (float) _data.getDocCount();
+    float lx=_upperX, ly=_upperY, rx=_upperX+_width, ry=_upperY+_height, tmp=0F;
+    Color baseColor = Color.yellow;
+    ClusterGroupObj gpObj;
+    int gpSize;
+
+    for (int i=0;i<noOfGroup; i++)
+    {
+        gpObj = _data.getClusterGroup(i);
+        gpSize = gpObj.getGroupSize();
+
+    //??? Color is not fixed !!!
+        if (i==0) baseColor=Color.cyan;
+        if (i==1) baseColor=Color.green;
+        if (i==2) baseColor=Color.blue;
+        if (i==3) baseColor=Color.orange;
+        if (i==4) baseColor=Color.pink;
+        if (i==5) baseColor=Color.red;
+
+      //-- West
+        if (rx>lx && ry>ly)
+        {
+            tmp = rx;
+            rx = ((float)gpSize/totalDocs)*area/(ry-ly)+lx;
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            lx = tmp;
+        }
+      //-- North
+        else if (lx>rx && ry>ly)
+        {
+            tmp = ry;
+            ry = ((float)gpSize/totalDocs)*area/(lx-rx)+ly;
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            ly = tmp;
+        }
+      //-- East
+        else if (lx>rx && ly>ry)
+        {
+            tmp = rx;
+            rx = lx - ((float)gpSize/totalDocs)*area/(ly-ry);
+
+            create_sub_rects(i, rx, ly, lx, ry,
+                gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            lx = tmp;
+        }
+      //-- South
+        else if (rx>lx && ly>ry)
+        {
+            tmp = ry;
+            ry = ly - ((float)gpSize/totalDocs)*area/(rx-lx);
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            ly = tmp;
+        }
+    } //-- for
+  }  //-- createRects
+
+
+  /**
+   * !!! BUG: Color hasn't fixed !!!
+   * create the rectangles for the clusters in the specified group
+   * @param gpNo is the index of the Group currently working on
+   * @param lx is the initial left x-coordinate (based on Rect coordination)
+   * @param ly is the initial left y-coordinate (ditto)
+   * @param rx is the initial right x-coordinate (ditto)
+   * @param ry is the initial right y-coordinate (ditto)
+   * @param cs is the list of clusters in the group
+   * @param total is the total number of documents found in the group
+   * @param col is the base colour of the group
+   */
+  private void create_sub_rects(int gpNo, float lx, float ly, float rx, float ry,
+                             ClusterObj[] cs, int total, Color col)
+  {
+
+    ZRect dummy = new ZRect(lx,ly,rx,ry,null);
+    float tmp, area = (float) (dummy.getWidth()*dummy.getHeight());
+
+    for(int i=0;i<cs.length;i++)
+    {
+
+     if (rx>lx && ry>ly)
+     {
+       tmp = rx;
+        rx = ((float)cs[i].getDocsSize()/(float)total)*area/(ry-ly)+lx;
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        lx = tmp;
+      }
+      //-- North
+      else if (lx>rx && ry>ly)
+      {
+        tmp = ry;
+        ry = ((float)cs[i].getDocsSize()/(float)total)*area/(lx-rx)+ly;
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        ly = tmp;
+      }
+      //-- East
+      else if (lx>rx && ly>ry)
+      {
+        tmp = rx;
+        rx = lx - ((float)cs[i].getDocsSize()/(float)total)*area/(ly-ry);
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        lx = tmp;
+      }
+      //-- South
+      else if (rx>lx && ly>ry)
+      {
+        tmp = ry;
+        ry = ly - ((float)cs[i].getDocsSize()/(float)total)*area/(rx-lx);
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        ly = tmp;
+      }
+
+//??? Color hasn't fixed !!!
+      col = darker(col);
+
+    }
+  }  //-- create_sub_rects
+
+
+    /**
+     * @see        java.awt.Color#darker
+     */
+    private Color darker(Color c) {
+    double FACTOR = 0.85;
+      return new Color(Math.max((int)(c.getRed()  *FACTOR), 0),
+       Math.max((int)(c.getGreen()*FACTOR), 0),
+       Math.max((int)(c.getBlue() *FACTOR), 0));
+    }
+
+
+
+  //---------------------------------------------------------------------
+  //-- pre: _zoomImages can't be null
+  public Image getLastZoomingImage()
+  {
+    return _zoomingImages[_zoomingImages.length-1];
+  }
+
+  public Image getZoomingImage(int i){return _zoomingImages[i];}
+
+
+  public void show_zoomIn(Graphics oldg, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    for(int i=0;i<_zoomingImages.length;i++)
+    {
+      //-- time delay for each frame
+      timeDelay();
+      g.drawImage(_zoomingImages[i],0,0,observer);
+    }
+    //-- this is time delay for the last frame and the new treemap view
+    timeDelay();
+  }
+
+
+  public void show_zoomOut(Graphics oldg, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    for(int i=_zoomingImages.length-1;i>=0;i--)
+    {
+      //-- time delay for each frame
+      timeDelay();
+      g.drawImage(_zoomingImages[i],0,0,observer);
+    }
+    //-- this is time delay for the last frame and the new treemap view
+    timeDelay();
+  }
+
+
+
+  private void timeDelay()
+  {
+    Thread delayThread = new Thread(this);
+    delayThread.start();
+    try{
+      delayThread.join();
+    }
+    catch (InterruptedException ex)
+    {
+    }
+  }
+
+
+  public void run()
+  {
+    long lastTime=0, time=0;
+    do
+    {
+      time = System.currentTimeMillis();
+      try{
+        Thread.sleep(__FRAMEDELAY);
+      }
+      catch (InterruptedException ex)
+      {
+      }
+      lastTime = time;
+    } while (time - lastTime > __FRAMEDELAY);
+
+  }
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * draw the animated zooming pictures on the buffer
+   * @param a list of images that is used for animation pictures
+   * @param observer
+   */
+  public void prepare_zooming(Image[] buffer, ImageObserver observer)
+  {
+    _zoomingImages = buffer;
+    prepare_zooming_buffer(observer);
+    create_zooming_picts();
+  }
+
+  /**
+   * draw the border on each buffer
+   * @param a list of buffer
+   */
+  private void prepare_zooming_buffer(ImageObserver observer)
+  {
+    Graphics2D g;
+    if (_zoomingImages==null)
+    {
+      System.err.println("Error - zooming buffer cannot be empty!!!");
+    }
+    else
+    {
+      //-- copy the static view to one buffer
+      g = (Graphics2D) _zoomingImages[0].getGraphics();
+      g.drawImage(_treemapView,0,0,observer);
+      //-- clear the main treemap area and leave the border
+      g.setColor(Color.white);
+      g.fill(new Rectangle.Float(_upperX, _upperY, _width, _height));
+      //-- copy that buffer to other buffers
+      for(int k=1;k<_zoomingImages.length;k++)
+      {
+        g = (Graphics2D) _zoomingImages[k].getGraphics();
+        g.drawImage(_zoomingImages[0],0,0,observer);
+      }
+    }
+  }
+
+
+  private void create_zooming_picts()
+  {
+    int[] edgeSize = {0,0,0,0};
+    float w_y, e_y, n_x, s_x, portion, tmp;
+    ZRect[] tmpRects;
+    Graphics tmpGph;
+    /**
+     * pre-calculate the edgeSize
+     * the edgeSize represents the total number of docs on each border
+     */
+    for(int i=0;i<_gpRects.length;i++)
+      if (!_gpRects[i].isChosen())
+        edgeSize[_gpRects[i].getDirection()] += _gpRects[i].getCapacity();
+      else
+        for(int j=0;j<_cRects[i].length;j++)
+          if(!_cRects[i][j].isChosen())
+            edgeSize[_gpRects[i].getDirection()] += _cRects[i][j].getCapacity();
+
+    //-- initise the starting points on each side
+    w_y=_upperY;                e_y=_upperY+__borderheight;
+    n_x=_upperX+__borderwidth;  s_x=_upperX;
+
+
+    for(int i=0;i<_gpRects.length;i++)
+    {
+      //-- prepare zooming the whole group, if the user has NOT chosen a cluster
+      //-- within the group
+      if (!_gpRects[i].isChosen())
+      {
+        portion = ((float)_gpRects[i].getCapacity()) /
+                  ((float) edgeSize[_gpRects[i].getDirection()]);
+        if (_gpRects[i].isWest())
+        {
+          tmp = w_y + (portion * (float)(_height-__borderheight));
+          _gpRects[i].translateTo(_upperX, w_y, _upperX+__borderwidth, tmp);
+          w_y = tmp;
+        }
+        else if (_gpRects[i].isEast())
+        {
+          tmp = e_y + (portion * (float)(_height-__borderheight));
+          _gpRects[i].translateTo(_upperX+_width-__borderwidth, e_y, _upperX+_width, tmp);
+          e_y = tmp;
+        }
+        else if (_gpRects[i].isNorth())
+        {
+          tmp = n_x + (portion * (float)(_width-__borderwidth));
+          _gpRects[i].translateTo(n_x, _upperY, tmp, _upperY+__borderheight);
+          n_x = tmp;
+
+        }
+        else
+        {
+          tmp = s_x + (portion * (float)(_width-__borderwidth));
+          _gpRects[i].translateTo(s_x, _upperY+_height-__borderheight, tmp, _upperY+_height);
+          s_x = tmp;
+        }
+
+        //-- generate a list of animation picture
+        tmpRects = _gpRects[i].zooming(_zoomingImages.length);
+
+        //-- draw the rectangles on the offsreen buffer
+        for(int j=0;j<_zoomingImages.length;j++)
+          tmpRects[j].paint(_zoomingImages[j].getGraphics());
+
+      }
+      else
+        //-- prepare zooming each cluster within the group
+        for (int j=0;j<_cRects[i].length;j++)
+          if (!_cRects[i][j].isChosen())
+          {
+            portion = ((float) _cRects[i][j].getCapacity()) /
+                      ((float) edgeSize[_gpRects[i].getDirection()]);
+            if (_gpRects[i].isWest())
+            {
+              tmp = w_y + (portion * (float)(_height-__borderheight));
+              _cRects[i][j].translateTo(_upperX, w_y, _upperX+__borderwidth, tmp);
+              w_y = tmp;
+            }
+            else if (_gpRects[i].isEast())
+            {
+              tmp = e_y + (portion * (float)(_height-__borderheight));
+              _cRects[i][j].translateTo(_upperX+_width-__borderwidth, e_y, _upperX+_width, tmp);
+              e_y = tmp;
+            }
+            else if (_gpRects[i].isNorth())
+            {
+              tmp = n_x + (portion * (float)(_width-__borderwidth));
+              _cRects[i][j].translateTo(n_x, _upperY, tmp, _upperY+__borderheight);
+              n_x = tmp;
+            }
+            else
+            {
+              tmp = s_x + (portion * (float)(_width-__borderwidth));
+              _cRects[i][j].translateTo(s_x, _upperY+_height-__borderheight, tmp, _upperY+_height);
+              s_x = tmp;
+            }  //-- if
+
+            //-- generate a list of animation picture
+            tmpRects = _cRects[i][j].zooming(_zoomingImages.length);
+
+            //-- draw the rectangles on the offsreen buffer
+            for(int k=0;k<_zoomingImages.length;k++)
+              tmpRects[k].paint(_zoomingImages[k].getGraphics());
+          }  //-- if
+    }  //-- for
+  }  //-- prepare_zooming
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Cluster.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Cluster.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Cluster.java	(revision 14150)
@@ -0,0 +1,117 @@
+package vishnu.testvis.visual;
+
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.lang.*;
+import java.util.zip.*;
+
+class Cluster
+{
+    static final int NOVALUE = 99999999;
+    static int MAX;
+    static final int EMPTYARRAY[] = {};
+    static int matrix[][];
+    static int data[][];
+    static int clusters[][];
+    static int keywords[];
+    static int maxDocuments;
+    static int finalCluster;
+
+    static void buildMatrix ()
+    {
+	float temp;
+	float theDiff;
+	int keyword1, keyword2;
+    	matrix = new int[MAX][MAX];
+        for (int word1=0;word1<MAX;word1++)
+        {
+	    keyword1 = keywords[word1];
+	    for (int word2=0; word2<MAX;word2++)
+            {
+		keyword2 = keywords[word2];
+                theDiff = 0;
+                for (int doc=0;doc<maxDocuments;doc++)
+                {	
+		    temp= data[doc][keyword1]-data[doc][keyword2];
+                    theDiff+= temp*temp;
+                }
+                temp = (int)Math.sqrt(theDiff);
+                matrix[word1][word2]=matrix[word2][word1]=(int)temp;
+            }
+         }
+	for (int row = 0;row <MAX; row++)
+	   matrix[row][row] = NOVALUE;
+    } 
+
+    static void findSmallest ()
+    {
+	int group = matrix[0].length-1;
+	int smallest = NOVALUE;
+	int smallRow = NOVALUE;
+	int smallCol = NOVALUE;
+	int row,col;
+	for (row =0; row<MAX; row++)
+	    for (col=0; col<MAX; col++)
+	       if (smallest > matrix[row][col])
+	        {
+		   smallest = matrix[row][col];
+		   smallRow = row;	
+		   smallCol = col;
+ 	        }
+	for (col=0; col<MAX; col++)
+	{
+	    if (matrix[smallRow][col] < matrix[smallCol][col])
+		matrix[smallRow][col] = matrix[smallCol][col];
+            matrix[smallCol][col] = NOVALUE;
+	}
+	addClusters(smallRow, smallCol);
+	for (row =0; row<MAX; row++)
+        {
+	    matrix[row][smallCol] = NOVALUE;
+            matrix[row][smallRow] = matrix[smallRow][row];
+	} 	
+    }
+
+    static void addClusters(int destination, int source)
+    {
+	int i;
+	int temp[] = new int[clusters[destination].length+clusters[source].length];
+	for (i = 0; i<clusters[destination].length;i++)
+	    temp[i] = clusters[destination][i];
+	for (i = 0; i<clusters[source].length;i++)
+	    temp[i+clusters[destination].length] = clusters[source][i];
+	clusters[destination] = temp;
+	clusters[source] = EMPTYARRAY;
+	finalCluster = destination;
+    }
+
+    static void preClusters()
+    {
+	clusters = new int[MAX][1];
+	for (int row =0; row <MAX; row++)
+	{
+	    clusters[row][0] = row;
+	}
+    }
+
+    static int[] maximunDistance (int[][] newData, int[] newKeywords, int documentCount)
+    {
+	data = newData;
+	keywords = newKeywords;
+	maxDocuments = documentCount;
+	MAX = keywords.length;
+	buildMatrix();
+	preClusters();
+	for (int i=1;i<MAX;i++)
+	{
+ 	    findSmallest();
+        }
+	int temp[] = new int[MAX];
+	for (int i=1;i<MAX;i++)
+	    temp[i] = keywords[clusters[finalCluster][i]];
+	return temp;
+    }
+}
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/CvManyPoints.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/CvManyPoints.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/CvManyPoints.java	(revision 14150)
@@ -0,0 +1,1124 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.applet.*;
+import java.awt.event.*;
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.awt.Cursor;
+import vishnu.testvis.object.*;
+import vishnu.datablock.*;
+
+
+public class CvManyPoints extends Canvas
+{
+    protected Vishnu displayFrame;
+    protected RadVizPanel radVizPanel;
+    protected DataManager dataManager;
+    protected int [] docs;
+    
+    /** constant square shape flag */
+    static final int SQUARESHAPE = 0;
+    /** constant round shape flag */
+    static final int ROUNDSHAPE = 1;
+    /** constant diamond shape flag */
+    static final int DIAMONDSHAPE = 2;
+    /** constant maximum keywords allowed */
+    static final int ABSMAXKEYWORD = 1000;
+    /** constant maximum documents allowed */
+    static final int ABSMAXDOC = 1000;
+    /** count of the documents */
+    protected int documentCount = 1000;
+    /** circle array size */
+    protected int maxCircleCount = 40;
+    /** number of circles */
+    protected int circleCount = 40;
+    /** number of spokes */
+    protected int spokeCount = 16;
+    /** radius of the display in fp units */
+    protected float r = 10.0F;
+    /** locked flag */
+    protected boolean isLocked = false;
+    /** projected flag */
+    protected boolean isProjected = true;
+    /** document selected flag */
+    protected boolean atDoc = false;
+    /** grid flag */
+    protected boolean displayGrid = false;
+    /** rim flag */
+    protected boolean displayRim = true;
+    /** doc point shape flag */
+    protected int docsShape = SQUARESHAPE;
+    /** edge node shape flag */
+    protected int nodesShape = ROUNDSHAPE;
+    /** colour of nodes */
+    protected Color defNodesColor = Color.black;
+    /** colour of docs */
+    protected Color defDocsColor = Color.orange;
+    /** colour of highlighted nodes */
+    protected Color defHighlight = Color.blue;
+    /** colour of selected documents */
+    protected Color defSelect = Color.red;
+    /** Radius divided by radians */
+    protected float magic = (float)((double)r / (Math.PI / 2.0) );
+    /** projection factor */
+    protected float factor = 3F;
+    /** image object used for double buffering graphics */
+    protected Image image=null;
+    /** graphics used for double buffering graphics */
+    protected Graphics gImage=null;
+    /** double buffered image size */
+    protected int imgW,imgH;
+    /** dimensions of the canvas */
+    protected Dimension dim;
+    /** the integer location of 0,0 fp axis */
+    protected int centerX, centerY;
+    /** virtual pixel size and canvas fp dimensions */
+    protected float pixelSize, rWidth = 25.0F, rHeight = 25.0F;
+    /** array of edge node locations */
+    protected Point2D[] points= null;
+    /** selected edge node */
+    protected Point2D tp =null;
+    /** selected edge node index */
+    protected int tpIndex = 0;
+    /** angle stored for rotating edge nodes */
+    protected float lastAngle;
+    /** Full 2D linked list of data */
+    
+    protected Pt2DNode matrix= null;
+    /** Array of partial lists for each key word */
+    protected Pt2DNode columns[] = null;
+    /** The currently selected document if any */
+    protected Pt2DNode selectDoc = null;
+    /** A class that stores the keywords and whether they are displayed */
+    protected KeyWordArray keyWordArray;
+    /** Array of linked lists that stores the data to be displayed */
+    protected SparseMatrix theDataMatrix;
+    /** A class that maps displayed keywords to stored keywords */
+    protected MapKeyWords mapping;
+    /** A two dimensional array of integers used for clustering */
+    //protected int [][] fullMatrix;
+    protected float [][] fullMatrix;
+    /** array of data to draw target */
+    protected int circles[][] = new int[maxCircleCount][4];
+    /** array of data to draw rim */
+    protected int rim[] = new int[4];
+    /** array of data to draw spokes */
+    protected int spokes[][] = new int [spokeCount][2];
+    /** font for the mouse 'balloon' */
+    protected Font balloonFont = null;
+    /** metrics of the font for the mouse 'balloon' */
+    protected FontMetrics balloonFontMetrics = null;
+    /** Flag if drawing needs to just pasted from the buffer or redrawn */
+    protected boolean isDirty = true;
+    /** Scale the non-projected output if it goes off the page */
+    protected float scale=1.0F;
+    /** Last scale used when circles calculated (Flat mode only) */
+    protected float lastCircleScale = 1.0F;
+
+    /**
+     *  Rotate a Point2D around the 0.0,0.0 axis by phi angle (in radians)
+     *  @param Point2D around the 0.0,0.0 axis by
+     *  @param phi angle (in radians)
+     */
+    protected void rotatePt(Point2D pt,float phi)
+    {
+        float c = (float)Math.cos(phi), s = (float)Math.sin(phi),
+            r11 = c, r12 = s, r21 = -s, r22 = c;
+        float xNew = pt.x * r11 + pt.y * r21,
+            yNew = pt.x * r12 + pt.y * r22;
+        pt.x = xNew; pt.y = yNew;
+	
+    }
+    /**
+    * This is a public gateway for export of the keyWordsArray class
+    */
+    public KeyWordArray getKeyWords()
+    {
+        return keyWordArray;
+    }
+
+
+    /**
+    *  Fill the integer array circles[circleCount][4] with the correct numbers for
+    *  circleCount concentric circles about the 0.0,0.0 axis
+    */
+
+    protected void setCircles()
+    {
+        float  gap =r/4.0F, t = r/4.0F;
+        rim[0]=iX(-r); rim[1]=iY(r);
+        rim[2]= (int) (2.0F * r / pixelSize);
+        rim[3] = rim[2];
+        for (int count=0;count < circleCount; count++)
+        {
+            Point2D pc = new Point2D(-t,0.0F);
+            projection(pc);
+            circles[count][0]=iX(pc.x);
+            circles[count][2]=(int)((-pc.x) * 2.0F /pixelSize);
+            pc.x=0.0F; pc.y= t;
+            projection(pc);
+            circles[count][1]=iY(pc.y);
+            circles[count][3]=(int)(pc.y * 2.0F /pixelSize);
+            t += gap;
+        }
+        if (!isProjected) lastCircleScale=scale;
+    }
+
+    /** Create an array of linked lists that point only to the tuples affected by
+    *   each particular node, thus allowing faster update of the the locations
+    *   and easy display of only the tuples on that column of data, the membership
+    *   of these lists is subset of the larger list at 'matrix'
+    */
+    protected void setColumns()
+    {
+        int keyWordCount=keyWordArray.getNumWords();
+        columns = new Pt2DNode[keyWordCount];
+        Pt2DNode tails[] = new Pt2DNode[keyWordCount];
+        Pt2DNode k = null;
+        for (int i = 0; i< keyWordCount;i++) tails[i] = columns[i]=null;
+
+        if(keyWordCount==0) return;
+        Pt2DNode m = matrix;
+        SparseNode j=null;
+
+        // This may appear a bit obsure;
+
+        // columns[] is the start of an array of linked lists representing
+        // each of the keywords. tails[] is a parallel linked list representing the
+        // last added node to the columns[] lists;
+
+        // m loops through the rows and j iterates through the columns of each row
+        // when j finds a reference to a particular column[index] row m is at is added to
+        // the linked list for that column[index]
+
+        while (m != null)
+        {
+            j = m.tuple.row;
+            while (j != null)
+            {
+                k = new Pt2DNode(m.tuple);
+                int column = j.getColumn();
+                if(column > keyWordCount)
+                    System.out.println("col "+ column + " of " + keyWordCount);
+                else
+                {
+                    if (tails[column]!=null)
+                        tails[column].setNext(k);
+                    else columns[column] = k;
+                    tails[column]=k;
+                }
+                j=j.getNext();
+            }
+            m=m.getNext();
+        }
+    }
+    
+    /** Build a linked list of Points and tuples down connected to lists of nodes in the
+     * tuples across.
+     * @param dataItems the list
+     * @param refs[] the document titles
+     * @param docCount the number of documents
+     * @param wordCount the number of visible words
+     */
+    protected void setDatalists(SparseMatrix dataItems, String refs[],int docCount, int wordCount)
+    {
+        Pt2DNode m = null, n=null;
+        boolean state;
+        for (int co=0;co<docCount;co++)
+        {
+            SparseNode start = dataItems.getRow(co);
+            Point2D pn = new Point2D(0.0F,0.0F);
+            state = dataItemXY(pn,start);
+
+            n = new Pt2DNode(pn.x,pn.y,co,start,refs[co]);
+            if (state) projection(n.tuple.pnt);
+            n.tuple.state =state;
+            if (state)
+            {
+                n.tuple.iPnt.x=iX(n.tuple.pnt.x);
+                n.tuple.iPnt.y=iY(n.tuple.pnt.y);
+            }
+            if (m != null) m.setNext(n);
+            else matrix = n;
+            m=n;
+        }
+        setColumns();
+    }
+
+    /** Travers the linked lists setting new location values for the document points
+     * @param ma a linked list of document nodes
+     */
+    protected boolean setDocPoints(Pt2DNode ma)
+    {
+        while (ma != null)
+        {
+            ma.tuple.state = dataItemXY(ma.tuple.pnt,ma.tuple.row);
+            if (ma.tuple.state)
+            {
+                //c++;
+                projection(ma.tuple.pnt);
+                ma.tuple.iPnt.x=iX(ma.tuple.pnt.x);
+                ma.tuple.iPnt.y=iY(ma.tuple.pnt.y);
+                if (ma.tuple.iPnt.x<0 || ma.tuple.iPnt.x>dim.width ||
+                    ma.tuple.iPnt.y<0 || ma.tuple.iPnt.y>dim.height)
+                    return  true;
+            }
+            ma=ma.getNext();
+        }
+        return false;
+    }
+
+    /** Fill the integer array spokes[16][2] with the correct numbers for
+    *   16 radial points on the outermost circle, by defining a point on
+    *  the circle and rotating it by PI/8 radians 16 times
+    */
+    protected void setSpokes()
+    {
+        float phi=(float)Math.PI/(float)(spokeCount/2);
+        Point2D pt = new Point2D(0.0F,r);
+        for (int count=0;count < spokeCount;count++)
+        {
+            spokes[count][0]=iX(pt.x);
+            spokes[count][1]=iY(pt.y);
+            rotatePt(pt,phi);
+        }
+    }
+
+    /** set the key word nodes in an even circle round the rim */
+    protected void setKeyNodes()
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+        float phi=(float)Math.PI/((float)keyWordCount/2.0F);
+        Point2D pt = new Point2D(0.0F,r);
+        for (int count=0;count < keyWordCount;count++)
+	    {
+		points[count] = new Point2D(pt.x,pt.y);
+		rotatePt(pt,phi);
+	    }
+    }
+    
+    /** find the x y coordinate of a tuple of data
+    *   @param theP point to be set
+    *   @param tuple linked list of data
+    *   @return true if tuple contains a reference to a visible keynode
+    */
+    protected boolean dataItemXY(Point2D theP, SparseNode tuple)
+    {
+        int index;
+        boolean result=false;
+        theP.x=0.0F; theP.y = 0.0F;
+        while (tuple != null)
+        {
+            if (keyWordArray.getState(tuple.getColumn()))
+            {
+                result = true;
+                index = mapping.mapWordsToNodes(tuple.getColumn());
+                theP.x += tuple.getValue()/r * points[index].x;
+                theP.y += tuple.getValue()/r * points[index].y;
+            }
+            tuple=tuple.getNext();
+        }
+        return result;
+    }
+
+    /** set the text balloon label at the correct location, to the correct size
+    *   and with the correct text
+    *   @param theString the text to display
+    *   @param x the x coordinate of the mouse
+    *   @param y the y coordinate of the mouse
+    */
+    void setBalloon(String theString,int x,int y)
+    {
+        Rectangle bounds1 = getBounds();
+	
+        x-=(bounds1.x)-30; y+=(bounds1.y)+20;
+	
+        radVizPanel.docLabel.setText(theString);
+        int width = balloonFontMetrics.stringWidth(theString)+2;
+        int height = balloonFontMetrics.getMaxAscent()+2;
+        radVizPanel.docLabel.setBounds(x,y,width,height);
+    }
+
+    /**  recalculate edge points and document positions in the light of
+    *   of changes to the keywords
+    */
+    public void recalc()
+    {
+        mapping = new MapKeyWords(keyWordArray);
+        tp =null;
+        scale=1.0F;
+        tpIndex = 0;
+        points= new Point2D[keyWordArray.getNumEnabled()];
+        setKeyNodes();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+        repaint();
+    }
+
+    /**
+    * constructor sets everything up gets creates an instance of DataLink and
+    * gets the data matrix and the keywords calls code to initialise display
+    * @param DisplayFrame the central object of the display
+    * @param width of canvas
+    * @param height of canvas
+    * @throws IOException
+    */
+    public CvManyPoints(Vishnu df) throws IOException
+    {
+        setVisible(false);
+       	this.displayFrame = df;
+        this.radVizPanel = displayFrame.radVizPanel;
+        this.dataManager = displayFrame.dataManager;
+	
+       	documentCount=0;
+       	int keyWordCount=0;
+        keyWordArray = new KeyWordArray();
+	
+       	initgr();
+       	setCircles();
+       	setSpokes();
+       	/**
+       	* add an anonymous class to listen for size changed events
+       	*/
+	
+        addComponentListener(new ComponentAdapter(){
+		public void componentResized(ComponentEvent evt)
+		{
+		    initgr();
+		    setCircles();
+		    setSpokes();
+		    while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+		    repaint();
+		}
+	    });
+	
+        // From Java in a Nutshell
+        // listen for mouse button pressed event and if on a node point
+        // record the current angle and set the value of tp
+        /**
+	 * add an anonomous class to listen for mouse pressed events
+	 */
+        addMouseListener(new MouseAdapter()
+	    {
+		public void mousePressed(MouseEvent evt)
+		{
+		    int intX = evt.getX();
+		    int intY = evt.getY();
+		    float x = fx(intX), y = fy(intY);
+		    tp = findPoint(x,y);
+		    if (tp == null){
+			selectDoc = findDoc(intX, intY);
+			if (selectDoc != null){
+			    StringBuffer buf = new StringBuffer("Doc: ");
+			    buf.append(selectDoc.tuple.ref);
+			    buf.append("  ");
+			    if (buf.length() > 30){
+				buf.setLength(30);
+				buf.append("... ");
+			    }
+			    String theString = new String(buf);
+			    //theOuterFrame.setDocSelectMenu(theString,selectDoc.tuple.index);
+			}   
+			//else theOuterFrame.setDocSelectMenu("No Document Selected      ",-1);
+			repaint();
+			return;
+		    }
+		    lastAngle=(float)Math.atan2(x,y);
+		    repaint();
+		}
+		
+		public void mouseClicked(MouseEvent event)
+		{
+		    int intX = event.getX();
+		    int intY = event.getY();
+		    
+		    selectDoc = findDoc(intX, intY);
+		    if (selectDoc != null){
+			if(event.getClickCount()>1){
+			    StringBuffer buf = new StringBuffer("Doc: ");
+			    buf.append(selectDoc.tuple.ref);
+			    buf.append("  ");
+			    if (buf.length() > 30){
+				buf.setLength(30);
+				buf.append("... ");
+			    }
+			    String theString = new String(buf);
+			    //theOuterFrame.setDocSelectMenu(theString,selectDoc.tuple.index);
+			    dataManager.fetch(selectDoc.tuple.index);
+			}
+		    }
+		}
+		
+		public void mouseReleased(MouseEvent event)
+		{
+		    Cursor releaseCursor = new Cursor(Cursor.DEFAULT_CURSOR);
+		    displayFrame.radVizPanel.setCursor(releaseCursor);
+		}
+	    });
+	
+        // listen for mouse button dragged event and if tp is set
+        // work out a new position for tp and the data points
+        /**
+	 * add an anonomous class to listen for mouse motion events
+	 */
+        addMouseMotionListener(new MouseMotionAdapter(){
+		public void mouseDragged(MouseEvent evt)
+		{
+		    Cursor dragCursor = new Cursor(Cursor.HAND_CURSOR);
+		    displayFrame.radVizPanel.setCursor(dragCursor);
+		    
+		    float x = fx(evt.getX()), y = fy(evt.getY());
+		    if (tp!=null){
+			float thisAngle = (float)Math.atan2(x,y);
+			if(isLocked){
+			    float phi = lastAngle-thisAngle;
+			    rotatePt(tp, phi);
+			}
+			else{
+			    tp.x = x;
+			    tp.y = y;
+			}
+			if (columns!= null){
+			    int index = mapping.mapNodesToWords(tpIndex);
+			    if (columns[index]!= null){
+				if(setDocPoints( columns[index] ) && !isProjected){
+				    do{
+					scale*=0.9;
+				    } while(setDocPoints( matrix ));
+				}
+				repaint();
+			    }
+			}
+			lastAngle=thisAngle;
+		    }
+		}
+		
+		
+		
+		/** listen for mouse movement and try and find a document
+		 * report its presence with a balloon label or ensure
+		 * the balloon label is hidden
+		 * @param evt the mouse event
+		 */
+		public void mouseMoved(MouseEvent evt)
+		{
+		    System.out.println("Moving");
+		    /* int intX = evt.getX();
+		    int intY = evt.getY();
+		    Pt2DNode theNode = findDoc(intX, intY);
+		    
+		    if (theNode != null){
+			if (atDoc) radVizPanel.docLabel.setVisible(false);
+			String theString = new String("Doc: " + theNode.tuple.ref);
+			setBalloon(theNode.tuple.ref,intX,intY);
+			radVizPanel.docLabel.setVisible(true);
+			atDoc = true;
+		    }
+		    else{
+			if (atDoc){
+			    radVizPanel.docLabel.setVisible(false);
+			}
+			atDoc = false;
+			}*/
+		    return;
+		}
+	    });
+    }
+
+    /* the Data loading happens in here now */
+    
+    public void update()
+    {
+        int maxWordsCount;
+        this.radVizPanel = displayFrame.radVizPanel;
+        this.dataManager = displayFrame.dataManager;
+
+        //if(dataManager.getDocCount()==0)return;
+        selectDoc = null;
+        tp =null;
+       	docs = dataManager.getDocs();
+       	documentCount = docs.length;
+        String [] words = dataManager.getWords();
+        maxWordsCount = words.length;
+        //maxWordsCount= in.setKeywordCount(maxWordsCount);
+        int keyWordCount = Math.min(maxWordsCount,12);
+       	keyWordArray = new KeyWordArray(words,words.length);
+        maxWordsCount = keyWordArray.getNumWords();
+
+
+       	mapping = new MapKeyWords(keyWordArray);
+
+        String titles[] = dataManager.getDescriptions();
+        if(titles==null) System.out.println("no descriptions read in");
+        points= new Point2D[keyWordCount];
+        balloonFont = new Font("Dialog", Font.PLAIN, 16);
+        balloonFontMetrics = getFontMetrics(balloonFont);
+        theDataMatrix = dataManager.getMatrix();
+        fullMatrix = theDataMatrix.getExpanded();
+        setKeyNodes();
+        setDatalists(theDataMatrix,titles,documentCount,maxWordsCount);
+        recalc();
+    }
+    /** Cluster the words round the edge
+    * @param type the type of clustering
+    */
+    public void cluster()
+    {
+        if(documentCount ==0) return;
+        int keyWordCount=keyWordArray.getNumEnabled();
+        int keylist[] = new int[keyWordCount];
+
+        for (int c=0;c<keylist.length;c++)
+        {
+            keylist[c]=mapping.mapNodesToWords(c);
+        }
+        if(fullMatrix==null) System.out.println("full matrix is null");
+        else
+        {
+            if(fullMatrix[0]==null) System.out.println("full matrix[0] is null");
+        }
+
+        keylist = HitCluster.maximunDistance (fullMatrix, keylist, documentCount);
+
+        mapping.reMapKeyWords(keylist,keyWordArray);
+
+        tp =null;
+        tpIndex = 0;
+        points= new Point2D[keyWordArray.getNumEnabled()];
+        setKeyNodes();
+        while (setDocPoints(matrix) && !isProjected) scale *=0.9;
+        repaint();
+    }
+    /**
+    * Overides setSize so that the drawing is correctly scaled on a setSize call
+    * @param new dimensions
+    */
+    public void setSize(Dimension d)
+    {
+        super.setSize(d);
+        initgr();
+        setCircles();
+        setSpokes();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+    }
+    /**
+    * Overides setSize so that the drawing is correctly scaled on a setSize call
+    * @param new width
+    * @param new height
+    */
+    public void setSize(int x, int y)
+    {
+        Dimension d = new Dimension(x,y);
+        setSize(d);
+    }
+
+
+
+    /**
+    * Tidy up the image before canvas is destroyed a bit like a destructor in C
+    */
+    public void dispose()
+    {
+        gImage.dispose();
+    }
+
+
+    /** Set the shape of the edge word nodes
+    * @param state selects the shape
+    */
+    void setNodeShape(int state)
+    {
+        nodesShape = state;
+        repaint();
+    }
+
+    /** Set the shape of the centre document nodes
+    *  @param state selects the shape
+    */
+    void setDocShape(int state)
+    {
+        docsShape = state;
+        repaint();
+    }
+
+    /** switches the projection on and off
+    * @param state is the switch
+    */
+    public void projection(boolean state)
+    {
+        // to do: code goes here.
+        isProjected = state;
+        circleCount = state?40:4;
+        setCircles();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9;
+        repaint();
+    }
+
+    /**
+    * Display or not the grid
+    * @param switches on and off
+    */
+
+    public void grid(boolean state)
+    {
+        displayGrid=state;
+        repaint();
+    }
+
+    /**
+    * Display or not the rim
+    * @param switches on and off
+    */
+    public void rim(boolean state)
+    {
+        displayRim=state;
+        repaint();
+    }
+
+    /**
+    * Lock or not the edge nodes so they can only be moved in a circle
+    * @param switches on and off
+    */
+    public void lock(boolean state)
+    {
+        isLocked = state;
+    }
+
+
+    /**
+    * Flattens the projection slightly
+    *
+    */
+    public void flatter()
+    {
+        if(!isProjected)
+            scale/=0.9;
+        else
+        {
+            factor *=0.9;
+            setCircles();
+        }
+        while (setDocPoints(matrix)&& !isProjected) scale*=0.9;
+        repaint();
+    }
+
+    /**
+    * Heightens the projection slightly
+    *
+    */
+    public void higher()
+    {
+        if(!isProjected)
+            scale*=0.9;
+        else
+        {
+            factor /=0.9;
+            setCircles();
+        }
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9;
+        repaint();
+    }
+
+
+
+
+    /**
+    * Translate to integer raster coordinate X from
+    * floating point coord X based around a notional centre
+    * @param x floating point x coordinate
+    * @return integer X coordinate
+    */
+    int iX(float x){return Math.round(centerX + x/pixelSize);}
+    /**
+    * Translate to integer raster coordinate Y from
+    * floating point coord Y based around a notional centre
+    * @param y floating point y coordinate
+    * @return integer Y coordinate
+    */
+    int iY(float y){return Math.round(centerY - y/pixelSize);}
+    /**
+    * Translate from integer raster coordinate X to
+    * floating point coord X based around a notional centre
+    * @param x integer x coordinate
+    * @return floating point X coordinate
+    */
+    float fx(int X){return (X - centerX) * pixelSize;}
+    /**
+    * Translate from integer raster coordinate Y to
+    * floating point coord Y based around a notional centre
+    * @param y integer y coordinate
+    * @return floating point Y coordinate
+    */
+    float fy(int Y){return (centerY - Y) * pixelSize;}
+
+    /**
+    * find if an edge keyword node is at the x y location
+    * @param x fp coordinate
+    * @param y fp coordinate
+    * @return fp Point at location or null
+    */
+    public Point2D findPoint(float x, float y)
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+        for (int i=0;i<keyWordCount;i++)
+        {
+            int intX = iX(x), intY = iY(y),
+                lX = iX(points[i].x)-3, hX = lX+6,
+                lY = iY(points[i].y)-3, hY = lY+6;
+
+            if (intX > lX && intX < hX && intY > lY && intY < hY)
+            {
+
+                tpIndex = i;
+                return points[i];
+            }
+        }
+        return null;
+    }
+
+    /**
+    * find if a Doc node is at the x y location
+    * @param x int coordinate
+    * @param y int coordinate
+    * @return fp Point at location or null
+    */
+    public Pt2DNode findDoc(int intX, int intY)
+    {
+        Pt2DNode thePnt=matrix;
+        while (thePnt!= null)
+        {
+            if (thePnt.tuple.state)
+            {
+                int   lX = thePnt.tuple.iPnt.x-3, hX = lX+6,
+                    lY = thePnt.tuple.iPnt.y-3, hY = lY+6;
+                if (intX > lX && intX < hX && intY > lY && intY < hY) return thePnt;
+            }
+            thePnt = thePnt.getNext();
+        }
+        return null;
+    }
+    /** project the point radially from the centre using the arctan function
+    *  if isProjected is true or scale them it false
+    *  @param point to be projected
+    *  @param ingore scaling flag
+    */
+    protected void projection(Point2D p)
+    {
+        if (!isProjected)
+        {
+            if(scale!=1.0)
+            {
+                p.x*=scale;
+                p.y*=scale;
+            }
+            return;
+        }
+        float newRad;
+        float rad = (float)Math.sqrt(p.x*p.x+p.y*p.y);
+        float sine = p.y/rad;
+        float cosine = p.x/rad;
+        newRad =(float)Math.atan(rad/(factor*Math.PI))*magic;
+        p.x = newRad * cosine;
+        p.y = newRad * sine;
+    }
+
+    /** set up graphic settings such as centre and pixelsize
+    */
+    protected void initgr()
+    {
+        dim = getSize();
+        int maxX = dim.width - 1, maxY = dim.height - 1;
+        pixelSize = Math.max(rWidth/maxX, rHeight/maxY);
+        centerX = maxX/2; centerY = maxY/2;
+    }
+
+
+    /////////Athina has been here!!!!!///////////////////////////////////////
+
+    /** draw a for document node shape at raster x y
+    *   @param x location
+    *   @param y location
+    *   @param g Graphics object
+    */
+    protected void drawItemPoint(int x, int y, Graphics g)
+    {
+
+        if (docsShape == SQUARESHAPE)
+            oval(g,x,y);
+        else if (docsShape == ROUNDSHAPE)
+            rect(g,x,y);
+        else
+            diamond(g,x,y);
+
+    }
+
+    /**
+    *   draw a for keyword node shape at raster x y
+    *   @param x location
+    *   @param y location
+    *   @param g Graphics object
+    */
+    void drawNodes(int x, int y, Graphics g)
+    {
+        if (nodesShape == 0)
+            oval(g,x,y);
+        else if (nodesShape == 1)
+            rect(g,x,y);
+        else
+            diamond(g,x,y);
+
+    }
+
+
+    /**
+    *  draw the key name near the edgenode
+    *  @param g Graphics class
+    *  @param count index of the name to draw
+    *  @param select if the node is selected.
+    */
+    void drawKeyName(Graphics g, int count, boolean select)
+    {
+        FontMetrics fontMetrics = g.getFontMetrics();
+        Font font = new Font("arial", Font.PLAIN, 16);
+        String word = keyWordArray.getWord(mapping.mapNodesToWords(count));
+        int posX= iX(points[count].x)-2, posY= iY(points[count].y)-2;
+        int width = fontMetrics.stringWidth(word);
+        if (points[count].x < 0.0)
+            posX -= width - (int) (points[count].x/2.00);
+        else
+            posX += (int) (points[count].x/2.00);
+
+        if (points[count].y < 0.0)
+            posY += fontMetrics.getMaxAscent()- (int)(points[count].y/2.00);
+        else
+            posY -= (int) (points[count].y/2.00);
+        g.setFont(font);
+        g.drawString(word,posX,posY);
+        if (select){ g.drawRect(posX-2,posY-15,width+4,20);}
+    }
+
+    /**
+    * draw a box at the edge nodes
+    * @param g Graphics object
+    */
+    protected void drawEdgeNodes(Graphics g)
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+
+        for (int count=0;count < keyWordCount; count++)
+        {
+            drawNodes(iX(points[count].x), iY(points[count].y), g);
+            drawKeyName(g,count,false);
+
+        }
+    }
+    /**
+    * Draw the documents that have the selected key word in them
+    * @param g Graphics object
+    */
+    protected void drawSelect(Graphics g)
+    {
+        if (tp!=null)
+        {
+            int x = iX(points[tpIndex].x), y = iY(points[tpIndex].y);
+            g.setColor(defSelect);
+            drawNodes(x, y, g);
+
+            Pt2DNode m = columns[mapping.mapNodesToWords(tpIndex)];
+            while (m != null)
+            {
+                x=m.tuple.iPnt.x;
+                y=m.tuple.iPnt.y;
+
+                drawItemPoint(x, y, g);
+
+                m=m.getNext();
+            }
+            g.setColor(defDocsColor);
+
+        }
+    }
+    /**
+    * draw the key word nodes that feature in the selected document
+    */
+    protected void drawDocSelect(Graphics g)
+    {
+        if (selectDoc==null) return;
+        g.setColor(defHighlight);
+        int x =selectDoc.tuple.iPnt.x, y =selectDoc.tuple.iPnt.y;
+        drawItemPoint(x, y, g);
+
+        SparseNode fn = selectDoc.tuple.row;
+        while (fn != null)
+        {
+            if (keyWordArray.getState(fn.getColumn()))
+            {
+                int index = mapping.mapWordsToNodes(fn.getColumn());
+                drawNodes(iX(points[index].x), iY(points[index].y), g);
+                drawKeyName(g,index,true);
+            }
+            fn=fn.getNext();
+
+        }
+        g.setColor(defNodesColor);
+
+    }
+
+    /**
+    * draw all the document crosses
+    * @param g Graphics object
+    */
+
+    protected void drawDocXs(Graphics g)
+    {
+        Pt2DNode m = matrix;
+        while (m != null)
+        {
+            if (m.tuple.state) drawItemPoint(m.tuple.iPnt.x, m.tuple.iPnt.y, g);
+            m=m.getNext();
+        }
+    }
+
+    /**
+    * draw the target
+    * @param g Graphics object
+    */
+
+    protected void drawTarget(Graphics g)
+    {
+        if (!isProjected && lastCircleScale!=scale)
+            setCircles();
+
+        if (displayRim)
+        {
+            g.setColor(Color.darkGray);
+            g.drawOval(rim[0],rim[1],rim[2],rim[3]);
+        }
+        if (displayGrid)
+        {
+            g.setColor(Color.lightGray);
+            for (int count=0;count < circleCount; count++)
+                g.drawOval(circles[count][0],circles[count][1],circles[count][2],circles[count][3]);
+            for (int count=0;count < spokeCount; count++)
+                g.drawLine(centerX, centerY,spokes[count][0],spokes[count][1]);
+        }
+    }
+
+    /**
+    * Repaint the screen overides canvas.repaint()
+    */
+
+    public void repaint()
+    {
+        isDirty=true;
+        super.repaint();
+    }
+
+    /**
+    * Repaint the screen overides canvas.update()
+    * to stop flickering
+    * @param g Graphics object
+    */
+
+    public void update(Graphics g)
+    {
+        paint(g);
+    }
+
+    /**
+    * draw a filled square with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+
+    public void rect(Graphics g, int x, int y)
+    {
+        g.drawRect(x-2,y-2,4,4);
+        g.fillRect(x-2,y-2,4,4);
+        repaint();
+    }
+
+    /**
+    * draw a filled circle with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+    public void oval(Graphics g, int x, int y)
+    {
+        g.drawOval(x-2,y-2,4,4);
+        g.fillOval(x-2,y-2,4,4);
+        repaint();
+    }
+
+    /**
+    * draw a filled diamond with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+    public void diamond(Graphics g, int x, int y)
+    {
+        int numpoints = 4;
+        int[] xpoints = new int[numpoints+1];
+        int[] ypoints = new int[numpoints+1];
+        for(int i=0; i < numpoints; i++) {
+            double angle = 2*Math.PI * i / numpoints;
+            xpoints[i] = (int)(x + 3*Math.cos(angle));
+            ypoints[i] = (int)(y - 3*Math.sin(angle));
+        }
+
+        g.fillPolygon(xpoints, ypoints, numpoints);
+        xpoints[numpoints] = xpoints[0];  ypoints[numpoints] = ypoints[0];
+        g.drawPolygon(xpoints, ypoints, numpoints+1);
+        repaint();
+    }
+
+    /**
+    * paint the whole thing
+    * @param g Graphics object
+    */
+
+    public void paint(Graphics g)
+    {
+	System.out.println("Repaint CvManyPoints");
+        if(!isVisible()) return;
+        if (isDirty==true)
+        {
+            if (imgW != dim.width || imgH != dim.height)
+            {
+                if( gImage != null ) gImage.dispose();
+                imgW = dim.width; imgH = dim.height;
+                image = createImage(imgW, imgH);
+                gImage = image.getGraphics();
+            }
+            gImage.setColor(Color.white);
+            gImage.fillRect(0, 0, imgW, imgH);
+
+            drawTarget(gImage);
+            gImage.setColor(defNodesColor);
+            drawEdgeNodes(gImage);
+            gImage.setColor(defDocsColor);
+            drawDocXs(gImage);
+
+            if (tp!=null)
+                drawSelect(gImage);
+            if (selectDoc!=null)
+                drawDocSelect(gImage);
+
+            //gImage.drawRect(1,1,dim.width-1,dim.height-1);
+        }
+        g.drawImage(image, 0, 0, null);
+
+        isDirty=false;
+    }
+}
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/HitCluster.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/HitCluster.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/HitCluster.java	(revision 14150)
@@ -0,0 +1,136 @@
+package vishnu.testvis.visual;
+
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.lang.*;
+import java.util.zip.*;
+
+class HitCluster
+{
+    static final int NOVALUE = 99999999;
+    static int MAX;
+    static final int EMPTYARRAY[] = {};
+    static int matrix[][];
+    // static int data[][];
+    static float data [][];
+    static int clusters[][];
+    static int keywords[];
+    static int maxDocuments;
+
+    static void buildMatrix ()
+    {
+	    int count;
+	    int keyword1, keyword2;
+    	matrix = new int[MAX][MAX];
+        for (int word1=0;word1<MAX;word1++)
+        {
+	        keyword1 = keywords[word1];
+	        for (int word2=word1+1; word2<MAX;word2++)
+            {
+ 		        keyword2 = keywords[word2];
+                count = 0;
+                for (int doc=0;doc<maxDocuments;doc++)
+                {
+		            if ( (data[doc][keyword1] > 0) && (data[doc][keyword2] > 0))
+			            count++;
+	            }
+                matrix[word1][word2]=matrix[word2][word1]=count;
+            }
+        }
+	    for (int row = 0;row <MAX; row++)
+	    {
+	        matrix[row][row] = NOVALUE;
+	        for (int col = 0; col <row; col++)
+		    matrix[row][col] = matrix[col][row];
+	    }
+    }
+
+    static void findLargest ()
+    {
+	    int group = matrix[0].length-1;
+	    int largest = -1;
+	    int smallRow = NOVALUE;
+	    int smallCol = NOVALUE;
+	    int row,col;
+	    for (row =0; row<MAX; row++)
+	        for (col=row+1; col<MAX; col++)
+	            if (largest < matrix[row][col])
+	            {
+		            largest = matrix[row][col];
+		            smallRow = row;
+		            smallCol = col;
+ 	            }
+	    for (col=0; col<MAX; col++)
+	    {
+	        if (matrix[smallRow][col] > matrix[smallCol][col])
+		        matrix[smallRow][col] = matrix[smallCol][col];
+            matrix[smallCol][col] = -1;
+	    }
+	    clusters[smallRow] = addClusters(smallRow, smallCol);
+	    clusters[smallCol] = EMPTYARRAY;
+	    for (row =0; row<MAX; row++)
+        {
+	        matrix[row][smallCol] = -1;
+            matrix[row][smallRow] = matrix[smallRow][row];
+	    }
+    }
+
+    static int[] addClusters(int c1, int c2)
+    {
+	    int i;
+	    int temp[] = new int[clusters[c1].length+clusters[c2].length];
+	    if (keywords[clusters[c1][0]] < keywords[clusters[c2][0]])
+	    {
+	        for (i = 0; i<clusters[c1].length;i++)
+	    	    temp[i] = clusters[c1][i];
+	        for (i = 0; i<clusters[c2].length;i++)
+	    	    temp[i+clusters[c1].length] = clusters[c2][i];
+	    }
+	    else
+	    {
+	        for (i = 0; i<clusters[c2].length;i++)
+	    	    temp[i] = clusters[c2][i];
+	        for (i = 0; i<clusters[c1].length;i++)
+	    	    temp[i+clusters[c2].length] = clusters[c1][i];
+	    }
+	    return temp;
+    }
+
+    static void preClusters()
+    {
+	    clusters = new int[MAX][1];
+	    for (int row =0; row <MAX; row++)
+	    {
+	        clusters[row][0] = row;
+	    }
+    }
+
+    static int[] maximunDistance (float[][] newData, int[] newKeywords, int documentCount)
+    {
+	    data = newData;
+	    keywords = newKeywords;
+	    maxDocuments = documentCount;
+	    if (documentCount > data.length)
+	    {
+	        System.out.println ("WARNING matrix sent is shorter the document Count!");
+	        System.out.println ("I will use the shorter length only.");
+	        documentCount = data.length;
+        }
+	    MAX = keywords.length;
+	    buildMatrix();
+	    preClusters();
+	    for (int i=1;i<MAX;i++)
+	    {
+ 	        findLargest();
+        }
+	    int temp[] = new int[MAX];
+	    for (int i=0;i<MAX;i++)
+	    {
+	        temp[i] = keywords[clusters[0][i]];
+//System.out.println (i+" = "+keywords[i]+" = "+clusters[0][i]);
+    	}
+	    return temp;
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Hlink.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Hlink.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Hlink.java	(revision 14150)
@@ -0,0 +1,118 @@
+
+/**
+ * Title:        VeniVidiVinci<p>
+ * Description:  An applet that displays visualisations of data sets and allows
+ * these visualisations to be manipulated and selections made<p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      Imperial College<p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.JEditorPane;
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+
+public class Hlink extends JEditorPane
+{
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+    public int numDisplayedDocs = 15;
+    private int width, length;
+
+    public Hlink(int width, int length , Vishnu currentMain)
+    {
+        this.currentMain=currentMain;
+        this.width=width;
+        this.length=length;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    public Hlink()
+    {
+        this.currentMain=null;
+        this.width=500;
+        this.length=300;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    public void jbInit() throws Exception
+    {
+        linkPane = new JScrollPane(this);
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener
+    {
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+                System.out.println("Mouse ACTIVATED -> " + he.getURL().toString());
+        }
+    }
+
+    public void update()
+    {
+       int [] docIDs = currentMain.dataManager.getDocs();
+        Vector subSet = new Vector();
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+
+            // Check that the element exists
+            if (docIDs.length <= i) break;
+
+            subSet.addElement(new Integer(docIDs[i]));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDescriptionStr(subSet);
+
+        setText(htmlDescription);
+
+    }
+
+    public void clear()
+    {
+        setText("");
+    }
+
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Ipoint2D.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Ipoint2D.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Ipoint2D.java	(revision 14150)
@@ -0,0 +1,17 @@
+package vishnu.testvis.visual;
+/** Integer point class 
+* (minimal)
+* @author Matthew Carey */
+
+public class Ipoint2D
+{
+   /** x value */
+   public int x;
+   /** y value */
+   public int y;
+   /** constructor 
+   * @param horizontal value
+   * @param vertical value
+   */
+   public Ipoint2D(int x, int y){this.x = x; this.y = y;}
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/KeyWordArray.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/KeyWordArray.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/KeyWordArray.java	(revision 14150)
@@ -0,0 +1,106 @@
+/** Class to store the keywords and their visible states
+* @author Matthew Carey
+*/
+package vishnu.testvis.visual;
+
+public class KeyWordArray
+{
+    /** constant default number enabled */
+    public static final int DEFAULT_ENABLED = 12;
+    /** the word array */
+    private String words[];
+    /** the enabled array */
+    private boolean enabled[];
+    /** the count */
+    private int count;
+    /**
+    * constructor
+    * @param word array
+    * @param count size
+    * @param number enabled
+    */
+    public KeyWordArray(String words[],int count,int numEnabled)
+    {
+        this.words = words;
+        this.count = count;
+        enabled = new boolean[count];
+        int theNumEnabled = count<numEnabled?count:numEnabled;
+        for (int c = 0; c < numEnabled && c < count; c++)
+            enabled[c]=true;
+        if (count > theNumEnabled)
+            for (int c = theNumEnabled; c < count; c++)
+                enabled[c]=false;
+    }
+    /**
+    * constructor
+    * @param word array
+    * @param count size
+    */
+    public KeyWordArray(String words[],int count)
+    {
+        this(words,count,DEFAULT_ENABLED);
+    }
+
+    public KeyWordArray()
+    {
+        this.count = 0;
+        this.words = null;
+        this.enabled = null;
+    }
+
+    /**
+    * Find the size of the array
+    * @return size of the array
+    */
+    public int getNumWords()
+    {
+        return count;
+    }
+
+    /**
+    * Find the word at index
+    * @param index of word
+    * @return word at index
+    */
+    public String getWord(int index)
+    {
+        if (index >=0 && index < count)
+           return this.words[index];
+        return "Error out of index";
+    }
+
+    /**
+    * Get state of word at index
+    * @param index of word
+    * @return state of word
+    */
+    public boolean getState(int index)
+    {
+        if (index >=0 && index < count)
+           return this.enabled[index];
+        return false;
+    }
+
+    /**
+    * Set state of word at index
+    * @param index of word
+    * @param new state of word
+    */
+    public void setState(int index, boolean state)
+    {
+        if (index >=0 && index < count)
+           this.enabled[index]=state;
+    }
+
+    /**
+    * Get number of enabled words
+    * @return number of enabled words
+    */
+    public int getNumEnabled()
+    {
+        int result = 0;
+        for (int c = 0; c < count; c++)
+            if (enabled[c]) result++;
+        return result;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/KeywordPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/KeywordPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/KeywordPanel.java	(revision 14150)
@@ -0,0 +1,188 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import java.awt.List;
+import java.applet.*;
+import javax.swing.*;
+
+/** A panel that is added to the frame containing an on screen list of keywords to select
+*   and deselect
+*   @author Athina Economou
+*/
+public class KeywordPanel extends JPanel
+{
+    /** the base frame */
+    protected RadVizPanel f;
+    protected KeyWordArray keyWordArray;
+    protected int wordCount;
+    protected Vector  selArrayIndex;
+    protected Vector  listArrayIndex;
+    protected List available;
+    protected List selected;
+    protected int largeFont = 16;
+    protected int smallFont = 14;
+
+
+    /** Constructor
+    * @param drawing canvas
+    * @param base frame
+    */
+    public KeywordPanel(final CvManyPoints canvas1, RadVizPanel f)
+    {
+        this.f = f;
+
+        keyWordArray = canvas1.getKeyWords();
+        wordCount = keyWordArray.getNumWords();
+        selArrayIndex = new Vector();
+        listArrayIndex = new Vector();
+        available = new List(12, false);
+        selected = new List(12, false);
+        // mc
+        SymAction lSymAction = new SymAction();
+
+//        Panel base = new Panel();
+        setLayout(new BorderLayout(10,0));
+        //setSize(50,400);
+        JPanel labels = new JPanel();
+        Font font1 = new Font("Monospaced", Font.PLAIN, largeFont);
+        labels.setFont(font1);
+        labels.setForeground(Color.black);
+        labels.add(new Label("Available           Displayed"));
+
+        JPanel rightPanel = new JPanel();
+        Font font = new Font("Dialog", Font.PLAIN, smallFont);
+        rightPanel.setFont(font);
+        JPanel leftPanel = new JPanel();
+        leftPanel.setFont(font);
+
+        for (int c=0; c< wordCount;c++)
+            if (keyWordArray.getState(c))
+            {
+                selected.add(keyWordArray.getWord(c));
+                selArrayIndex.addElement(new Integer(c));
+            }
+            else
+            {
+                available.add(keyWordArray.getWord(c));
+                listArrayIndex.addElement(new Integer(c));
+            }
+        available.addActionListener(lSymAction);
+        available.addItemListener(new ItemListener()
+            {
+                public void itemStateChanged(ItemEvent event)
+                {
+                    if(event==null) return;
+                    try
+                    {
+                        int index =((List)event.getItemSelectable()).getSelectedIndex();
+                        if(index >= 0)
+                        {
+                            String keywords = ((List)event.getItemSelectable()).getSelectedItem();
+                            Integer result = (Integer)listArrayIndex.elementAt(index);
+                            selected.add(keywords);
+                            selArrayIndex.addElement(result);
+
+                            available.remove(keywords);
+                            listArrayIndex.removeElementAt(index);
+
+                            for (int c=0; c<keyWordArray.getNumWords(); c++)
+                                keyWordArray.setState(c,false);
+
+                            for (int c=0;c<(int)selArrayIndex.size();c++)
+                            {
+                                Integer result1 = (Integer)selArrayIndex.elementAt(c);
+                                int index1 = result1.intValue();
+                             keyWordArray.setState(index1,true);
+                            } // for
+                            canvas1.recalc();
+                        } // if
+                    } //try
+                    catch (ArrayIndexOutOfBoundsException e)
+                    {
+                    } // catch
+                } //itemstate changed
+            }); // ItemListener
+            //mc
+            selected.addActionListener(lSymAction);
+            selected.addItemListener(new ItemListener()
+            {
+                public void itemStateChanged(ItemEvent event)
+                {
+                    //System.out.println("calling getSeleced index");
+                    try
+                    {
+                        int index =((List)event.getItemSelectable()).getSelectedIndex();
+                    //System.out.println("Index");
+                    //System.out.println(index);
+                        if(index >= 0)
+                        {
+                            String keywords = ((List)event.getItemSelectable()).getSelectedItem();
+                            Integer result = (Integer)selArrayIndex.elementAt(index);
+
+                            available.add(keywords);
+                            listArrayIndex.addElement(result);
+
+                            selected.remove(keywords);
+                            selArrayIndex.removeElementAt(index);
+
+                            for (int c=0; c<keyWordArray.getNumWords(); c++)
+                                keyWordArray.setState(c,false);
+
+                            for (int c=0;c<(int)selArrayIndex.size();c++)
+                            {
+                                Integer result1 = (Integer)selArrayIndex.elementAt(c);
+                                int index1 = result1.intValue();
+                                keyWordArray.setState(index1,true);
+                            } // for
+                            canvas1.recalc();
+                        } // if
+                    } // try
+                    catch (ArrayIndexOutOfBoundsException e)
+                    {
+                    }
+                }
+            });
+
+            leftPanel.add(available);
+            rightPanel.add(selected);
+            add(labels, "North");
+            add(rightPanel, "East");
+            add(leftPanel, "West");
+            //base.add(text, "South");
+            setVisible(true);
+
+    }
+    public void update()
+    {
+        keyWordArray = f.canvas1.getKeyWords();
+        wordCount = keyWordArray.getNumWords();
+        selected.removeAll();
+        available.removeAll();
+        selArrayIndex.removeAllElements();
+        listArrayIndex.removeAllElements();
+        for (int c=0; c< wordCount;c++)
+            if (keyWordArray.getState(c))
+            {
+                selected.add(keyWordArray.getWord(c));
+                selArrayIndex.addElement(new Integer(c));
+            }
+            else
+            {
+                available.add(keyWordArray.getWord(c));
+                listArrayIndex.addElement(new Integer(c));
+            }
+    }
+
+
+    /** A dummy action listener to absorb double mouse clicks */
+    class SymAction implements java.awt.event.ActionListener
+    {
+        public void actionPerformed(java.awt.event.ActionEvent event)
+        {
+            Object object = event.getSource();
+            //System.out.println("Action event");
+        }
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/ListPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/ListPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/ListPanel.java	(revision 14150)
@@ -0,0 +1,127 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.*;
+import java.awt.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class ListPanel extends JPanel
+{
+    JList documentsList = new JList();
+    BorderLayout borderLayout1 = new BorderLayout();
+    Border border1;
+    JLabel descriptionLabel = new JLabel();
+    Vishnu displayFrame;
+    JPanel southPanel = new JPanel();
+    JButton fetchButton = new JButton();
+
+    public ListPanel()
+    {
+        displayFrame=null;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    public ListPanel(Vishnu d)
+    {
+        displayFrame=d;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void jbInit() throws Exception
+    {
+        border1 = BorderFactory.createEmptyBorder(10,10,10,10);
+        this.setLayout(borderLayout1);
+        this.setBorder(border1);
+        descriptionLabel.setFont(new java.awt.Font("Monospaced", 0, 14));
+        descriptionLabel.setText("Document descriptions");
+        documentsList.setFont(new java.awt.Font("Monospaced", 0, 12));
+        documentsList.setToolTipText("");
+        documentsList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+        documentsList.setVisibleRowCount(10);
+        documentsList.addMouseListener(new java.awt.event.MouseAdapter()
+        {
+
+            public void mouseClicked(MouseEvent e)
+            {
+                documentsList_mouseClicked(e);
+            }
+        });
+        fetchButton.setFont(new java.awt.Font("Monospaced", 0, 16));
+        fetchButton.setBorder(BorderFactory.createEtchedBorder());
+        fetchButton.setText("Fetch");
+        fetchButton.addMouseListener(new java.awt.event.MouseAdapter()
+        {
+
+            public void mouseClicked(MouseEvent e)
+            {
+                fetchButton_mouseClicked(e);
+            }
+        });
+        this.add(documentsList, BorderLayout.CENTER);
+        this.add(descriptionLabel, BorderLayout.NORTH);
+        this.add(southPanel, BorderLayout.SOUTH);
+        southPanel.add(fetchButton, null);
+    }
+
+    public void update()
+    {
+       documentsList.setListData(displayFrame.dataManager.getDescriptions());
+       repaint();
+    }
+
+    void documentsList_mouseClicked(MouseEvent e)
+    {
+        int index = documentsList.getSelectedIndex();
+        if (index >=0)
+            displayFrame.dataManager.fetch(index);
+    }
+
+    void fetchButton_mouseClicked(MouseEvent e)
+    {
+        int index = documentsList.getSelectedIndex();
+        if (index >=0)
+            displayFrame.dataManager.fetch(index);
+    }
+
+    void fetchButton_mouseEntered(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mouseExited(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mousePressed(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mouseReleased(MouseEvent e)
+    {
+
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/MapKeyWords.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/MapKeyWords.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/MapKeyWords.java	(revision 14150)
@@ -0,0 +1,78 @@
+package vishnu.testvis.visual;
+ /** This class is maps the display array of keywords onto the stored.
+  *  array
+  *  @author Matthew Carey</a>
+  */
+public class MapKeyWords
+{
+   /** array map for display to stored */
+   protected int nodesToWord[] = null;
+   /** array map for stored to display */
+   protected int wordsToNodes[] = null;
+   /**
+   * map for display to stored
+   * @param display index
+   * @return stored index
+   */
+   public int mapNodesToWords(int index)
+   {
+        return nodesToWord[index];
+   }
+   /**
+   * map for stored to display
+   * @param stored index
+   * @return the display index
+   */
+   public int mapWordsToNodes(int index)
+   {
+        return wordsToNodes[index];
+   }
+   /**
+   * constructor
+   * @param keywordarray instance containing list of key words and states
+   */
+   public MapKeyWords(KeyWordArray keyWordArray)
+   {
+        int c, n=0;
+        nodesToWord = null;
+        wordsToNodes = null;
+        nodesToWord = new int [keyWordArray.getNumEnabled()];
+        wordsToNodes = new int [keyWordArray.getNumWords()];
+        int totalKeyWord=keyWordArray.getNumWords();
+        for (c=0;c < totalKeyWord; c++)
+        {
+             if (keyWordArray.getState(c))
+             {
+                wordsToNodes[c]=n;
+                nodesToWord[n]=c;
+                n++;
+             }
+             else
+                wordsToNodes[c]=-1;
+        }
+   }
+   /**
+   * re-Map the KeyWords with a new vector
+   * @param array the new vector
+   * @param keywordarray instance containing list of key words and states
+   */
+   public void reMapKeyWords(int array[],KeyWordArray keyWordArray)
+   {
+        if (array.length != keyWordArray.getNumEnabled())
+            return;
+        int c;
+        for (c=0;c<array.length;c++)
+        {
+            if (!keyWordArray.getState(array[c]))
+               return;
+        }
+
+        for (c=0;c<array.length;c++)
+        {
+                wordsToNodes[array[c]]=c;
+                nodesToWord[c]=array[c];
+        }
+   }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java	(revision 14150)
@@ -0,0 +1,136 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.util.*;
+
+/**
+ * A custom component that displays multiple lines of text with specified
+ * margins and alignment.  In Java 1.1, we could extend Component instead
+ * of Canvas, making this a more efficient "Lightweight component"
+ * from Java Examples in a Nutshell,
+ * @author Flanagan, David - O'Rielly 1997 NY
+ */
+public class MultiLineLabel extends Canvas {
+  // User-specified attributes
+  protected String label;             // The label, not broken into lines
+  protected int margin_width;         // Left and right margins
+  protected int margin_height;        // Top and bottom margins
+  protected int alignment;            // The alignment of the text.
+  public static final int LEFT = 0, CENTER = 1, RIGHT = 2; // alignment values
+  // Computed state values
+  protected int num_lines;            // The number of lines
+  protected String[] lines;           // The label, broken into lines
+  protected int[] line_widths;        // How wide each line is
+  protected int max_width;            // The width of the widest line
+  protected int line_height;          // Total height of the font
+  protected int line_ascent;          // Font height above baseline
+  protected boolean measured = false; // Have the lines been measured?
+
+  // Here are five versions of the constructor
+  public MultiLineLabel(String label, int margin_width,
+                        int margin_height, int alignment) {
+    this.label = label;                 // Remember all the properties
+    this.margin_width = margin_width;
+    this.margin_height = margin_height;
+    this.alignment = alignment;
+    newLabel();                         // Break the label up into lines
+  }
+  public MultiLineLabel(String label, int margin_width, int margin_height) {
+    this(label, margin_width, margin_height, LEFT);
+  }
+  public MultiLineLabel(String label, int alignment) {
+    this(label, 10, 10, alignment);
+  }
+  public MultiLineLabel(String label) { this(label, 10, 10, LEFT); }
+  public MultiLineLabel() { this(""); }
+
+  // Methods to set and query the various attributes of the component
+  // Note that some query methods are inherited from the superclass.
+  public void setLabel(String label) {
+    this.label = label;
+    newLabel();               // Break the label into lines
+    measured = false;         // Note that we need to measure lines
+    repaint();                // Request a redraw
+  }
+  public void setFont(Font f) {
+    super.setFont(f);         // tell our superclass about the new font
+    measured = false;         // Note that we need to remeasure lines
+    repaint();                // Request a redraw
+  }
+  public void setForeground(Color c) {
+    super.setForeground(c);   // tell our superclass about the new color
+    repaint();                // Request a redraw (size is unchanged)
+  }
+  public void setAlignment(int a) { alignment = a; repaint(); }
+  public void setMarginWidth(int mw) { margin_width = mw; repaint(); }
+  public void setMarginHeight(int mh) { margin_height = mh; repaint(); }
+  public String getLabel() { return label; }
+  public int getAlignment() { return alignment; }
+  public int getMarginWidth() { return margin_width; }
+  public int getMarginHeight() { return margin_height; }
+
+  /**
+   * This method is called by a layout manager when it wants to
+   * know how big we'd like to be.  In Java 1.1, getPreferredSize() is
+   * the preferred version of this method.  We use this deprecated version
+   * so that this component can interoperate with 1.0 components.
+   */
+  public Dimension getPreferredSize() {
+    if (!measured) measure();
+    return new Dimension(max_width + 2*margin_width,
+                         num_lines * line_height + 2*margin_height);
+  }
+
+  /**
+   * This method is called when the layout manager wants to know
+   * the bare minimum amount of space we need to get by.
+   * For Java 1.1, we'd use getMinimumSize().
+   */
+  public Dimension getMinimumSize() { return getPreferredSize(); }
+
+  /**
+   * This method draws the label (same method that applets use).
+   * Note that it handles the margins and the alignment, but that
+   * it doesn't have to worry about the color or font--the superclass
+   * takes care of setting those in the Graphics object we're passed.
+   */
+  public void paint(Graphics g) {
+    int x, y;
+    Dimension size = this.getSize();  // use getSize() in Java 1.1
+    if (!measured) measure();
+    y = line_ascent + (size.height - num_lines * line_height)/2;
+    for(int i = 0; i < num_lines; i++, y += line_height) {
+      switch(alignment) {
+      default:
+      case LEFT:    x = margin_width; break;
+      case CENTER:  x = (size.width - line_widths[i])/2; break;
+      case RIGHT:   x = size.width - margin_width - line_widths[i]; break;
+      }
+      g.drawString(lines[i], x, y);
+    }
+  }
+
+  /** This internal method breaks a specified label up into an array of lines.
+   *  It uses the StringTokenizer utility class. */
+  protected synchronized void newLabel() {
+    StringTokenizer t = new StringTokenizer(label, "\n");
+    num_lines = t.countTokens();
+    lines = new String[num_lines];
+    line_widths = new int[num_lines];
+    for(int i = 0; i < num_lines; i++) lines[i] = t.nextToken();
+  }
+
+  /** This internal method figures out how the font is, and how wide each
+   *  line of the label is, and how wide the widest line is. */
+  protected synchronized void measure() {
+    FontMetrics fm = this.getFontMetrics(this.getFont());
+    line_height = fm.getHeight();
+    line_ascent = fm.getAscent();
+    max_width = 0;
+    for(int i = 0; i < num_lines; i++) {
+      line_widths[i] = fm.stringWidth(lines[i]);
+      if (line_widths[i] > max_width) max_width = line_widths[i];
+    }
+    measured = true;
+  }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/PlainPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/PlainPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/PlainPanel.java	(revision 14150)
@@ -0,0 +1,153 @@
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+import javax.swing.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Date;
+
+import javax.swing.JPanel;
+import java.awt.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class PlainPanel extends JPanel
+{
+    Vishnu displayFrame;
+    BorderLayout borderLayout1;
+    Border border1;
+    JScrollPane theScrollPane;
+    JEditorPane theEditorPane;
+
+
+  // This method returns an HTML table
+    public String makeHTMLTable()
+    {
+        String [] descriptions = displayFrame.dataManager.getDescriptions();
+        int [] docNums = displayFrame.dataManager.getDocs();
+        // Set up an output stream we can print the table to.
+        // This is easier than concatenating strings all the time.
+        int number;
+        if (docNums==null) number = 0;
+        else number = docNums.length;
+
+        StringWriter sout = new StringWriter();
+        PrintWriter out = new PrintWriter(sout);
+
+        // Print out the table
+	Dimension d = displayFrame.getSize();
+	int totalWidth = d.width-40;
+	int textWidth = totalWidth - 100;
+	
+        out.println("<TABLE BORDER= 2 WIDTH=" + totalWidth + "><TR>");
+
+        out.println("<TH WIDTH= 100>Document Number</TH>");
+        out.println("<TH WIDTH = " + textWidth + ">Sample Text</TH></TR>");
+
+        for(int i=0; i < docNums.length; i++)
+        {
+            out.print("<TR><TD><A HREF=\"");
+            out.print(docNums[i]);
+            out.print("\">");
+            out.print(docNums[i]);
+            out.print("</A></TD><TD>");
+            out.print(descriptions [i]);
+            out.println("</TD></TR>");
+        }
+        out.println("</TABLE>");
+        out.close();
+
+        // Get the string of HTML from the StringWriter and return it.
+        return sout.toString();
+    }
+
+    public PlainPanel()
+    {
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+    public PlainPanel(Vishnu d)
+    {
+        displayFrame = d;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+// The sychronized qualifier is placed here because when two plain
+// panels are created in different threads almost simulataniously
+// we seem to get interference happening
+    public synchronized void jbInit() throws Exception
+    {
+        borderLayout1 = new BorderLayout();
+        theEditorPane = new JEditorPane();
+
+        border1 = BorderFactory.createEmptyBorder(10,10,10,10);
+        theEditorPane.setEditable(false);               // we're browsing not editing
+        theEditorPane.setContentType("text/html");      // must specify HTML text
+        theEditorPane.setText("<TABLE BORDER= 2 WIDTH=640><TR>"+
+                                "<TH WIDTH= 100>Document Number</TH>" +
+                                "<TH WIDTH = 520>Sample Text</TH></TR>" +
+                                "</TABLE>");  // specify the text to display
+        theEditorPane.setFont(new java.awt.Font("Monospaced", 0, 12));
+    // Set up the JEditorPane to handle clicks on hyperlinks
+        theEditorPane.addHyperlinkListener(new HyperlinkListener() {
+                public void hyperlinkUpdate(HyperlinkEvent e)
+                {
+	            // Handle clicks; ignore mouseovers and other link-related events
+	                if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED)
+                    {
+	            // Get the HREF of the link and display it.
+                        String hpresult = e.getDescription();
+                        int index = Integer.parseInt(hpresult);
+                        if (index > 0)
+                            displayFrame.dataManager.fetchDocNo(index);
+	                }
+                }
+            });
+
+        // Put the JEditorPane in a scrolling window and display it.
+
+        this.setLayout(borderLayout1);
+        this.setFont(new java.awt.Font("Monospaced", 0, 12));
+        this.setBorder(border1);
+        theScrollPane = new JScrollPane();
+        //theScrollPane.setVerticalScrollBarPolicy(JScrollPane.VERTICAL_SCROLLBAR_ALWAYS);
+        this.add(theScrollPane, BorderLayout.CENTER);
+        theScrollPane.getViewport().add(theEditorPane, null);
+        setSize(600, 400);
+        setVisible(true);
+    }
+
+    public void update()
+    {
+       theEditorPane.setText(makeHTMLTable());
+    }
+
+    public void paint(Graphics g)
+    {
+	update();
+    }
+
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java	(revision 14150)
@@ -0,0 +1,42 @@
+/**
+* The head of the rows of floating point lists contains drawing and
+* title information about a document as well as the document index
+* @author Matthew Carey
+*/
+package vishnu.testvis.visual;
+
+import vishnu.datablock.*;
+
+public class Pnt2DTupleNode
+{
+    /** index of the document */
+    public int index;
+    /** start of linked list */
+    public SparseNode row;
+    /** cartesian floating point location */
+    public Point2D pnt;
+    /** raster integer locatoin */
+    public Ipoint2D iPnt;
+    /** drawn or not */
+    public boolean state;
+    /** document title */
+    public String ref;
+    /**
+    * Constructor
+    * @param cartesian x
+    * @param cartesain y
+    * @param index
+    * @param linked list of data
+    * @param title
+    */
+    Pnt2DTupleNode(float x, float y ,int i, SparseNode fn, String reference)
+    {
+        state = false;
+        index = i;
+        pnt = new Point2D(x,y);
+        iPnt = new Ipoint2D(0,0);
+        ref=reference;
+        if (reference==null) ref = "Not defined";
+        row=fn;
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Pt2DNode.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Pt2DNode.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Pt2DNode.java	(revision 14150)
@@ -0,0 +1,53 @@
+package vishnu.testvis.visual;
+import vishnu.datablock.*;
+/**
+* Downward linked list wrapper class enables several lists to share
+* same row nodes
+* @author Matthew Carey
+*/
+
+public class Pt2DNode
+{
+     /** next node */
+     Pt2DNode next;
+     /** row data */
+     Pnt2DTupleNode tuple;
+     /** constructor
+     * @param another node to link to
+     */
+     Pt2DNode(Pnt2DTupleNode t)
+     {
+        tuple = t;
+        next = null;
+     }
+     /**
+     * constructor
+     * @param cartesian x
+     * @param cartesian y
+     * @param document index
+     * @param data row
+     * @param document title
+     */
+     Pt2DNode(float x, float y ,int i, SparseNode t, String s)
+     {
+        tuple = new Pnt2DTupleNode(x,y,i,t,s);
+        next = null;
+     }
+     /**
+     * Get next node
+     * @return next node
+     */
+     Pt2DNode getNext()
+     {
+        return next;
+     }
+     /**
+     * Set next node
+     * @param another node to be next
+     */
+     public void setNext(Pt2DNode n)
+     {
+        n.next=next;
+        next = n;
+     }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/RadVizPanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/RadVizPanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/RadVizPanel.java	(revision 14150)
@@ -0,0 +1,300 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class RadVizPanel extends JPanel
+{
+    protected CvManyPoints canvas1;
+    public java.awt.Label docLabel;
+    protected KeywordPanel keyWordPanel;
+    protected Panel south = new Panel();
+
+    protected Border border1;
+
+    protected JPanel buttonPanel = new JPanel();
+    protected JPanel buttonBasePanel = new JPanel();
+    protected JPanel labelPanel = new JPanel();
+    protected JLabel buttonLabel = new JLabel();
+    protected JButton zoomInButton = new JButton();
+    protected JButton zoomOutButton = new JButton();
+    protected JButton resetButton = new JButton();
+    protected JButton clusterButton = new JButton();
+
+    protected JCheckBox showGridCheckBox = new JCheckBox();
+    protected JCheckBox lockCheckBox = new JCheckBox();
+    protected JCheckBox showRimCheckBox = new JCheckBox();
+    protected JCheckBox projectCheckBox = new JCheckBox();
+
+    protected int smallFont = 14;
+    protected int largeFont = 16;
+    protected Border border3;
+    protected BorderLayout borderLayout1 = new BorderLayout(10,0);
+    protected Vishnu parent;
+
+    public RadVizPanel(Vishnu p)
+    {
+        parent = p;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+    private void jbInit() throws Exception
+    {
+        border1 = BorderFactory.createEmptyBorder(5,5,5,5);
+        border3 = BorderFactory.createEmptyBorder(10,10,10,10);
+        this.setLayout(borderLayout1);
+        this.setBorder(border3);
+        this.setPreferredSize(new Dimension(600, 400));
+        canvas1 = new CvManyPoints (parent);
+	
+	// This is used as a balloon by the canvas1
+        docLabel = new java.awt.Label("  ");
+        docLabel.setVisible(false);
+
+	// place it out of view if does become visible
+        docLabel.setBounds(12,-20,10,10); 
+        docLabel.setBackground(new Color(16776895));
+        docLabel.setFont(new Font("Dialog", Font.PLAIN, largeFont));
+	
+        add(docLabel);
+        add(canvas1, "Center");
+	
+        keyWordPanel = new KeywordPanel(canvas1, this);
+	keyWordPanel.setVisible(false);
+	
+        buttonBasePanel.setLayout(new BorderLayout(10,10));
+        buttonPanel.setLayout(new GridLayout(0,2,10,10));
+        buttonLabel.setText("Settings");
+        buttonLabel.setForeground(Color.black);
+        buttonLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	
+        zoomInButton.setText("Zoom in");
+        zoomInButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        zoomInButton.setBorder(BorderFactory.createEtchedBorder());
+        zoomInButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    zoomInButton_mouseClicked(e);
+		}
+	    });
+        zoomOutButton.setText("Zoom Out");
+        zoomOutButton.setFont(new java.awt.Font("Monospaced", 0,largeFont));
+        zoomOutButton.setBorder(BorderFactory.createEtchedBorder());
+        zoomOutButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    zoomOutButton_mouseClicked(e);
+		}
+	    });
+        resetButton.setText("Reset");
+        resetButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        resetButton.setBorder(BorderFactory.createEtchedBorder());
+        resetButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    resetButton_mouseClicked(e);
+		}
+	    });
+        clusterButton.setText("Cluster");
+        clusterButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        clusterButton.setBorder(BorderFactory.createEtchedBorder());
+        clusterButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    clusterButton_mouseClicked(e);
+		}
+	    });
+	
+        showGridCheckBox.setText("Show Grid");
+        showGridCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        showGridCheckBox.setSelected(false);
+        showGridCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        showGridCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    showGridCheckBox_itemStateChanged(e);
+		}
+	    });
+        showRimCheckBox.setText("Show Rim");
+        showRimCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        showRimCheckBox.setSelected(true);
+        showRimCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        showRimCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    showRimCheckBox_itemStateChanged(e);
+		}
+	    });
+	
+        projectCheckBox.setText("Projection");
+        projectCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        projectCheckBox.setSelected(true);
+        projectCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        projectCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    projectCheckBox_itemStateChanged(e);
+		}
+	    });
+        lockCheckBox.setText("Lock Radius");
+        lockCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        lockCheckBox.setSelected(false);
+        lockCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        lockCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    lockCheckBox_itemStateChanged(e);
+		}
+	    });
+	
+	
+        buttonPanel.add(zoomInButton);
+        buttonPanel.add(zoomOutButton);
+        buttonPanel.add(resetButton);
+        buttonPanel.add(clusterButton);
+        buttonPanel.add(showGridCheckBox);
+        buttonPanel.add(showRimCheckBox);
+        buttonPanel.add(projectCheckBox);
+        buttonPanel.add(lockCheckBox);
+	
+        labelPanel.add(buttonLabel);
+        buttonBasePanel.add(buttonPanel,"Center");
+        buttonBasePanel.add(labelPanel,"North");
+        south.setLayout(new BorderLayout(10,0));
+	south.add(keyWordPanel,"West");
+        south.add(buttonBasePanel,"East");
+        south.setVisible(false);
+        add(south,"South");
+    }
+    
+    
+    public void setBothVisible(boolean f)
+    {
+        keyWordPanel.setVisible(f);
+        canvas1.setVisible(f);
+        south.setVisible(f);
+    }
+    
+    
+    public void addNotify()
+    {
+	//Record the size of the window prior to calling parents addNotify.
+        Dimension d = getSize();
+
+        super.addNotify();
+	
+        if (fComponentsAdjusted)
+	    return;
+	
+	// Adjust components according to the insets
+        setSize(getInsets().left + getInsets().right + d.width,
+		getInsets().top + getInsets().bottom + d.height);
+        Component components[] = getComponents();
+        for (int i = 0; i < components.length; i++)
+	    {
+		Point p = components[i].getLocation();
+		p.translate(getInsets().left, getInsets().top);
+		components[i].setLocation(p);
+	    }
+        fComponentsAdjusted = true;
+    }
+    
+    
+    void  zoomOutButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.higher();
+    }
+
+
+    void  zoomInButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.flatter();
+    }
+
+
+    void  resetButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.recalc();
+    }
+
+
+    void  clusterButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.cluster();
+    }
+
+
+    void showGridCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.grid(showGridCheckBox.isSelected());
+    }
+
+
+    void showRimCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.rim(showRimCheckBox.isSelected());
+    }
+
+
+    void projectCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.projection(projectCheckBox.isSelected());
+    }
+
+
+    void lockCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.lock(lockCheckBox.isSelected());
+    }
+
+
+    public void update()
+    {
+        canvas1.update();
+        keyWordPanel.update();
+    }
+
+
+    public void setVisible(boolean f)
+    {
+        super.setVisible(f);
+        setBothVisible(f);
+    }
+
+
+    public void paint(Graphics g)
+    {
+	System.out.println("Updating RadVizPanel");
+	//canvas1.update();
+	keyWordPanel.update();
+    }
+    
+    protected boolean fComponentsAdjusted = false;
+}
+
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/SimplePanel.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/SimplePanel.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/SimplePanel.java	(revision 14150)
@@ -0,0 +1,205 @@
+
+/**
+ * Title:        VeniVidiVinci<p>
+ * Description:  An applet that displays visualisations of data sets and allows
+ * these visualisations to be manipulated and selections made<p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      Imperial College<p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.*;
+import javax.swing.plaf.basic.*;
+import java.awt.*;
+import java.awt.event.*;
+
+
+/** Panel containing the graphical search interface */
+public class SimplePanel extends JPanel {
+
+    JTextField textField;
+    Vishnu currentMain;
+
+
+    public class InfoLabel extends JLabel {
+        void update() {
+            int startIndex = localHLinkPanel.index +1;
+            int endIndex = localHLinkPanel.index + localHLinkPanel.numDisplayedDocs;
+            int numDocs = currentMain.dataManager.getDocCount();
+            if (numDocs < endIndex) endIndex = numDocs;
+
+            setText("Documents "+ startIndex + "-" + endIndex + " out of " + numDocs);
+        }
+
+    }
+
+    InfoLabel infoLabel = new InfoLabel();
+
+    Hlink localHLinkPanel;
+
+    JButton backButton, nextButton;
+    public SimplePanel(Vishnu theMain)
+    {
+        this.currentMain = theMain;
+        localHLinkPanel = new Hlink(600,400,currentMain);
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    public SimplePanel()
+
+    {
+        this.currentMain = null;
+        localHLinkPanel = new Hlink();
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    private void jbInit() throws Exception
+    {
+        setName("Simple ");
+
+
+        backButton = new JButton("<<");
+        backButton.setBorder(BorderFactory.createEtchedBorder());
+        backButton.setEnabled(false);
+        backButton.addMouseListener( new MouseAdapter() {
+            public void mouseClicked(MouseEvent e) {
+
+                if (!backButton.isEnabled()) return;
+
+                nextButton.setEnabled(true);
+
+                localHLinkPanel.index -= localHLinkPanel.numDisplayedDocs;
+                localHLinkPanel.update();
+                infoLabel.update();
+
+                if (localHLinkPanel.index == 0) {
+                    backButton.setEnabled(false);
+                }
+            }
+        });
+        nextButton = new JButton(">>");
+        nextButton.setBorder(BorderFactory.createEtchedBorder());
+        nextButton.setEnabled(false);
+        nextButton.addMouseListener( new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+
+                if (!nextButton.isEnabled()) return;
+
+                backButton.setEnabled(true);
+
+                localHLinkPanel.index += localHLinkPanel.numDisplayedDocs;
+                localHLinkPanel.update();
+                infoLabel.update();
+
+                if (localHLinkPanel.index + localHLinkPanel.numDisplayedDocs >= currentMain.dataManager.getDocCount())
+                {
+                    nextButton.setEnabled(false);
+                }
+            }
+        });
+
+        //
+
+        // Set the new layout
+        GridBagLayout gridBag = new GridBagLayout();
+        setLayout(gridBag);
+
+        GridBagConstraints constraints = new GridBagConstraints();
+
+        constraints.fill = GridBagConstraints.BOTH;
+        constraints.insets = new Insets(10,3,10,3);
+        constraints.ipadx = 0;
+        constraints.ipady = 8;
+
+
+
+        // Link panel
+        constraints.ipady = 4;
+        constraints.gridx = 0;
+        constraints.gridy = GridBagConstraints.RELATIVE;
+        constraints.gridwidth = GridBagConstraints.REMAINDER;
+        constraints.gridheight = 8;
+
+        gridBag.setConstraints(localHLinkPanel.linkPane,constraints);
+        add(localHLinkPanel.linkPane);
+
+        // Info label
+        constraints.gridx = 0;
+        constraints.gridy = GridBagConstraints.RELATIVE;
+        constraints.gridwidth = 6;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+        infoLabel.setMinimumSize(new Dimension(500,20));
+        infoLabel.setMaximumSize(new Dimension(500,20));
+        gridBag.setConstraints(infoLabel,constraints);
+        add(infoLabel);
+
+        // Back button
+
+        constraints.fill = GridBagConstraints.NONE;
+        constraints.gridx = GridBagConstraints.RELATIVE;
+        constraints.gridy = 9;
+        constraints.gridwidth = 2;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+        Dimension buttonDimension = new Dimension(55,30);
+        backButton.setMaximumSize(buttonDimension);
+        backButton.setPreferredSize(buttonDimension);
+        backButton.setMinimumSize(buttonDimension);
+
+        gridBag.setConstraints(backButton,constraints);
+        add(backButton);
+
+        // Next button
+        constraints.gridx = GridBagConstraints.RELATIVE;
+        constraints.gridy = 9;
+        constraints.gridwidth = 2;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+
+        nextButton.setMaximumSize(buttonDimension);
+        nextButton.setPreferredSize(buttonDimension);
+        nextButton.setMinimumSize(buttonDimension);
+
+        gridBag.setConstraints(nextButton,constraints);
+        add(nextButton);
+
+        // Global layout
+        setSize(600,400);
+        setPreferredSize(new Dimension(600,400));
+        setMinimumSize(new Dimension(600,400));
+
+        setVisible(true);
+    }
+
+
+    public void update() {
+
+        localHLinkPanel.index = 0;
+
+        infoLabel.update();
+        localHLinkPanel.update();
+
+        if (currentMain.dataManager.getDocCount() > localHLinkPanel.numDisplayedDocs) {
+            nextButton.setEnabled(true);
+        }
+
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Vishnu.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Vishnu.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/Vishnu.java	(revision 14150)
@@ -0,0 +1,445 @@
+package vishnu.testvis.visual;
+import vishnu.testvis.treemap.visual.TMNavigator;
+import vishnu.testvis.sammon.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.evaluate.*;
+import vishnu.testvis.dendro.*;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Vector;
+import java.util.Properties;
+import java.applet.*;
+import java.net.*;
+import vishnu.datablock.*;
+
+public class Vishnu extends JApplet
+{
+    public JTabbedPane theTabbedPane = null;
+    public PlainPanel plainPanel = null;
+    public GraphicalPane sammonPanel = null;
+    public RadVizPanel radVizPanel = null;
+    public DendroPanel dendroPanel = null;
+    public JButton findButton = null;
+    
+    public DataManager dataManager = null;
+    
+    public Logging log = null;
+    public Queries queries = null;
+    public boolean searchStarted = false;
+    public boolean firstSearch = true;
+    public boolean refine = false;
+    
+    public JTextField searchBox = null;
+    public JComboBox collCombo = null;
+    public JComboBox viewCombo = null;
+    protected int largeFont = 16;
+    
+    public String query = null;
+    public String coll = null;
+    public String view = null;
+    
+    public String server_address = null;
+    
+    protected Dimension size = null;
+        
+    public void init()
+    {
+	String library = getParameter("library");
+	if (library == null ) {
+	    System.err.println("Error: library parameter is null!!");
+	    // what should we do?
+	    return;
+	}
+	String viscgi = getParameter("viscgi");
+	
+	query = getParameter("query");
+        coll = getParameter("collection");
+	view = getParameter("view");
+	
+	server_address = library;
+	if (viscgi != null) {
+	    // if viscgi parameter is set (even if empty) we assume that
+	    // the server is running  cgi-like with arguments in the url
+	    server_address += viscgi;
+	    server_address = tidy_URL(server_address, true);
+	} else {
+	    server_address = tidy_URL(server_address, false);
+	}
+	
+	// What kind of engine the dataManager should use
+	String engine = getParameter("engine");
+ 
+	String w = getParameter("width");
+	String h = getParameter("height");
+	
+	if( (w != null) && (h != null) )
+	    size = new Dimension(Integer.parseInt(w),Integer.parseInt(h));
+        else size = new Dimension(1000,800);
+	
+	dataManager = new DataManager(this, engine);
+	
+        this.setSize(size); 
+        
+	theTabbedPane = new JTabbedPane();
+
+	radVizPanel = new RadVizPanel(this);
+        radVizPanel.setName("RadialVis");
+        
+        plainPanel = new PlainPanel(this);
+        plainPanel.setName("PlainVis");
+        
+        sammonPanel = new GraphicalPane(this);
+        sammonPanel.setName("SammonVis");
+        
+        dendroPanel = new DendroPanel(this);
+        dendroPanel.setName("DendroVis");
+	
+	findButton = new JButton();
+        try {
+	    jbInit();
+	} catch(Exception e){e.printStackTrace();}
+	
+	initCollections();
+	
+	if( query != null && !query.equals("null") ){
+	    searchBox.setText(query);
+	    findQuery();
+	}
+    }
+
+    protected void initCollections() {
+
+	Collection current_collection=null;
+	String current_view = null;
+	if( coll == null || coll.equals("") || coll.equals("null") ) {
+	    current_collection = ((Collection)dataManager.collections.elementAt(0));
+	    coll = current_collection.dir;
+	} else {
+	    // try to find the one specified in the parameters
+	    for( int i = 0; i < dataManager.collections.size(); i++ ){
+		Collection c = (Collection)dataManager.collections.elementAt(i);
+		if( c.dir.equals(coll) ){
+		    current_collection = c;
+		    current_view = view; // use the one from the parameters
+		    break;
+		}
+	    }
+	}
+	if (current_collection == null) {
+	    // could still be null if we specified an invalid collection in the parameters
+	    current_collection = ((Collection)dataManager.collections.elementAt(0));
+	    coll = current_collection.dir;
+	}
+	if (current_view == null) {
+	    if (current_collection.views != null) {
+		current_view = (String)current_collection.views[0];
+	    }
+	}
+	dataManager.currentCollection = current_collection;
+	dataManager.currentView = current_view;
+	this.view = current_view;
+	dataManager.requestCollection();
+	System.out.println("Requesting collection " + current_collection.name);
+	collCombo.setSelectedItem(current_collection);
+	if (viewCombo != null && current_view != null) {
+	    viewCombo.setSelectedItem(current_view);
+	    
+	}
+	    
+    }
+
+    protected void jbInit() throws Exception
+    {
+	JPanel pane = new JPanel();
+	setContentPane(pane);
+	pane.setBorder(BorderFactory.createLineBorder(Color.black, 2));
+        pane.setLayout(new BorderLayout());
+        
+	JPanel searchPanel = new JPanel();
+	searchPanel.setLayout(new BorderLayout(5,5));
+	searchPanel.setBackground(SystemColor.control);
+        searchBox = new JTextField();
+	searchBox.setPreferredSize(new Dimension((int)(size.width/5), 26));
+        searchBox.setEditable(true);
+        searchBox.addKeyListener(new java.awt.event.KeyAdapter(){
+                public void keyPressed(KeyEvent e)
+                {
+                    findButton_keyPressed(e);
+                }
+            });
+	
+	JLabel searchLabel = new JLabel();
+        searchLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        searchLabel.setForeground(Color.black);
+        searchLabel.setText("Search:");
+
+	searchPanel.add(searchLabel, BorderLayout.WEST);
+	searchPanel.add(searchBox, BorderLayout.CENTER);
+
+	JPanel collPanel = new JPanel();
+	collPanel.setLayout(new BorderLayout(5,5));
+	collPanel.setBackground(SystemColor.control);
+
+	collCombo = new JComboBox(dataManager.collections);
+	collCombo.setEditable(false);
+	collCombo.addItemListener(new java.awt.event.ItemListener() {
+		public void itemStateChanged(ItemEvent i) {
+		    if (i.getStateChange()==ItemEvent.SELECTED){
+			dataManager.currentCollection = (Collection)i.getItem();
+			collCombo.hidePopup();
+			coll = dataManager.currentCollection.dir;
+			if (dataManager.currentCollection.views != null) {
+			    viewCombo.setModel(new DefaultComboBoxModel(dataManager.currentCollection.views));
+			    viewCombo.setEnabled(true);
+			    view = (String)viewCombo.getItemAt(0);
+			    dataManager.currentView = view;
+
+			} else {
+			    viewCombo.setModel(new DefaultComboBoxModel());
+			    viewCombo.setEnabled(false);
+			    view = null;
+			    dataManager.currentView = view;
+			}
+			dataManager.requestCollection();
+
+			update();
+			
+		    }
+		    
+		}
+		});
+	collCombo.setRenderer(new javax.swing.DefaultListCellRenderer() {
+		public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus)
+		{
+		    setText(((Collection)value).name);
+		    setFont(list.getFont());
+		    return this;
+		}
+	    });
+
+	JLabel collLabel = new JLabel();           
+	collLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        collLabel.setForeground(Color.black); 
+        collLabel.setText("Collection:");
+	
+	collPanel.add(collLabel, BorderLayout.WEST);
+	collPanel.add(collCombo, BorderLayout.CENTER);
+
+	JPanel viewPanel = new JPanel();
+	viewPanel.setLayout(new BorderLayout(5,5));
+	viewPanel.setBackground(SystemColor.control);
+
+	viewCombo = new JComboBox();
+	viewCombo.setEnabled(false);
+
+	JLabel viewLabel = new JLabel();           
+	viewLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        viewLabel.setForeground(Color.black); 
+        viewLabel.setText("View:");
+	
+	viewPanel.add(viewLabel, BorderLayout.WEST);
+	viewPanel.add(viewCombo, BorderLayout.CENTER);
+
+	JPanel topPanel = new JPanel();
+        topPanel.setLayout(new BorderLayout(10,10));
+	topPanel.setBackground(SystemColor.control);
+        topPanel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	
+	JPanel backPanel = new JPanel();   
+        backPanel.setPreferredSize(new Dimension(size.width,size.height));
+        backPanel.setBackground(SystemColor.control);
+        backPanel.setLayout(new BorderLayout());
+        
+	theTabbedPane.setBackground(SystemColor.control);
+        theTabbedPane.setFont(new java.awt.Font("Monospaced", Font.BOLD, largeFont));
+        theTabbedPane.addChangeListener(new javax.swing.event.ChangeListener(){
+		public void stateChanged(ChangeEvent e)
+		{
+		    theTabbedPane_stateChanged(e);
+		}
+	    });
+	
+        findButton.setBackground(SystemColor.control);
+        findButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        findButton.setBorder(BorderFactory.createEtchedBorder());
+        findButton.setText("Find");
+        findButton.addKeyListener(new java.awt.event.KeyAdapter(){
+		public void keyPressed(KeyEvent e)
+		{
+		    findButton_keyPressed(e);
+		}
+	    });
+        
+        findButton.addMouseListener(new java.awt.event.MouseAdapter(){
+		public void mouseClicked(MouseEvent e)
+		{
+		    findButton_mouseClicked(e);
+		}
+	    });
+	
+        pane.add(backPanel, BorderLayout.CENTER);
+        backPanel.add(theTabbedPane, BorderLayout.CENTER);
+	
+        theTabbedPane.add(plainPanel, "PlainVis");
+        theTabbedPane.add(radVizPanel, "RadialVis");
+        theTabbedPane.add(sammonPanel,"SammonVis");
+        theTabbedPane.add(dendroPanel,"DendroVis");
+        theTabbedPane.setForegroundAt(0,Color.blue);
+	
+        pane.add(topPanel, BorderLayout.NORTH);
+	
+	JPanel boxesPanel = new JPanel();
+	boxesPanel.setLayout(new BorderLayout(5,5));
+	boxesPanel.setBackground(SystemColor.control);
+	boxesPanel.add(collPanel, BorderLayout.WEST);
+	boxesPanel.add(viewPanel, BorderLayout.EAST);
+	
+	topPanel.add(boxesPanel, BorderLayout.WEST);
+	topPanel.add(searchPanel, BorderLayout.CENTER);
+	topPanel.add(findButton, BorderLayout.EAST);
+	
+        theTabbedPane.setPreferredSize(size);
+
+    }
+    
+    
+    public void update()
+    {
+	plainPanel.update();
+        radVizPanel.update();
+        dendroPanel.update();        
+        sammonPanel.update();
+    }
+    
+        
+    void findQuery()
+    {
+	refine = false;
+	
+        int docCount;
+	String text = null;
+	
+	text = (String)searchBox.getText();
+	if(text.trim().length() < 1) return;
+	query = text.trim();
+	
+	if (viewCombo != null) {
+	    String view1 = null;
+	    if (viewCombo.isEnabled()) {
+		view1 = (String)viewCombo.getSelectedItem();	    
+	    }
+	    if (!view1.equals(this.view)) {
+		this.view = view1;
+		dataManager.currentView = view1;
+		dataManager.requestCollection(); // resets previous data
+	    }
+	}
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	
+	int count = 1;
+	String q = query;
+	docCount = dataManager.query(q);
+	if (docCount == 0 ) {
+      	    JOptionPane.showMessageDialog(this,"No documents found related to " + text);
+	    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    return;
+	}
+
+        setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+    
+
+    void theTabbedPane_stateChanged(ChangeEvent e)
+    {
+	for( int i = 0; i < theTabbedPane.getTabCount(); i++ )
+        {
+            theTabbedPane.setForegroundAt(i,Color.black);
+        }
+        
+        int tab = theTabbedPane.getSelectedIndex();
+        
+        theTabbedPane.setForegroundAt(tab,Color.blue);
+    }
+
+
+  void findButton_keyPressed(KeyEvent e)
+  {
+      int key = e.getKeyCode();
+      if (key==10)
+	  {
+	      findButton.setSelected(true);
+	      findQuery();
+	      findButton.setSelected(false);
+	  }
+  }
+    
+    void findButton_mouseClicked(MouseEvent e)
+    {
+	findQuery();
+    }
+    
+   
+    // Tidy up URLs
+    //
+    // Ensure a URL address (as string) has a protocol, host, and file.
+    //
+    // If the URL is a CGI script URL, it should be tidied up so that it is
+    // appropriate to tage attrib=value pairs on the end.  This means it
+    // must either end with a "?" or (if it contains a question-mark
+    // internally) end with a "&".
+    // taken from Phind.java, Greenstone digital library.
+    String tidy_URL(String address, boolean isCGI) {
+
+	// System.err.println("tidy URL: " + address);
+	
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+	    
+	}
+	
+	// if the URL is a cgi script, make sure it has a "?" in ti,
+	// and that it ends with a "?" or "&"
+	if (isCGI) {
+	    if (address.indexOf((int) '?') == -1) { 
+		address = address + "?";
+	    } else if (!address.endsWith("?")) {
+		address = address + "&";
+	    }
+	}
+
+	return address;
+    }
+
+}
+
+
+
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/VishnuSingle.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/VishnuSingle.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/testvis/visual/VishnuSingle.java	(revision 14150)
@@ -0,0 +1,118 @@
+package vishnu.testvis.visual;
+import vishnu.testvis.treemap.visual.TMNavigator;
+import vishnu.testvis.sammon.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.evaluate.*;
+import vishnu.testvis.dendro.*;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Vector;
+import java.util.Properties;
+import java.applet.*;
+import java.net.*;
+import vishnu.datablock.*;
+
+public class VishnuSingle extends Vishnu
+{
+
+    protected void initCollections() {
+	// put in a dummy Collection
+	Collection c = new Collection(coll, coll, coll, 0, coll, null);
+	dataManager.currentCollection = c;
+    }
+
+    // overwrite this to remove the collection stuff - cos we are working within one collection only
+    protected void jbInit() throws Exception {
+	JPanel pane = new JPanel();
+	setContentPane(pane);
+	pane.setBorder(BorderFactory.createLineBorder(Color.black, 2));
+        pane.setLayout(new BorderLayout());
+        
+	JPanel searchPanel = new JPanel();
+	searchPanel.setLayout(new BorderLayout(5,5));
+	searchPanel.setBackground(SystemColor.control);
+        searchBox = new JTextField();
+	searchBox.setPreferredSize(new Dimension((int)(size.width/5), 26));
+        searchBox.setEditable(true);
+        searchBox.addKeyListener(new java.awt.event.KeyAdapter(){
+                public void keyPressed(KeyEvent e)
+                {
+                    findButton_keyPressed(e);
+                }
+            });
+	
+	JLabel searchLabel = new JLabel();
+        searchLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        searchLabel.setForeground(Color.black);
+        searchLabel.setText("Search:");
+
+	searchPanel.add(searchLabel, BorderLayout.WEST);
+	searchPanel.add(searchBox, BorderLayout.CENTER);
+
+
+	JPanel topPanel = new JPanel();
+        topPanel.setLayout(new BorderLayout(10,10));
+	topPanel.setBackground(SystemColor.control);
+        topPanel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	
+	JPanel backPanel = new JPanel();   
+        backPanel.setPreferredSize(new Dimension(size.width,size.height));
+        backPanel.setBackground(SystemColor.control);
+        backPanel.setLayout(new BorderLayout());
+        
+	theTabbedPane.setBackground(SystemColor.control);
+        theTabbedPane.setFont(new java.awt.Font("Monospaced", Font.BOLD, largeFont));
+        theTabbedPane.addChangeListener(new javax.swing.event.ChangeListener(){
+		public void stateChanged(ChangeEvent e)
+		{
+		    theTabbedPane_stateChanged(e);
+		}
+	    });
+	
+        findButton.setBackground(SystemColor.control);
+        findButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        findButton.setBorder(BorderFactory.createEtchedBorder());
+        findButton.setText("Find");
+        findButton.addKeyListener(new java.awt.event.KeyAdapter(){
+		public void keyPressed(KeyEvent e)
+		{
+		    findButton_keyPressed(e);
+		}
+	    });
+        
+        findButton.addMouseListener(new java.awt.event.MouseAdapter(){
+		public void mouseClicked(MouseEvent e)
+		{
+		    findButton_mouseClicked(e);
+		}
+	    });
+	
+        pane.add(backPanel, BorderLayout.CENTER);
+        backPanel.add(theTabbedPane, BorderLayout.CENTER);
+	
+        theTabbedPane.add(plainPanel, "PlainVis");
+        theTabbedPane.add(radVizPanel, "RadialVis");
+        theTabbedPane.add(sammonPanel,"SammonVis");
+        theTabbedPane.add(dendroPanel,"DendroVis");
+        theTabbedPane.setForegroundAt(0,Color.blue);
+	
+        pane.add(topPanel, BorderLayout.NORTH);
+	
+	topPanel.add(searchPanel, BorderLayout.CENTER);
+	topPanel.add(findButton, BorderLayout.EAST);
+	
+        theTabbedPane.setPreferredSize(size);
+
+    }
+    
+    
+}
+
+
+
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Base64.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Base64.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Base64.java	(revision 14150)
@@ -0,0 +1,1281 @@
+package vishnu.util;
+
+import java.io.*;
+/**
+ * Encodes and decodes to and from Base64 notation.
+ *
+ * <p>
+ * Change Log:
+ * </p>
+ * <ul>
+ *  <li>v2.0.1 - Fixed an error when decoding a single byte, that is, when the
+ *   encoded data was a single byte.</li>
+ *  <li>v2.0 - I got rid of methods that used booleans to set options. 
+ *   Now everything is more consolidated and cleaner. The code now detects
+ *   when data that's being decoded is gzip-compressed and will decompress it
+ *   automatically. Generally things are cleaner. You'll probably have to
+ *   change some method calls that you were making to support the new
+ *   options format (<tt>int</tt>s that you "OR" together).</li>
+ *  <li>v1.5.1 - Fixed bug when decompressing and decoding to a             
+ *   byte[] using <tt>decode( String s, boolean gzipCompressed )</tt>.      
+ *   Added the ability to "suspend" encoding in the Output Stream so        
+ *   you can turn on and off the encoding if you need to embed base64       
+ *   data in an otherwise "normal" stream (like an XML file).</li>  
+ *  <li>v1.5 - Output stream pases on flush() command but doesn't do anything itself.
+ *      This helps when using GZIP streams.
+ *      Added the ability to GZip-compress objects before encoding them.</li>
+ *  <li>v1.4 - Added helper methods to read/write files.</li>
+ *  <li>v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.</li>
+ *  <li>v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream
+ *      where last buffer being read, if not completely full, was not returned.</li>
+ *  <li>v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.</li>
+ *  <li>v1.3.3 - Fixed I/O streams which were totally messed up.</li>
+ * </ul>
+ *
+ * <p>
+ * I am placing this code in the Public Domain. Do with it as you will.
+ * This software comes with no guarantees or warranties but with
+ * plenty of well-wishing instead!
+ * Please visit <a href="http://iharder.net/xmlizable">http://iharder.net/base64</a>
+ * periodically to check for updates or to contribute improvements.
+ * </p>
+ *
+ * @author Robert Harder
+ * @author rob@iharder.net
+ * @version 2.0
+ */
+public class Base64
+{
+    
+/* ********  P U B L I C   F I E L D S  ******** */   
+    
+    
+    /** No options specified. Value is zero. */
+    public final static int NO_OPTIONS = 0;
+    
+    /** Specify encoding. */
+    public final static int ENCODE = 1;
+    
+    
+    /** Specify decoding. */
+    public final static int DECODE = 0;
+    
+    
+    /** Specify that data should be gzip-compressed. */
+    public final static int GZIP = 2;
+    
+    
+    /** Don't break lines when encoding (violates strict Base64 specification) */
+    public final static int DONT_BREAK_LINES = 8;
+    
+    
+/* ********  P R I V A T E   F I E L D S  ******** */  
+    
+    
+    /** Maximum line length (76) of Base64 output. */
+    private final static int MAX_LINE_LENGTH = 76;
+    
+    
+    /** The equals sign (=) as a byte. */
+    private final static byte EQUALS_SIGN = (byte)'=';
+    
+    
+    /** The new line character (\n) as a byte. */
+    private final static byte NEW_LINE = (byte)'\n';
+    
+    
+    /** The 64 valid Base64 values. */
+    private final static byte[] ALPHABET =
+    {
+        (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G',
+        (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N',
+        (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', 
+        (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z',
+        (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g',
+        (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n',
+        (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', 
+        (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z',
+        (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', 
+        (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/'
+    };
+    
+    /** 
+     * Translates a Base64 value to either its 6-bit reconstruction value
+     * or a negative number indicating some other meaning.
+     **/
+    private final static byte[] DECODABET =
+    {   
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,                 // Decimal  0 -  8
+        -5,-5,                                      // Whitespace: Tab and Linefeed
+        -9,-9,                                      // Decimal 11 - 12
+        -5,                                         // Whitespace: Carriage Return
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 14 - 26
+        -9,-9,-9,-9,-9,                             // Decimal 27 - 31
+        -5,                                         // Whitespace: Space
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,              // Decimal 33 - 42
+        62,                                         // Plus sign at decimal 43
+        -9,-9,-9,                                   // Decimal 44 - 46
+        63,                                         // Slash at decimal 47
+        52,53,54,55,56,57,58,59,60,61,              // Numbers zero through nine
+        -9,-9,-9,                                   // Decimal 58 - 60
+        -1,                                         // Equals sign at decimal 61
+        -9,-9,-9,                                      // Decimal 62 - 64
+        0,1,2,3,4,5,6,7,8,9,10,11,12,13,            // Letters 'A' through 'N'
+        14,15,16,17,18,19,20,21,22,23,24,25,        // Letters 'O' through 'Z'
+        -9,-9,-9,-9,-9,-9,                          // Decimal 91 - 96
+        26,27,28,29,30,31,32,33,34,35,36,37,38,     // Letters 'a' through 'm'
+        39,40,41,42,43,44,45,46,47,48,49,50,51,     // Letters 'n' through 'z'
+        -9,-9,-9,-9                                 // Decimal 123 - 126
+        /*,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 127 - 139
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 140 - 152
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 153 - 165
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 166 - 178
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 179 - 191
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 192 - 204
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 205 - 217
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 218 - 230
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 231 - 243
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9         // Decimal 244 - 255 */
+    };
+    
+    private final static byte BAD_ENCODING    = -9; // Indicates error in encoding
+    private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding
+    private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding
+
+    
+    /** Defeats instantiation. */
+    private Base64(){}
+    
+    
+    
+/* ********  E N C O D I N G   M E T H O D S  ******** */    
+    
+    
+    /**
+     * Encodes the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     *
+     * @param threeBytes the array to convert
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes )
+    {   
+        return encode3to4( threeBytes, 3 );
+    }   // end encodeToBytes
+    
+    
+    
+    /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     *
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes, int numSigBytes )
+    {  
+        byte[] dest = new byte[4];
+        encode3to4( threeBytes, 0, numSigBytes, dest, 0 );
+        return dest;
+    }
+    
+     /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     * Code can reuse a byte array by passing a four-byte array as <var>b4</var>.
+     *
+     * @param b4 A reusable byte array to reduce array instantiation
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.5.1
+     */
+    private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes )
+    {
+        encode3to4( threeBytes, 0, numSigBytes, b4, 0 );
+        return b4;
+    }   // end encode3to4
+
+    
+    /**
+     * Encodes up to three bytes of the array <var>source</var>
+     * and writes the resulting four Base64 bytes to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 3 for
+     * the <var>source</var> array or <var>destOffset</var> + 4 for
+     * the <var>destination</var> array.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param numSigBytes the number of significant bytes in your array
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the <var>destination</var> array
+     * @since 1.3
+     */
+    private static byte[] encode3to4( 
+     byte[] source, int srcOffset, int numSigBytes,
+     byte[] destination, int destOffset )
+    {
+        //           1         2         3  
+        // 01234567890123456789012345678901 Bit position
+        // --------000000001111111122222222 Array position from threeBytes
+        // --------|    ||    ||    ||    | Six bit groups to index ALPHABET
+        //          >>18  >>12  >> 6  >> 0  Right shift necessary
+        //                0x3f  0x3f  0x3f  Additional AND
+        
+        // Create buffer with zero-padding if there are only one or two
+        // significant bytes passed in the array.
+        // We have to shift left 24 in order to flush out the 1's that appear
+        // when Java treats a value as negative that is cast from a byte to an int.
+        int inBuff =   ( numSigBytes > 0 ? ((source[ srcOffset     ] << 24) >>>  8) : 0 )
+                     | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 )
+                     | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 );
+
+        switch( numSigBytes )
+        {
+            case 3:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = ALPHABET[ (inBuff       ) & 0x3f ];
+                return destination;
+                
+            case 2:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            case 1:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = EQUALS_SIGN;
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            default:
+                return destination;
+        }   // end switch
+    }   // end encode3to4
+    
+    
+    
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * The object is not GZip-compressed before being encoded.
+     *
+     * @param serializableObject The object to encode
+     * @return The Base64-encoded object
+     * @since 1.4
+     */
+    public static String encodeObject( java.io.Serializable serializableObject )
+    {
+        return encodeObject( serializableObject, NO_OPTIONS );
+    }   // end encodeObject
+    
+
+
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     * @param serializableObject The object to encode
+     * @param options Specified options
+     * @return The Base64-encoded object
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeObject( java.io.Serializable serializableObject, int options )
+    {
+        // Streams
+        java.io.ByteArrayOutputStream  baos  = null; 
+        java.io.OutputStream           b64os = null; 
+        java.io.ObjectOutputStream     oos   = null; 
+        java.util.zip.GZIPOutputStream gzos  = null;
+        
+        // Isolate options
+        int gzip           = (options & GZIP);
+        int dontBreakLines = (options & DONT_BREAK_LINES);
+        
+        try
+        {
+            // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream
+            baos  = new java.io.ByteArrayOutputStream();
+            b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+    
+            // GZip?
+            if( gzip == GZIP )
+            {
+                gzos = new java.util.zip.GZIPOutputStream( b64os );
+                oos  = new java.io.ObjectOutputStream( gzos );
+            }   // end if: gzip
+            else
+                oos   = new java.io.ObjectOutputStream( b64os );
+            
+            oos.writeObject( serializableObject );
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            return null;
+        }   // end catch
+        finally
+        {
+            try{ oos.close();   } catch( Exception e ){}
+            try{ gzos.close();  } catch( Exception e ){}
+            try{ b64os.close(); } catch( Exception e ){}
+            try{ baos.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return new String( baos.toByteArray() );
+    }   // end encode
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source )
+    {
+        return encodeBytes( source, 0, source.length, NO_OPTIONS );
+    }   // end encodeBytes
+    
+
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int options )
+    {   
+        return encodeBytes( source, 0, source.length, options );
+    }   // end encodeBytes
+    
+    
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source, int off, int len )
+    {
+        return encodeBytes( source, off, len, NO_OPTIONS );
+    }   // end encodeBytes
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int off, int len, int options )
+    {
+        // Isolate options
+        int dontBreakLines = ( options & DONT_BREAK_LINES );
+        int gzip           = ( options & GZIP   );
+        
+        // Compress?
+        if( gzip == GZIP )
+        {
+            java.io.ByteArrayOutputStream  baos  = null;
+            java.util.zip.GZIPOutputStream gzos  = null;
+            Base64.OutputStream            b64os = null;
+            
+    
+            try
+            {
+                // GZip -> Base64 -> ByteArray
+                baos = new java.io.ByteArrayOutputStream();
+                b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+                gzos  = new java.util.zip.GZIPOutputStream( b64os ); 
+            
+                gzos.write( source, off, len );
+                gzos.close();
+            }   // end try
+            catch( java.io.IOException e )
+            {
+                e.printStackTrace();
+                return null;
+            }   // end catch
+            finally
+            {
+                try{ gzos.close();  } catch( Exception e ){}
+                try{ b64os.close(); } catch( Exception e ){}
+                try{ baos.close();  } catch( Exception e ){}
+            }   // end finally
+
+            return new String( baos.toByteArray() );
+        }   // end if: compress
+        
+        // Else, don't compress. Better not to use streams at all then.
+        else
+        {
+            // Convert option to boolean in way that code likes it.
+            boolean breakLines = dontBreakLines == 0;
+            
+            int    len43   = len * 4 / 3;
+            byte[] outBuff = new byte[   ( len43 )                      // Main 4:3
+                                       + ( (len % 3) > 0 ? 4 : 0 )      // Account for padding
+                                       + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines      
+            int d = 0;
+            int e = 0;
+            int len2 = len - 2;
+            int lineLength = 0;
+            for( ; d < len2; d+=3, e+=4 )
+            {
+                encode3to4( source, d+off, 3, outBuff, e );
+
+                lineLength += 4;
+                if( breakLines && lineLength == MAX_LINE_LENGTH )
+                {   
+                    outBuff[e+4] = NEW_LINE;
+                    e++;
+                    lineLength = 0;
+                }   // end if: end of line
+            }   // en dfor: each piece of array
+
+            if( d < len )
+            {
+                encode3to4( source, d+off, len - d, outBuff, e );
+                e += 4;
+            }   // end if: some padding needed
+
+            return new String( outBuff, 0, e );
+        }   // end else: don't compress
+        
+    }   // end encodeBytes
+    
+
+    
+    
+    
+/* ********  D E C O D I N G   M E T H O D S  ******** */
+    
+    
+    /**
+     * Decodes the first four bytes of array <var>fourBytes</var>
+     * and returns an array up to three bytes long with the
+     * decoded values.
+     *
+     * @param fourBytes the array with Base64 content
+     * @return array with decoded values
+     * @since 1.3
+     */
+    private static byte[] decode4to3( byte[] fourBytes )
+    {
+        byte[] outBuff1 = new byte[3];
+        int    count    = decode4to3( fourBytes, 0, outBuff1, 0 );
+        byte[] outBuff2 = new byte[ count ];
+        
+        for( int i = 0; i < count; i++ )
+            outBuff2[i] = outBuff1[i];
+        
+        return outBuff2;
+    }
+    
+    
+    
+    
+    /**
+     * Decodes four bytes from array <var>source</var>
+     * and writes the resulting bytes (up to three of them)
+     * to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 4 for
+     * the <var>source</var> array or <var>destOffset</var> + 3 for
+     * the <var>destination</var> array.
+     * This method returns the actual number of bytes that 
+     * were converted from the Base64 encoding.
+     * 
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the number of decoded bytes converted
+     * @since 1.3
+     */
+    private static int decode4to3( byte[] source, int srcOffset, byte[] destination, int destOffset )
+    {
+        // Example: Dk==
+        if( source[ srcOffset + 2] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 );
+            
+            destination[ destOffset ] = (byte)( outBuff >>> 16 );
+            return 1;
+        }
+        
+        // Example: DkL=
+        else if( source[ srcOffset + 3 ] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6 );
+            
+            destination[ destOffset     ] = (byte)( outBuff >>> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>>  8 );
+            return 2;
+        }
+        
+        // Example: DkLE
+        else
+        {
+            try{
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 )
+            //              | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6)
+                          | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF )      );
+
+            
+            destination[ destOffset     ] = (byte)( outBuff >> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>  8 );
+            destination[ destOffset + 2 ] = (byte)( outBuff       );
+
+            return 3;
+            }catch( Exception e){
+                System.out.println(""+source[srcOffset]+ ": " + ( DECODABET[ source[ srcOffset     ] ]  ) );
+                System.out.println(""+source[srcOffset+1]+  ": " + ( DECODABET[ source[ srcOffset + 1 ] ]  ) );
+                System.out.println(""+source[srcOffset+2]+  ": " + ( DECODABET[ source[ srcOffset + 2 ] ]  ) );
+                System.out.println(""+source[srcOffset+3]+  ": " + ( DECODABET[ source[ srcOffset + 3 ] ]  ) );
+                return -1;
+            }   //e nd catch
+        }
+    }   // end decodeToBytes
+    
+    
+    
+    
+    /**
+     * Very low-level access to decoding ASCII characters in
+     * the form of a byte array. Does not support automatically
+     * gunzipping or any other "fancy" features.
+     *
+     * @param source The Base64 encoded data
+     * @param off    The offset of where to begin decoding
+     * @param len    The length of characters to decode
+     * @return decoded data
+     * @since 1.3
+     */
+    public static byte[] decode( byte[] source, int off, int len )
+    {
+        int    len34   = len * 3 / 4;
+        byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output
+        int    outBuffPosn = 0;
+        
+        byte[] b4        = new byte[4];
+        int    b4Posn    = 0;
+        int    i         = 0;
+        byte   sbiCrop   = 0;
+        byte   sbiDecode = 0;
+        for( i = off; i < off+len; i++ )
+        {
+            sbiCrop = (byte)(source[i] & 0x7f); // Only the low seven bits
+            sbiDecode = DECODABET[ sbiCrop ];
+            
+            if( sbiDecode >= WHITE_SPACE_ENC ) // White space, Equals sign or better
+            {
+                if( sbiDecode >= EQUALS_SIGN_ENC )
+                {
+                    b4[ b4Posn++ ] = sbiCrop;
+                    if( b4Posn > 3 )
+                    {
+                        outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn );
+                        b4Posn = 0;
+                        
+                        // If that was the equals sign, break out of 'for' loop
+                        if( sbiCrop == EQUALS_SIGN )
+                            break;
+                    }   // end if: quartet built
+                    
+                }   // end if: equals sign or better
+                
+            }   // end if: white space, equals sign or better
+            else
+            {
+                System.err.println( "Bad Base64 input character at " + i + ": " + source[i] + "(decimal)" );
+                return null;
+            }   // end else: 
+        }   // each input character
+                                   
+        byte[] out = new byte[ outBuffPosn ];
+        System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); 
+        return out;
+    }   // end decode
+    
+    
+    
+    
+    /**
+     * Decodes data from Base64 notation, automatically
+     * detecting gzip-compressed data and decompressing it.
+     *
+     * @param s the string to decode
+     * @return the decoded data
+     * @since 1.4
+     */
+    public static byte[] decode( String s )
+    {   
+        byte[] bytes = s.getBytes();
+        bytes = decode( bytes, 0, bytes.length );
+        
+        // Check to see if it's gzip-compressed
+        // GZIP Magic Two-Byte Number: 0x8b1f (35615)
+        if( bytes.length >= 2 )
+        {
+            
+            int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);       
+            if(
+            bytes != null &&      // In case decoding returned null
+            bytes.length >= 4  && // Don't want to get ArrayIndexOutOfBounds exception
+            java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) 
+            {
+                java.io.ByteArrayInputStream  bais = null;
+                java.util.zip.GZIPInputStream gzis = null;
+                java.io.ByteArrayOutputStream baos = null;
+                byte[] buffer = new byte[2048];
+                int    length = 0;
+
+                try
+                {
+                    baos = new java.io.ByteArrayOutputStream();
+                    bais = new java.io.ByteArrayInputStream( bytes );
+                    gzis = new java.util.zip.GZIPInputStream( bais );
+
+                    while( ( length = gzis.read( buffer ) ) >= 0 )
+                    {
+                        baos.write(buffer,0,length);
+                    }   // end while: reading input
+
+                    // No error? Get new bytes.
+                    bytes = baos.toByteArray();
+
+                }   // end try
+                catch( java.io.IOException e )
+                {
+                    // Just return originally-decoded bytes
+                }   // end catch
+                finally
+                {
+                    try{ baos.close(); } catch( Exception e ){}
+                    try{ gzis.close(); } catch( Exception e ){}
+                    try{ bais.close(); } catch( Exception e ){}
+                }   // end finally
+
+            }   // end if: gzipped
+        }   // end if: bytes.length >= 2
+        
+        return bytes;
+    }   // end decode
+
+
+    
+
+    /**
+     * Attempts to decode Base64 data and deserialize a Java
+     * Object within. Returns <tt>null</tt> if there was an error.
+     *
+     * @param encodedObject The Base64 data to decode
+     * @return The decoded and deserialized object
+     * @since 1.5
+     */
+    public static Object decodeToObject( String encodedObject )
+    {
+        // Decode and gunzip if necessary
+        byte[] objBytes = decode( encodedObject );
+        
+        java.io.ByteArrayInputStream  bais = null;
+        java.io.ObjectInputStream     ois  = null;
+        Object obj = null;
+        
+        try
+        {
+            bais = new java.io.ByteArrayInputStream( objBytes );
+            ois  = new java.io.ObjectInputStream( bais );
+        
+            obj = ois.readObject();
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        catch( java.lang.ClassNotFoundException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        finally
+        {
+            try{ bais.close(); } catch( Exception e ){}
+            try{ ois.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return obj;
+    }   // end decodeObject
+    
+    
+    /* ********  I N N E R   C L A S S   I N P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * It will read data from another
+     * {@link java.io.InputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterInputStream
+     * @since 1.3
+     */
+    public static class InputStream extends java.io.FilterInputStream
+    {
+        private int     options;        // Options specified
+        private boolean encode;         // Encoding or decoding
+        private int     position;       // Current position in the buffer
+        private byte[]  buffer;         // Small buffer holding converted data
+        private int     bufferLength;   // Length of buffer (3 or 4)
+        private int     numSigBytes;    // Number of meaningful bytes in the buffer
+        private int     lineLength;
+        private boolean breakLines;     // Break lines at less than 80 characters
+        
+        
+        /**
+         * Constructs a InputStream in DECODE mode.
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @since 1.3
+         */
+        public InputStream( java.io.InputStream in )
+        {   
+            this( in, DECODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a InputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.InputStream( in, Base64.DECODE )</code>
+         *
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @param options Specified options
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 2.0
+         */
+        public InputStream( java.io.InputStream in, int options )
+        {   
+            super( in );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.breakLines = breakLines;
+            this.encode = encode;
+            this.bufferLength = encode ? 4 : 3;
+            this.buffer   = new byte[ bufferLength ];
+            this.position = -1;
+            this.lineLength = 0;
+        }   // end constructor
+        
+        /**
+         * Reads enough of the input stream to convert
+         * to/from Base64 and returns the next byte.
+         *
+         * @return next byte
+         * @since 1.3
+         */
+        public int read() throws java.io.IOException 
+        { 
+            // Do we need to get data?
+            if( position < 0 )
+            {
+                if( encode )
+                {
+                    byte[] b3 = new byte[3];
+                    int numBinaryBytes = 0;
+                    for( int i = 0; i < 3; i++ )
+                    {
+                        try
+                        { 
+                            int b = in.read();
+                            
+                            // If end of stream, b is -1.
+                            if( b >= 0 )
+                            {
+                                b3[i] = (byte)b;
+                                numBinaryBytes++;
+                            }   // end if: not end of stream
+                            
+                        }   // end try: read
+                        catch( java.io.IOException e )
+                        {   
+                            // Only a problem if we got no data at all.
+                            if( i == 0 )
+                                throw e;
+                            
+                        }   // end catch
+                    }   // end for: each needed input byte
+                    
+                    if( numBinaryBytes > 0 )
+                    {
+                        encode3to4( b3, 0, numBinaryBytes, buffer, 0 );
+                        position = 0;
+                        numSigBytes = 4;
+                    }   // end if: got data
+                    else
+                    {
+                        return -1;
+                    }   // end else
+                }   // end if: encoding
+                
+                // Else decoding
+                else
+                {
+                    byte[] b4 = new byte[4];
+                    int i = 0;
+                    for( i = 0; i < 4; i++ )
+                    {
+                        // Read four "meaningful" bytes:
+                        int b = 0;
+                        do{ b = in.read(); }
+                        while( b >= 0 && DECODABET[ b & 0x7f ] <= WHITE_SPACE_ENC );
+                        
+                        if( b < 0 )
+                            break; // Reads a -1 if end of stream
+                        
+                        b4[i] = (byte)b;
+                    }   // end for: each needed input byte
+                    
+                    if( i == 4 )
+                    {
+                        numSigBytes = decode4to3( b4, 0, buffer, 0 );
+                        position = 0;
+                    }   // end if: got four characters
+                    else if( i == 0 ){
+                        return -1;
+                    }   // end else if: also padded correctly
+                    else
+                    {
+                        // Must have broken out from above.
+                        throw new java.io.IOException( "Improperly padded Base64 input." );
+                    }   // end 
+                    
+                }   // end else: decode
+            }   // end else: get data
+            
+            // Got data?
+            if( position >= 0 )
+            {
+                // End of relevant data?
+                if( /*!encode &&*/ position >= numSigBytes )
+                    return -1;
+                
+                if( encode && breakLines && lineLength >= MAX_LINE_LENGTH )
+                {
+                    lineLength = 0;
+                    return '\n';
+                }   // end if
+                else
+                {
+                    lineLength++;   // This isn't important when decoding
+                                    // but throwing an extra "if" seems
+                                    // just as wasteful.
+                    
+                    int b = buffer[ position++ ];
+
+                    if( position >= bufferLength )
+                        position = -1;
+
+                    return b & 0xFF; // This is how you "cast" a byte that's
+                                     // intended to be unsigned.
+                }   // end else
+            }   // end if: position >= 0
+            
+            // Else error
+            else
+            {   
+                // When JDK1.4 is more accepted, use an assertion here.
+                throw new java.io.IOException( "Error in Base64 code reading stream." );
+            }   // end else
+        }   // end read
+        
+        
+        /**
+         * Calls {@link #read} repeatedly until the end of stream
+         * is reached or <var>len</var> bytes are read.
+         * Returns number of bytes read into array or -1 if
+         * end of stream is encountered.
+         *
+         * @param dest array to hold values
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @return bytes read into array or -1 if end of stream is encountered.
+         * @since 1.3
+         */
+        public int read( byte[] dest, int off, int len ) throws java.io.IOException
+        {
+            int i;
+            int b;
+            for( i = 0; i < len; i++ )
+            {
+                b = read();
+                
+                //if( b < 0 && i == 0 )
+                //    return -1;
+                
+                if( b >= 0 )
+                    dest[off + i] = (byte)b;
+                else if( i == 0 )
+                    return -1;
+                else
+                    break; // Out of 'for' loop
+            }   // end for: each byte read
+            return i;
+        }   // end read
+        
+    }   // end inner class InputStream
+    
+    
+    
+    
+    
+    
+    /* ********  I N N E R   C L A S S   O U T P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * A OutputStream will write data to another
+     * {@link java.io.OutputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterOutputStream
+     * @since 1.3
+     */
+    public static class OutputStream extends java.io.FilterOutputStream
+    {
+        private int     options;
+        private boolean encode;
+        private int     position;
+        private byte[]  buffer;
+        private int     bufferLength;
+        private int     lineLength;
+        private boolean breakLines;
+        private byte[]  b4; // Scratch used in a few places
+        private boolean suspendEncoding;
+        
+        /**
+         * Constructs a OutputStream in ENCODE mode.
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out )
+        {   
+            this( out, ENCODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a OutputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.OutputStream( out, Base64.ENCODE )</code>
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @param options Specified options.
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out, int options )
+        {   
+            super( out );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.bufferLength = encode ? 3 : 4;
+            this.buffer       = new byte[ bufferLength ];
+            this.position     = 0;
+            this.lineLength   = 0;
+            this.suspendEncoding = false;
+            this.b4           = new byte[4];
+        }   // end constructor
+        
+        
+        /**
+         * Writes the byte to the output stream after
+         * converting to/from Base64 notation.
+         * When encoding, bytes are buffered three
+         * at a time before the output stream actually
+         * gets a write() call.
+         * When decoding, bytes are buffered four
+         * at a time.
+         *
+         * @param theByte the byte to write
+         * @since 1.3
+         */
+        public void write(int theByte) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theByte );
+                return;
+            }   // end if: supsended
+            
+            // Encode?
+            if( encode )
+            {
+                buffer[ position++ ] = (byte)theByte;
+                if( position >= bufferLength )  // Enough to encode.
+                {
+                    out.write( encode3to4( b4, buffer, bufferLength ) );
+
+                    lineLength += 4;
+                    if( breakLines && lineLength >= MAX_LINE_LENGTH )
+                    {
+                        out.write( NEW_LINE );
+                        lineLength = 0;
+                    }   // end if: end of line
+
+                    position = 0;
+                }   // end if: enough to output
+            }   // end if: encoding
+
+            // Else, Decoding
+            else
+            {
+                // Meaningful Base64 character?
+                if( DECODABET[ theByte & 0x7f ] > WHITE_SPACE_ENC )
+                {
+                    buffer[ position++ ] = (byte)theByte;
+                    if( position >= bufferLength )  // Enough to output.
+                    {
+                        int len = Base64.decode4to3( buffer, 0, b4, 0 );
+                        out.write( b4, 0, len );
+                        //out.write( Base64.decode4to3( buffer ) );
+                        position = 0;
+                    }   // end if: enough to output
+                }   // end if: meaningful base64 character
+                else if( DECODABET[ theByte & 0x7f ] != WHITE_SPACE_ENC )
+                {
+                    throw new java.io.IOException( "Invalid character in Base64 data." );
+                }   // end else: not white space either
+            }   // end else: decoding
+        }   // end write
+        
+        
+        
+        /**
+         * Calls {@link #write} repeatedly until <var>len</var> 
+         * bytes are written.
+         *
+         * @param theBytes array from which to read bytes
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @since 1.3
+         */
+        public void write( byte[] theBytes, int off, int len ) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theBytes, off, len );
+                return;
+            }   // end if: supsended
+            
+            for( int i = 0; i < len; i++ )
+            {
+                write( theBytes[ off + i ] );
+            }   // end for: each byte written
+            
+        }   // end write
+        
+        
+        
+        /**
+         * Method added by PHIL. [Thanks, PHIL. -Rob]
+         * This pads the buffer without closing the stream.
+         */
+        public void flushBase64() throws java.io.IOException 
+        {
+            if( position > 0 )
+            {
+                if( encode )
+                {
+                    out.write( encode3to4( b4, buffer, position ) );
+                    position = 0;
+                }   // end if: encoding
+                else
+                {
+                    throw new java.io.IOException( "Base64 input not properly padded." );
+                }   // end else: decoding
+            }   // end if: buffer partially full
+
+        }   // end flush
+
+        
+        /** 
+         * Flushes and closes (I think, in the superclass) the stream. 
+         *
+         * @since 1.3
+         */
+        public void close() throws java.io.IOException
+        {
+            // 1. Ensure that pending characters are written
+            flushBase64();
+
+            // 2. Actually close the stream
+            // Base class both flushes and closes.
+            super.close();
+            
+            buffer = null;
+            out    = null;
+        }   // end close
+        
+        
+        
+        /**
+         * Suspends encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void suspendEncoding() throws java.io.IOException 
+        {
+            flushBase64();
+            this.suspendEncoding = true;
+        }   // end suspendEncoding
+        
+        
+        /**
+         * Resumes encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void resumeEncoding()
+        {
+            this.suspendEncoding = false;
+        }   // end resumeEncoding
+        
+        
+        
+    }   // end inner class OutputStream
+    
+    
+}   // end class Base64
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Hgram.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Hgram.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Hgram.java	(revision 14150)
@@ -0,0 +1,378 @@
+package vishnu.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.imageio.*;
+import javax.imageio.stream.*;
+import java.io.*;
+import java.util.*;
+import java.awt.image.*;
+
+public class Hgram extends JFrame
+{
+    public static int TOP = 0;
+    public static int LARGEST = 1;
+
+    // data is an array with each entry being the frequency of a bin
+    public static int BIN_DATA = 0;
+
+    // data comes in raw format and needs to be allocated to bins
+    public static int RAW_DATA = 1;
+    
+    double[] data;
+    
+    // default size of frame
+    int width = 400;
+    int height = 400;
+
+    // axis limits of histogram
+    double xmin;
+    double xmax;
+
+    // number of bins, bin frequencies and width
+    int noBins;
+    double bins[];
+    double binWidth;
+    
+    BufferedImage offScreen=null;
+    Dimension offScreenSize;
+    Graphics2D offGraphics;
+    
+    String label = null;
+    
+    double median = -1;
+    int med_bin = -1;
+    int mode = -1;
+    
+    
+    public Hgram()
+    {
+	init();
+    }
+
+    
+    public void setData(double[] d, int m){
+	data = d; 
+	mode = m;
+	xmin = getMinimum(data);
+	xmax = getMaximum(data);
+	createBins(data,noBins,xmin, xmax);
+    }
+
+      
+    public void setBins(double[] b){ bins = b; }
+
+
+    public void setBinNumber(int i){ noBins = i; }
+
+    
+    public void setMaximum(double i){ xmax = i; }
+    
+    
+    public void setMinimum(double i){ xmin = i; }
+
+
+    public void setBinWidth(double i){ binWidth = i; }
+    
+    
+    public void setLabel(String l){ label = l; }
+  
+
+    public void init()
+    {
+	this.setDefaultLookAndFeelDecorated(true);
+        this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+	this.setSize(width,height);
+	this.setLocation(0,0);
+	this.setVisible(true);
+	
+	this.addComponentListener( new ComponentAdapter(){
+		public void componentResized(ComponentEvent evt){
+		    update();
+		}
+	    });
+	
+	
+	this.addMouseListener(new MouseAdapter(){
+		public void mousePressed(MouseEvent evt)
+		{
+    		    int intX = evt.getX();
+	    	    int intY = evt.getY();
+		}
+		
+		public void mouseClicked(MouseEvent event)
+		{
+		    int x = event.getX();
+		    int y = event.getY();
+		    
+		    if( event.getClickCount() == 1 )
+			{
+			}
+		}
+	    });
+	
+	
+	this.addMouseMotionListener(new MouseMotionAdapter()
+	    {
+		public void mouseMoved(MouseEvent event)
+		{
+		    int x = event.getX();
+		    int y = event.getY();
+		    getBin(x,y);
+		}
+	    });
+    }
+
+    
+    public int getBin(int x, int y){ return -1; }
+    
+    
+    private int getMaxBin()
+    {
+	int maxIndex = -1;
+	double max = 0;
+	
+	for( int i = 0; i < bins.length; i++ ){
+	    if( bins[i] > max ){
+		max = bins[i];
+		maxIndex = i;
+	    }
+	}
+	
+	return maxIndex;
+    }
+    
+    
+    public void plot()
+    {
+	update();
+    }
+    
+      
+    public double getMaximum(double[] d)
+    {
+	double max = 0; 
+
+	for( int i = 0; i < d.length; i++ ){
+	    if( d[i] > max ) max = d[i];
+	}
+	return max;
+    }
+ 
+
+
+    public double getMinimum(double[] d)
+    {
+	double min = d[0]*d[0]*1000;
+
+	for( int i = 0; i < d.length; i++ )
+	    {
+		if( d[i] < min ) min = d[i];
+	    }
+	return min;
+    }
+
+
+    public double getFirstMaximum(double[] d)
+    {
+	double max = 0;
+	for( int i = 0; i < bins.length; i++ )
+	    {
+		if( bins[i] > max )
+		    {
+			max = bins[i];
+		    }
+	    }
+	return max;
+    }
+
+
+    public double getSecondMaximum(double[] d)
+    {
+	double max_1 = 0;
+	double max_2 = 0;
+	for( int i = 0; i < d.length; i++ ){
+	    if( d[i] > max_1 ){
+		max_2 = max_1;
+		max_1 = d[i];
+	    }
+	    else if( d[i] > max_2 ){
+		max_2 = d[i];
+	    }
+	}
+	return max_2;
+    }
+
+ 
+    public void createBins(double[] data, int noBins, double xmin, double xmax)
+    {
+	bins = new double[noBins];
+	binWidth = (double)(xmax-xmin)/noBins;
+	
+	for( int i = 0; i < data.length; i++ ){
+	    int bin = (int)Math.round((double)(data[i]-xmin)/binWidth);
+	    if( bin >= noBins ) bin = noBins-1;
+	    bins[bin]++;
+	}
+	
+	for( int i = 0; i < noBins; i++ ){
+	    bins[i] = (double)bins[i]/data.length;
+	}
+    }
+    
+    
+    public void update()
+    {
+	if( bins == null ) return;
+	
+	Dimension d = this.getSize();
+	width = d.width;
+	height = d.height;
+        offScreen = new BufferedImage( width, height, BufferedImage.TYPE_INT_ARGB ); 
+	offGraphics = (Graphics2D)offScreen.getGraphics();
+        offGraphics.setColor(Color.white);
+        offGraphics.fillRect(0,0,width,height);
+    	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+        offGraphics.setStroke(myStroke);	
+	
+	int xaxis = (int)(0.8 * width); 
+	int xBuffer = (int)(0.1 * width);
+	
+	int yaxis = (int)(0.8 * d.height);
+	int yBuffer = (int)(0.1 * d.height);
+	
+	double maxBinLength = (double)xaxis/bins.length;
+	int effBinLength = (int)(0.8 * maxBinLength);
+	int binBuffer = (int)(0.1 * effBinLength);
+	
+	double max_1 = getFirstMaximum(bins);
+	double max_2 = getSecondMaximum(bins);
+	double max = max_1;
+	boolean cut = false;
+	
+	if( max_1 > 1.5 * max_2 ){
+	    cut = true;
+	    max = max_2;
+	}
+	
+	double pixelPerUnit = (double)yaxis/max; 
+	
+
+	/**** draw the bins ****/
+
+	for( int i = 0; i < bins.length; i++ ){
+	    int level = (int)(bins[i]* pixelPerUnit);
+	    double bound = (double)((int)(100 * (i * (double)(xmax-xmin)/bins.length + xmin)))/100;
+	    offGraphics.setColor(Color.blue);
+	    offGraphics.fillRect(xBuffer + (int)(i*maxBinLength)+binBuffer,height-yBuffer-level,effBinLength,level);
+	}
+
+	if( label != null ){
+	    FontMetrics fm   = offGraphics.getFontMetrics(offGraphics.getFont());
+	    int x = width/2 - fm.stringWidth(label)/2;		
+	    offGraphics.drawString(label,x,yBuffer);
+	}
+
+
+	/**** draw axes and ticks ****/
+
+	offGraphics.setColor(Color.blue);
+	
+	int x = xBuffer + binBuffer;
+	int y = height-yBuffer;
+	offGraphics.drawLine(x, y+5, x, y-5);
+	
+	String tickStr = String.valueOf(xmin);
+	offGraphics.drawString(tickStr,x-5,y+20);
+	
+	x = xBuffer + binBuffer+(int)(bins.length*maxBinLength)+binBuffer;
+	y = height-yBuffer;
+	offGraphics.drawLine(x, y+5, x, y-5);
+	tickStr = String.valueOf(xmax);
+	offGraphics.drawString(tickStr,x-5,y+20);
+	
+        repaint();
+    }
+    
+    
+    public void paint(Graphics g)
+    {
+        Graphics2D g2 = (Graphics2D)g;
+	
+	if( offScreen!=null )
+	    g2.drawImage(offScreen, 0, 0, null);
+    }
+    
+    
+    public void printBins()
+    {
+	for( int i = 0; i < bins.length; i++ ){
+	    double low = i*binWidth;
+	    double up = (i+1)*binWidth;
+	    System.out.println("Bin.." + i + " " + low + " " + up + " " + bins[i]); 
+	}
+    }
+
+
+    public void save(String name, String type)
+    {
+	
+	try{
+	    Iterator imageWriters =
+		ImageIO.getImageWritersByFormatName(type);
+	    ImageWriter imageWriter = (ImageWriter)imageWriters.next();
+	    File file = new File( name + "." + type );
+	    ImageOutputStream ios =
+		ImageIO.createImageOutputStream( file );
+	    imageWriter.setOutput( ios );
+	    imageWriter.write( offScreen );
+	} catch( IOException ie ) {
+	    ie.printStackTrace();
+	}
+    }
+
+}
+
+ // accepts raw data
+    /*public Hgram(double[] data, int noBins, double min, double max)
+    {
+	super("JHistogram");
+	init();
+	createBins(data,noBins,min,max);
+	update();
+	}*/
+    
+    
+    // accepts both raw and binned data
+    /*public Hgram(double[] data, int noBins, int mode)
+    {
+	super("JHistogram");
+	this.mode = mode;
+	init();
+	createBins(data,noBins,getMinimum(data),getMaximum(data));
+	update();
+	}*/
+    
+
+
+/*
+	int level = (int)(bins[med_bin]* pixelPerUnit);
+	offGraphics.fillRect(xBuffer + (int)(med_bin*maxBinLength)+binBuffer,height-yBuffer-level,effBinLength,level);
+	String med_string = String.valueOf(median);
+	med_string = med_string.substring(0,med_string.indexOf('.')+3);
+	offGraphics.drawString(med_string, xBuffer + (int)(med_bin*maxBinLength)+binBuffer-10,height-yBuffer-level-10);
+	    
+	//saveHistogram(offScreen);
+	
+	repaint();
+	    }
+	    }*/
+
+  /*
+    public Hgram(double[] data, int noBins)
+    {
+	init();
+	createBins(data,noBins,getMinimum(data),getFirstMaximum(data));
+	update();
+	}*/
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Misc.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Misc.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Misc.java	(revision 14150)
@@ -0,0 +1,240 @@
+package vishnu.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+
+public class Misc {
+
+    // quickSort
+    private static void qs(int l, int r, double v[][])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		double pivot = v[1][(l+r)/2];
+		do
+		{
+
+			while( v[1][i] < pivot )
+			{
+				i++;
+			} 
+			while( v[1][j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					double t1 = v[1][i];	
+					double t0 = v[0][i];
+					v[1][i] = v[1][j];
+					v[1][j] = t1;
+					v[0][i] = v[0][j];
+					v[0][j] = t0;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+		
+	}
+    }
+
+    
+    
+    // quickSort
+    private static void qs(int l, int r, double v[])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		double pivot = v[(l+r)/2];
+		do
+		{
+
+			while( v[i] < pivot )
+			{
+				i++;
+			} 
+			while( v[j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					double t1 = v[i];	
+					v[i] = v[j];
+					v[j] = t1;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+		
+	}
+    }
+
+
+ // quickSort
+    private static void qs(int l, int r, int v[])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		int pivot = v[(l+r)/2];
+		do
+		{
+
+			while( v[i] < pivot )
+			{
+				i++;
+			} 
+			while( v[j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					int t1 = v[i];	
+					v[i] = v[j];
+					v[j] = t1;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+	}
+    }
+
+
+ // quickSort
+    private static void qs(int l, int r, int[] v, Object[] objArr)
+    {
+        if( l < r ){
+            int i = l;
+            int j = r;
+
+            int pivot = v[(l+r)/2];
+
+            do{
+                while( v[i] < pivot ){
+                    i++;
+                }
+                while( v[j] > pivot ){
+                    j--;
+                }
+
+                if( i <= j ){
+                    if( i < j ){
+                        int t1 = v[i];
+                        v[i] = v[j];
+                        v[j] = t1;
+
+                        Object tobj = objArr[i];
+                        objArr[i] = objArr[j];
+                        objArr[j] = tobj;
+ }
+                    i++;
+                    j--;
+                }
+
+            }
+            while( i <= j );
+            qs(l,j,v,objArr);
+            qs(i,r,v,objArr);
+        }
+    }
+
+
+    // quickSort
+    private static void qs(int l, int r, float[] v, Object[] objArr)
+    {
+	if( l < r ){
+	    int i = l;
+	    int j = r;
+	    
+	    float pivot = v[(l+r)/2];
+	    
+	    do{
+		while( v[i] < pivot ){
+		    i++;
+		} 
+		while( v[j] > pivot ){
+		    j--;
+		}
+		
+		if( i <= j ){
+		    if( i < j ){
+			float t1 = v[i];	
+			v[i] = v[j];
+			v[j] = t1;
+			
+			Object tobj = objArr[i];
+			objArr[i] = objArr[j];
+			objArr[j] = tobj;
+		    }
+		    i++;
+		    j--;
+		}
+		
+	    }
+	    while( i <= j );
+	    qs(l,j,v,objArr);
+	    qs(i,r,v,objArr);
+	}
+    }
+    
+
+    public static void quickSort(int[] v)
+    {
+	qs(0,v.length-1,v);
+    }
+
+    public static void quickSort(float[] v, Object[] objArr)
+    {
+	qs(0,v.length-1,v,objArr);
+    }
+
+    public static void quickSort(int[] v, Object[] objArr)
+    {
+	qs(0,v.length-1,v,objArr);
+    }
+
+    public static void quickSort(double v[])
+    {
+	qs(0,v.length-1,v);
+    }
+
+
+    public static void quickSort(double v[][])
+    {
+	qs(0,v[0].length-1,v);
+    }
+}
+    
+    
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Sammon.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Sammon.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/Sammon.java	(revision 14150)
@@ -0,0 +1,247 @@
+package vishnu.util;
+
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.Point2D;
+
+public class Sammon
+{
+
+    public static final double MF =       0.3;		/* Magic Factor          */
+    public static final double _MAX_Err = 0.00005;	/* Max Mapping Error     */
+    public static final int _MAX_M =  1000;		/* Max Iteration Number  */
+    public static final int _MAX_N =  1000;		/* Max Number of Entries */
+    public static final int _MAX_L =  128;		/* Max Dimension for X   */
+
+
+    double[][] Xs;
+    double[][] Ys;
+    int number;
+    int dimension;
+    double total_Xs_dists;
+
+/* SS298: Added function for scaling values between 0 and 1
+* n : number of entries
+*/
+    void scale_values(int n)
+    {
+
+	// get the maximum and the minimum of Ys array
+
+    // get the maximum and the minimum of Ys array
+
+        double maxX = Ys[0][0];
+        double minX = Ys[0][0];
+        double maxY = Ys[0][1];
+        double minY = Ys[0][1];
+
+        int i;
+        for (i = 1 ; i < n ; i++)
+        {
+
+            if (maxX < Ys[i][0]) maxX = Ys[i][0];
+            if (minX > Ys[i][0]) minX = Ys[i][0];
+            if (maxY < Ys[i][1]) maxY = Ys[i][1];
+            if (minY > Ys[i][1]) minY = Ys[i][1];
+        }
+        maxY -= minY;
+        maxX -= minX;
+        for (i = 0 ; i < n ; i++)
+        {
+            if (maxX > 0)
+            {
+                Ys[i][0] -= minX;
+                Ys[i][0] /= maxX;
+                Ys[i][0] *=0.8;
+                Ys[i][0] +=0.1;
+            }
+            else Ys[i][0]=0.5;
+            if (maxY > 0)
+            {
+                Ys[i][1] -= minY;
+                Ys[i][1] /= maxY;
+                Ys[i][1] *= 0.8;
+                Ys[i][1] += 0.1;
+            }
+            else Ys[i][1]=0.5;
+        }
+
+    }
+
+
+
+/* return a Euclidean distance between two 'd'-dimensional vectors 'xs' and 'ys'
+*/
+    double eucl_dist(int d, double[] xs, double[] ys)
+    {
+        double ans=0.0;
+        double tmp;
+        int i;
+        for(i=0;i<d;i++)
+        {
+            tmp = xs[i]-ys[i];
+            ans += tmp * tmp;
+        }
+        return Math.sqrt(ans);
+    }
+
+
+/* n - number of entries
+* d - dimension
+* post:
+*   return an array of eucidean distances for X
+*   and assign its total distance (will be used in the mapping error) to 'total'
+*/
+    double[] compute_Xs_eucl_dists()
+    {
+        double tmp[];
+        int i,j,k;
+        tmp = new double[number*(number-1)/2];
+
+        total_Xs_dists=0.0;
+        k=0;
+        for(i=1;i<number;i++)
+            for(j=0;j<i;j++)
+            {
+                tmp[k] = eucl_dist(dimension, Xs[i], Xs[j]);
+                total_Xs_dists+=tmp[k];
+                k++;
+            }
+            return tmp;
+    }
+    
+    
+    public Point2D [] getMapping()
+    {
+        Point2D [] sam = new Point2D[Ys.length];
+        for (int c=0;c<Ys.length;c++)
+	    {
+            float fx = (float) Ys[c][0];
+            float fy = (float) Ys[c][1];
+            sam[c]=new Point2D(fx,fy);
+        }
+        return sam;
+    }
+
+
+
+/*----------------------------------------------------------------*/
+    
+    public Sammon(double [][] centroids)
+    {
+	
+        int i, j, m, p;
+        double[] Xs_dists;
+
+        double dist_star,dist, alpha, beta, gamma, zeta, err;
+        double uppDer0, uppDer1, lowDer0, lowDer1;
+        double maxX=0.0;
+        double minX=0.0;
+        double maxY=0.0;
+        double minY=0.0;
+
+        number = centroids.length;
+        dimension = centroids[0].length;
+
+        Xs = centroids;
+        Ys = new double[number][2];
+
+        Xs_dists = compute_Xs_eucl_dists();
+
+        p=0;
+        for (i=0;i<number;i++)
+        {
+            Ys[i][0] =  Xs[i][0];
+            double value = Xs_dists.length>0?Xs_dists[(i==(number-1))?i:p]:
+                          0.0;
+            Ys[i][1] =  Xs[i][0]+value;
+            p+=(number-(i+1));
+        }
+
+        m=0;  /* iteration counter */
+        do{
+
+            for(p=0;p<number;p++)
+            {
+                uppDer0=uppDer1=lowDer0=lowDer1=0.0;
+
+
+                /* calcuate partial derivatives */
+
+                for(j=0;j<number;j++)
+                {
+                    if (p==j) continue;
+                    dist_star = Xs_dists[(j>p) ? (j*(j-1)/2+p) : (p*(p-1)/2+j)];
+
+                    dist = eucl_dist(2,Ys[p],Ys[j]);
+                    if (dist == 0) dist = 0.00001;
+                    alpha = dist_star-dist;
+                    beta = dist_star*dist;
+                    if (beta == 0) beta = 0.00000000000001;
+                    gamma = alpha/beta;
+
+                    /* 1st dimension */
+                    zeta = Ys[p][0]-Ys[j][0];
+                    uppDer0 += gamma * zeta;
+                    lowDer0 += (alpha-(zeta*zeta/dist) * (1+alpha/dist))/beta;
+
+                    /* 2nd dimension */
+                    zeta = Ys[p][1]-Ys[j][1];
+                    uppDer1 += gamma * zeta;
+                    lowDer1 += (alpha-(zeta*zeta/dist) * (1+alpha/dist))/beta;
+                }  /* for */
+
+                Ys[p][0] = Ys[p][0]+MF*uppDer0/Math.abs(lowDer0);
+                Ys[p][1] = Ys[p][1]+MF*uppDer1/Math.abs(lowDer1);
+                if(p==0)
+                {
+                    maxX = Ys[0][0];
+                    minX = Ys[0][0];
+                    maxY = Ys[0][1];
+                    minY = Ys[0][1];
+                }
+                else
+                {
+                  if (maxX < Ys[p][0]) maxX = Ys[p][0];
+                  if (minX > Ys[p][0]) minX = Ys[p][0];
+                  if (maxY < Ys[p][1]) maxY = Ys[p][1];
+                  if (minY > Ys[p][1]) minY = Ys[p][1];
+                }
+
+
+            }  /* for */
+
+
+        /* calcuate the mapping error */
+
+            err=0.0;
+            p=0;
+            for(j=1;j<number;j++)
+                for(i=0;i<j;i++)
+                {
+                    dist_star = Xs_dists[p++];
+                    alpha = dist_star - eucl_dist(2,Ys[j],Ys[i]);
+                    err += (alpha*alpha/dist_star);
+                }
+            err=err / total_Xs_dists;
+
+            m++;
+        } while ((err>_MAX_Err) && (m!=_MAX_M));
+
+
+        if(maxY==minY)
+        {
+          System.out.println("Warning: simulated 'y' values in mapping");
+          for(p=0;p<number;p++)
+            Ys[p][1]=Math.random();
+        }
+        if(maxX==minX)
+        {
+          System.out.println("Warning: simulated 'x' values in mapping");
+          for(p=0;p<number;p++)
+            Ys[p][0]=Math.random();
+        }
+
+        scale_values(number);
+    }
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/SpringUtilities.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/SpringUtilities.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/SpringUtilities.java	(revision 14150)
@@ -0,0 +1,194 @@
+package vishnu.util;
+
+import javax.swing.*;
+import javax.swing.SpringLayout;
+import java.awt.*;
+
+/**
+ * A 1.4 file that provides utility methods for
+ * creating form- or grid-style layouts with SpringLayout.
+ * These utilities are used by several programs, such as
+ * SpringBox and SpringCompactGrid.
+ */
+public class SpringUtilities {
+    /**
+     * A debugging utility that prints to stdout the component's
+     * minimum, preferred, and maximum sizes.
+     */
+    public static void printSizes(Component c) {
+        System.out.println("minimumSize = " + c.getMinimumSize());
+        System.out.println("preferredSize = " + c.getPreferredSize());
+        System.out.println("maximumSize = " + c.getMaximumSize());
+    }
+
+    /**
+     * Aligns the first <code>rows</code> * <code>cols</code>
+     * components of <code>parent</code> in
+     * a grid. Each component is as big as the maximum
+     * preferred width and height of the components.
+     * The parent is made just big enough to fit them all.
+     *
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param initialX x location to start the grid at
+     * @param initialY y location to start the grid at
+     * @param xPad x padding between cells
+     * @param yPad y padding between cells
+     */
+    public static void makeGrid(Container parent,
+                                int rows, int cols,
+                                int initialX, int initialY,
+                                int xPad, int yPad) {
+        SpringLayout layout;
+        try {
+            layout = (SpringLayout)parent.getLayout();
+        } catch (ClassCastException exc) {
+            System.err.println("The first argument to makeGrid must use SpringLayout.");
+            return;
+        }
+
+        Spring xPadSpring = Spring.constant(xPad);
+        Spring yPadSpring = Spring.constant(yPad);
+        Spring initialXSpring = Spring.constant(initialX);
+        Spring initialYSpring = Spring.constant(initialY);
+        int max = rows * cols;
+
+        //Calculate Springs that are the max of the width/height so that all
+        //cells have the same size.
+        Spring maxWidthSpring = layout.getConstraints(parent.getComponent(0)).
+                                    getWidth();
+        Spring maxHeightSpring = layout.getConstraints(parent.getComponent(0)).
+                                    getWidth();
+        for (int i = 1; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                            parent.getComponent(i));
+
+            maxWidthSpring = Spring.max(maxWidthSpring, cons.getWidth());
+            maxHeightSpring = Spring.max(maxHeightSpring, cons.getHeight());
+        }
+
+        //Apply the new width/height Spring. This forces all the
+        //components to have the same size.
+        for (int i = 0; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                            parent.getComponent(i));
+
+            cons.setWidth(maxWidthSpring);
+            cons.setHeight(maxHeightSpring);
+        }
+
+        //Then adjust the x/y constraints of all the cells so that they
+        //are aligned in a grid.
+        SpringLayout.Constraints lastCons = null;
+        SpringLayout.Constraints lastRowCons = null;
+        for (int i = 0; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                                 parent.getComponent(i));
+            if (i % cols == 0) { //start of new row
+                lastRowCons = lastCons;
+                cons.setX(initialXSpring);
+            } else { //x position depends on previous component
+                cons.setX(Spring.sum(lastCons.getConstraint(SpringLayout.EAST), xPadSpring));
+            }
+
+            if (i / cols == 0) { //first row
+                cons.setY(initialYSpring);
+            } else { //y position depends on previous row
+                cons.setY(Spring.sum(lastRowCons.getConstraint(SpringLayout.SOUTH),
+                                     yPadSpring));
+            }
+            lastCons = cons;
+        }
+
+        //Set the parent's size.
+        SpringLayout.Constraints pCons = layout.getConstraints(parent);
+        pCons.setConstraint(SpringLayout.SOUTH,
+                            Spring.sum(
+                                Spring.constant(yPad),
+                                lastCons.getConstraint(SpringLayout.SOUTH)));
+        pCons.setConstraint(SpringLayout.EAST,
+                            Spring.sum(
+                                Spring.constant(xPad),
+                                lastCons.getConstraint(SpringLayout.EAST)));
+    }
+
+    /* Used by makeCompactGrid. */
+    private static SpringLayout.Constraints getConstraintsForCell(
+                                                int row, int col,
+                                                Container parent,
+                                                int cols) {
+        SpringLayout layout = (SpringLayout) parent.getLayout();
+        Component c = parent.getComponent(row * cols + col);
+        return layout.getConstraints(c);
+    }
+
+    /**
+     * Aligns the first <code>rows</code> * <code>cols</code>
+     * components of <code>parent</code> in
+     * a grid. Each component in a column is as wide as the maximum
+     * preferred width of the components in that column;
+     * height is similarly determined for each row.
+     * The parent is made just big enough to fit them all.
+     *
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param initialX x location to start the grid at
+     * @param initialY y location to start the grid at
+     * @param xPad x padding between cells
+     * @param yPad y padding between cells
+     */
+    public static void makeCompactGrid(Container parent,
+                                       int rows, int cols,
+                                       int initialX, int initialY,
+                                       int xPad, int yPad) {
+        SpringLayout layout;
+        try {
+            layout = (SpringLayout)parent.getLayout();
+        } catch (ClassCastException exc) {
+            System.err.println("The first argument to makeCompactGrid must use SpringLayout.");
+            return;
+        }
+
+        //Align all cells in each column and make them the same width.
+        Spring x = Spring.constant(initialX);
+        for (int c = 0; c < cols; c++) {
+            Spring width = Spring.constant(0);
+            for (int r = 0; r < rows; r++) {
+                width = Spring.max(width,
+                                   getConstraintsForCell(r, c, parent, cols).
+                                       getWidth());
+            }
+            for (int r = 0; r < rows; r++) {
+                SpringLayout.Constraints constraints =
+                        getConstraintsForCell(r, c, parent, cols);
+                constraints.setX(x);
+                constraints.setWidth(width);
+            }
+            x = Spring.sum(x, Spring.sum(width, Spring.constant(xPad)));
+        }
+
+        //Align all cells in each row and make them the same height.
+        Spring y = Spring.constant(initialY);
+        for (int r = 0; r < rows; r++) {
+            Spring height = Spring.constant(0);
+            for (int c = 0; c < cols; c++) {
+                height = Spring.max(height,
+                                    getConstraintsForCell(r, c, parent, cols).
+                                        getHeight());
+            }
+            for (int c = 0; c < cols; c++) {
+                SpringLayout.Constraints constraints =
+                        getConstraintsForCell(r, c, parent, cols);
+                constraints.setY(y);
+                constraints.setHeight(height);
+            }
+            y = Spring.sum(y, Spring.sum(height, Spring.constant(yPad)));
+        }
+
+        //Set the parent's size.
+        SpringLayout.Constraints pCons = layout.getConstraints(parent);
+        pCons.setConstraint(SpringLayout.SOUTH, y);
+        pCons.setConstraint(SpringLayout.EAST, x);
+    }
+}
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/XMLUtil.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/XMLUtil.java	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/src/vishnu/util/XMLUtil.java	(revision 14150)
@@ -0,0 +1,71 @@
+/*********************************************************************
+ Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.util;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class XMLUtil {
+    
+    /** returns the (first) child element with the given name */
+    public static Node getChildByTagName(Node n, String name) {
+
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		return child;
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+
+    /** takes a list of elements, and returns an array of strings 
+     * of the values of attribute att_name */
+    public static String [] getAttributeValuesFromList(Element list,
+						       String att_name) {
+	
+	NodeList children = list.getChildNodes();
+	
+	int num_nodes = children.getLength();
+	String []ids = new String[num_nodes];
+	for (int i=0; i<num_nodes; i++) {
+	    Element e = (Element)children.item(i);
+	    String id = e.getAttribute(att_name);	    
+	    ids[i] = id;	    
+	}
+	
+	return ids;
+    }
+
+    /** extracts the text out of a node */
+    public static String getNodeText(Element param) {
+	param.normalize();
+	Node n = param.getFirstChild();
+	while (n!=null && n.getNodeType() !=Node.TEXT_NODE) {
+	    n=n.getNextSibling();
+	}
+	if (n == null) {
+	    return "";
+	}
+	return n.getNodeValue();
+    }
+
+}
Index: /other-projects/trunk/greenstone3-extension/vishnu/web/WEB-INF/web.xml
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/web/WEB-INF/web.xml	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/web/WEB-INF/web.xml	(revision 14150)
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<!-- this is based on a sample web.xml file provided by 
+http://jakarta.apache.org/tomcat/tomcat-4.0-doc/appdev/web.xml.txt
+-->
+<web-app>
+
+  
+  <!-- General description of your web application -->
+  
+  <display-name>VISHNU</display-name>
+  <description>
+    Collection server for Imperial College's new Vishnu Visualisation tool.
+  </description>
+  
+  <context-param>
+    <param-name>webmaster</param-name>
+    <param-value>enter your email address here</param-value>
+    <description>
+      The EMAIL address of the administrator to whom questions
+      and comments about this application should be addressed.
+    </description>
+  </context-param>
+  
+
+  <!-- Servlet definitions for the servlets that make up
+  your web application, including initialization
+  parameters.  
+  -->
+  <servlet>
+    <servlet-name>visualiser</servlet-name>
+    <description>imperial college vishnu servlet</description>
+    <servlet-class>vishnu.server.VisServlet</servlet-class>
+    <init-param>
+      <param-name>collectionsHome</param-name>
+      <!-- edit this to provide the full path to the collections directory-->
+      <!-- make sure the path separators are correct for your platform -->
+      <param-value>path-to-vishnu/collections</param-value>
+    </init-param>
+  </servlet>
+
+  <!-- Define mappings that are used by the servlet container to
+  translate a particular request URI (context-relative) to a
+  particular servlet.  The examples below correspond to the
+  servlet descriptions above.  -->
+  <servlet-mapping>
+    <servlet-name>visualiser</servlet-name>
+    <url-pattern>/server</url-pattern>
+  </servlet-mapping>
+  <!-- Define the default session timeout for your application,
+  in minutes.  From a servlet or JSP page, you can modify
+  the timeout for a particular session dynamically by using
+  HttpSession.getMaxInactiveInterval(). -->
+  
+  <session-config>
+    <session-timeout>30</session-timeout>    <!-- 30 minutes -->
+  </session-config>
+  
+  
+</web-app>
+
+
Index: /other-projects/trunk/greenstone3-extension/vishnu/web/index.html
===================================================================
--- /other-projects/trunk/greenstone3-extension/vishnu/web/index.html	(revision 14150)
+++ /other-projects/trunk/greenstone3-extension/vishnu/web/index.html	(revision 14150)
@@ -0,0 +1,64 @@
+<html>
+  <head>
+    <title>Imperial College Visualizer</title>
+  </head>
+  <body>
+<h1>VISHNU:</h1>
+
+<h2>Overview</h2>
+This is a document search engine with several new visualization front
+ends that aid navigation through the set of documents returned by a
+query. Keywords from the document set are identified and selected on
+the fly. These keywords are then used to compute document clusters. The
+clusters can be displayed and explored using three visualizations:
+Sammon cluster view, Dendro map or radial interactive visualization.<br>      
+
+<h2>Quick start</h2>
+<p>Choose a collection from the colection menu. Type a query into the search box and click the Find button. Each pane will then display the query
+results in a different manner.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight:bold;">PlainVis</span> ---&nbsp; This shows a ranked list of documents
+retrieved. Clicking on the link for a document (in the left most
+column) will bring up a new window containing the document.<br>
+</p>
+
+<p style="margin-left: 40px;"><span style="font-weight: bold;">RadialVis</span> --- This shows the
+key words from the retrieved document set around a circle with the
+documents represented by dots within the circle. The dots are arranged
+closest to the words that contain them. Clicking on a word will
+highlight the documents containing that word. Correspondingly, clicking
+on a dot will show the words that it contains. Words can be dragged
+which will move the dots, thus changing the influence of that particular
+word on the arrangement.<br>
+At the bottom part of the pane key words can be selected and de-selected.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight: bold;">SammonVis</span> --- This displays
+documents clustered by key words. As you mouse over a cluster a menu is
+displayed. This shows the number of documents in the cluster, and, at
+the bottom of the menu, the top ranked key words used to form the
+cluster. Clicking on the number of documents brings up the key words
+and their percentages in the left hand frame and a summary of the
+documents in the right hand frame. It is possible to drill down two
+levels through this menu. To return to a higher level use the button at
+the top of the frame.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight: bold;">DendroVis</span> ---&nbsp; This
+displays the document clusters arranged in a hierarchical format. As
+you mouse over nodes in the display a menu similar to the SammonVis
+menu is displayed. You can drill down through the hierarchy to examine
+deeper nodes. The key words are displayed at each node. The menu
+indicates if these are from the outer twigs (bottom) of the hierarchy.
+Clicking on the number of documents in the node displays the keywords
+and document summaries.<br>
+</p>
+
+      
+      <p>
+	<center>
+	<applet codebase='lib' code='vishnu.testvis.visual.Vishnu.class' archive='vishnu.jar' width='900' height='700'>
+	  <param name='library' value='http://localhost:8080/vishnu/server'/>
+	</applet>
+      </p>
+    </center>
+  </body>
+</html>
