Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/Build.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/Build.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/Build.java	(revision 12189)
@@ -0,0 +1,75 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class Build
+{
+  BuildManager manager;
+
+  public Build(List inputRoots, String site, String collection, String outputDir)
+  { this.manager = new BuildManager(inputRoots, site, collection, outputDir);
+  }
+
+  public void run()
+  { this.manager.run();
+  }
+
+  public static void main(String args[])
+  { int a = 0;
+    List inputDirs = new ArrayList();
+    String collection = null;
+    String site = null;
+    String outputDir  = null;
+
+    while (a < args.length)
+    { System.out.println(args[a]);
+      if (args[a].equals("-inputdir")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  inputDirs.add(args[a+1]);
+	  a ++;	
+	}
+      }
+      else if (args[a].equals("-collect")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  collection = args[a+1];
+	  a ++;
+	}
+      }
+      else if (args[a].equals("-site")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  site = args[a+1];
+	  a ++;
+	}
+      }	  
+      else if (args[a].equals("-outputDir")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{ outputDir = args[a+1];
+	  a ++;
+	}
+      }
+      a ++;
+    }
+
+    if (site == null) {
+	System.err.println("you must specify the site name");
+	System.exit(0);
+    }
+    if (collection == null && inputDirs.size() == 0) {
+	System.err.println("At least one input directory must be given, or a collection name");
+	System.exit(0);
+    }
+    
+    Build build = new Build(inputDirs, site, collection, outputDir);
+    build.run();
+
+    return;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/BuildManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/BuildManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/BuildManager.java	(revision 12189)
@@ -0,0 +1,154 @@
+package org.greenstone.gsdl3.gs3build;
+
+import org.greenstone.gsdl3.gs3build.doctypes.*;
+import org.greenstone.gsdl3.gs3build.indexers.*;
+import org.greenstone.gsdl3.gs3build.notifier.NotifierManager;
+import org.greenstone.gsdl3.gs3build.extractor.*;
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.collection.*;
+
+import java.io.File;
+import java.util.List;
+
+public class BuildManager 
+{
+    CollectionManager collectionManager;
+    RecogniserManager recogniserManager;
+    IndexerManager    indexerManager;
+    ClassifierManager classifierManager;
+    ExtractorManager  extractorManager;
+    NotifierManager   notifierManager;
+    DocumentList      docList;
+    List              inputRoots;
+    String            outputDir;
+    String            archiveDir;
+    
+    public BuildManager(List inputRoots, String site, String collection, String outputDir) 
+    {
+	// set up the collection manager
+	this.collectionManager = new CollectionManager(site, collection);
+	this.collectionManager.setBuildManager(this);
+	
+	// set up the list of input directories
+	if (inputRoots.size() == 0) {
+	    inputRoots.add(this.collectionManager.getImportDirectory());
+	    System.out.println("Adding collection import directory to input list " + this.collectionManager.getImportDirectory());
+	}
+	this.inputRoots = inputRoots;
+	
+	// set up the output directories (building and archive)
+	this.outputDir  = outputDir != null ? outputDir : this.collectionManager.getBuildDirectory();
+	File outputDirFile = new File(this.outputDir);
+	if (!outputDirFile.isDirectory()) {
+	    System.err.println("Unable to open directory " + this.outputDir + " for writing");
+	    System.exit(1);
+	}
+	System.out.println("Output directory set to " + this.outputDir);
+	
+	// set up the archive directory - this will definitely exist
+	this.archiveDir = this.collectionManager.getArchiveDirectory();
+
+	// the list that documents get added to when they are recognised
+	this.docList = new DocumentList(new DocumentIDFactory(this.collectionManager), 
+					this.collectionManager.getDatabase());
+	
+	// set up the various managers
+	this.recogniserManager = new RecogniserManager(this.docList);
+	this.extractorManager  = new ExtractorManager(this.docList);
+	this.classifierManager = new ClassifierManager(this.docList, collectionManager.getDatabase());
+	this.indexerManager = new IndexerManager(this.docList);
+	this.notifierManager = new NotifierManager();
+	
+	// configure the collection - this will add classifiers, indexers, recognisers to the various managers.
+	this.collectionManager.configureCollection();
+	
+	// add in expanders by hand cos they are not in the config file yet
+	// TODO
+	// add in extractors by hand cos they are not in the config file yet
+	this.extractorManager.addExtractor(new IndexExtractor());
+	this.extractorManager.addExtractor(new MetaXMLExtractor());
+	
+	// add in enrichers by hand cos they are not in the config file yet
+	// TODO
+		
+	// prepare a file crawler on the etc directory, and a crawl observer to respond to
+	// the appropriate files.
+	FileCrawler fileCrawler = new FileCrawler(new File(this.collectionManager.getEtcDirectory()), 
+						  new CollectionCrawlObserver(this));
+
+	fileCrawler.crawl();    
+    }
+  
+    public String getLastBuildSequence()
+    { 
+	return this.collectionManager.getCollectionMetadata("gsdl3", "buildsequence");
+    }
+
+    public ClassifierManager getClassifierManager()
+    { 
+	return this.classifierManager;
+    }
+
+    public IndexerManager getIndexerManager()
+    { 
+	return this.indexerManager;
+    }
+
+    // TODO: validation phase
+    public RecogniserManager getRecogniserManager()
+    {
+	return this.recogniserManager;
+    }
+    
+    public void addIndexer(IndexerInterface iface)
+    { 
+	iface.configure(IndexerManager.outputDir, this.outputDir);
+	this.indexerManager.addIndexer(iface);
+    }
+
+    public void run()
+    { 
+	// Initialise collection manager - note start of build
+	this.collectionManager.startBuild();
+	
+	// TODO: add expansion (e.g. Zip files)
+    
+	// Crawl the file tree - will recognise documents
+	for (int i = 0; i < this.inputRoots.size(); i ++) {
+	    FileCrawler fileCrawler = new FileCrawler(new File((String) this.inputRoots.get(i)), recogniserManager);
+
+	    fileCrawler.crawl();
+	}
+
+	// Extract phase, etc.
+	this.extractorManager.extractDocuments();
+	this.classifierManager.classifyDocuments();
+	this.indexerManager.indexDocuments();
+
+	// Timestamp management - update all timestamps on modified dates...
+	// 
+	// This should only occur at the end of building in case the build is cancelled...
+	docList.updateTimestamps(this.collectionManager.getBuildTimestamp());
+	
+	// TODO: validation phase
+	this.notifierManager.detectEvents(this.collectionManager);
+	
+	if (this.archiveDir != null) {
+	    this.docList.writeDocuments(new File(this.archiveDir));
+	    //this.docList.writeSQLDocuments(this.collectionManager.getDatabase());
+	}
+	
+	this.collectionManager.setCollectionMetadata("gsdl3", "documentCount", Integer.toString(this.docList.getCount()));
+    
+	// TODO: write out collection configuration file (service lists, etc.)
+    
+	this.collectionManager.endBuild();
+    }
+  
+    public void addInputDirectory(String inputDir)
+    {
+	this.inputRoots.add(inputDir);
+    }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 12189)
@@ -0,0 +1,594 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.util.Date;
+import java.util.Calendar;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.GregorianCalendar;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+
+import java.net.URL;
+import java.net.URLEncoder;
+
+import java.sql.*;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+import org.greenstone.gsdl3.gs3build.collection.*;
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.indexers.*;
+import org.greenstone.gsdl3.gs3build.doctypes.RecogniserInterface;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+/**
+ *  Store and hold collection-level configuration information for a collection.
+ *  This should be used by BuildManager to work out which classes, etc. to load
+ *  at build time, and as a repository for the collection-level metadata, and
+ *  a means of loading and saving the same to a file or database, as is seen
+ *  fit in the final development of gs3.
+ */
+
+public class CollectionManager
+{
+  GregorianCalendar  lastBuildDate;  // pretty obvious
+  String             adminEmail;     // the email address of the administrator of the
+		                     // collection
+  int                buildDocNo;     // used to generate document identifiers
+  CollectionMetadata metadata;       // collection-level metadata
+  GS3SQLConnection   database;       // the database to store everything in
+  String             collectionHome;
+  String             siteHome;
+  String             collectionName;
+  String             qualifiedCollectionName; // used as the database name 
+  String             notifyHost;
+
+  BuildManager       buildManager;
+   
+  class CollectionClassifier
+  { URL    file;
+    String type;
+    List fields;
+    String sort;
+    String className;
+
+    public CollectionClassifier(URL parentURL, String type, String className, Node node)
+    { this.type = type;
+      this.className = className;
+      this.fields = new ArrayList();
+
+      NodeList children = node.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++) {
+	Node child = children.item(c);
+
+	if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  String name = child.getNodeName();
+	  
+	  if (name.equals("file")) {
+	    NamedNodeMap atts = children.item(c).getAttributes();
+	    Node attribute = atts.getNamedItem("URL");
+	    String urlString = attribute.getNodeValue();
+	    if (urlString == null)
+	      continue;
+
+	    System.out.println("Path is" + parentURL.getPath());
+	    System.out.println("Host is" + parentURL.getHost());
+	    System.out.println(urlString);
+
+	    try {
+	      URL url = new URL(parentURL, urlString);
+	      this.file = url;
+	      System.out.println(url);
+	    }
+	    catch (java.net.MalformedURLException malEx) {
+	      System.out.println(malEx);
+	    }
+	  }
+	  else if (name.equals("field")) {
+	    String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	    this.fields.add(fieldName.toString());
+	  }
+	  else if (name.equals("sort")) {
+	    String sortName = DOMUtils.getNodeChildText(children.item(c));
+	    this.sort = sortName;
+	  }
+	}
+      }
+    }
+
+    public ClassifierInterface getClassifier()
+    { ClassifierInterface classifier = null;
+
+      if (this.type == null) {
+	return null;
+      }
+      System.out.println("Creating a classifier of type " + this.type.toLowerCase());
+      
+      if (this.type.toLowerCase().equals("hierarchy")) {
+	System.out.println("  hierarchy file is " + this.file);
+	classifier = new HierarchyClassifier(this.className, this.file, this.fields, this.sort);
+      }
+      else if (this.type.toLowerCase().equals("azlist")) {
+	classifier = new AZListClassifier(this.className, this.fields);
+      }
+
+      return classifier;
+    }
+  }
+   
+  /**
+   *  Create the collection manager for a given collection
+   *
+   * @param site the name of the site
+   *  @param collection <code>String</code> the name of the collection
+   */
+  public CollectionManager(String site, String collection) {
+    
+    String gsdl3Root = System.getProperty("GSDL3HOME");
+    if (gsdl3Root == null) {
+      System.out.println("Error: Unable to locate GSDL3HOME");
+      System.exit(1);
+      //return;
+    }
+    
+    this.siteHome = GSFile.siteHome(gsdl3Root, site);
+    File site_dir = new File(this.siteHome);
+    System.out.println(site_dir);
+    if (!site_dir.exists()) {
+      System.out.println("Error: Non-existant site ("+site+") specified");
+      System.exit(1);
+    }
+    site_dir = null;
+    this.collectionHome = GSFile.collectionBaseDir(this.siteHome, collection);
+    
+    File collect_dir = new File(this.collectionHome);
+    if (!collect_dir.exists()) {
+      System.out.println("Error: Non-existant collection ("+collection+") specified in site "+site);
+      System.exit(1);
+    }
+    collect_dir = null;
+
+    this.collectionName = collection;
+    this.qualifiedCollectionName = site+"_"+collection;
+
+    this.database = GS3SQLConnectionFactory.getGS3SQLConnection(this.qualifiedCollectionName);
+    /*    if (this.database != null) {
+      this.database.clearCollection(collection);
+      this.database = null;
+    }
+    */
+    if (this.database == null) {
+	this.database = GS3SQLConnectionFactory.getGS3SQLConnection("test");
+	if (this.database == null) {
+	    System.err.println("Can't connect to the mysql database. Please make sure your mysql is running, and that the correct passwords (if any) are specified in "+gsdl3Root+"/WEB-INF/classes/global.properties");
+	    System.exit(1);
+	}
+	boolean success = this.database.initCollection(this.qualifiedCollectionName);
+	if (!success) {
+	    System.err.println("couldn't init collection " + this.qualifiedCollectionName);
+	    System.exit(1);
+	}
+    }
+
+    this.metadata = new CollectionMetadata();
+
+    File buildDirectory = new File(getBuildDirectory());
+    if (!buildDirectory.exists()) {
+      buildDirectory.mkdir();
+    }
+    if (!buildDirectory.isDirectory()) {
+	System.err.println("Unable to open directory " + buildDirectory + " for writing");
+	System.exit(1);
+    }
+
+    File archiveDirectory = new File(getArchiveDirectory());
+    if (!archiveDirectory.exists()) {
+      archiveDirectory.mkdir();
+    }
+    if (!archiveDirectory.isDirectory()) {
+	System.err.println("Unable to open directory " + archiveDirectory + " for writing");
+	System.exit(1);
+    }
+
+    this.buildDocNo = 1;
+
+    try {
+      GS3SQLSelect select = new GS3SQLSelect("build");
+      select.addField("*");
+      Statement statement = this.database.createStatement();
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	System.out.println("Reading all keys");
+	do {
+	  String key = results.getString("buildKey");
+	  String value = results.getString("buildValue");
+
+	  if (key.equals("NextSeqNo")) {
+	    this.buildDocNo = Integer.parseInt(value);
+	  }
+	  else if (key.equals("lastBuildDate")) {
+	    int year, month, day;
+	    year = Integer.parseInt(value.substring(0, 4));
+	    month = Integer.parseInt(value.substring(4, 6)) - 1; // -1 because Gregorian Calendar perversely treats January as 0, etc.
+	    day = Integer.parseInt(value.substring(6, 8));
+	    
+	    this.lastBuildDate = new GregorianCalendar(year, month, day);
+	  }
+	} while (results.next());
+      }
+      statement.close();
+    }
+    catch (SQLException ex)
+      { System.out.println(ex);
+      }
+  }
+
+  public void setBuildManager(BuildManager build_man) 
+  { this.buildManager = build_man;
+  }
+
+  private void configureBrowsers(Node node, File etcFile)
+  { CollectionClassifier classifier = null;
+    URL etcURL = null;
+
+    etcURL = URLTools.getFileURL(etcFile);
+
+    NodeList children = node.getChildNodes();
+    for (int c = 0; c < children.getLength(); c ++)
+    { // assume that non-element children are irrelevant
+      if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE)
+      { continue;
+      }
+
+      String name = children.item(c).getNodeName();
+      System.out.println(name);
+
+      if (name.equals(GSXML.CLASSIFIER_ELEM))
+      { NamedNodeMap atts = children.item(c).getAttributes();
+
+        // get the type attribute
+	Node attribute = atts.getNamedItem(GSXML.TYPE_ATT);
+	if (attribute == null) {
+	  continue;
+	}
+	String type = attribute.getNodeValue();
+
+        // get the type attribute
+	attribute = atts.getNamedItem(GSXML.NAME_ATT);
+	if (attribute == null) {
+	  continue;
+	}
+	String className = attribute.getNodeValue();
+
+	classifier = new CollectionClassifier(etcURL, type, className, children.item(c));
+	
+	System.out.println("Found classifier " + type);
+	
+	// attach the classifier
+	ClassifierInterface classify = classifier.getClassifier();
+	this.buildManager.getClassifierManager().addClassifier(classify);
+      }  
+    }
+  }
+
+  public void configureCollection()
+  { File collectionConfig = new File(GSFile.collectionConfigFile(this.collectionHome));
+
+    // get the File and read it in
+    try
+    {
+      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder = factory.newDocumentBuilder();
+      Document document = builder.parse(collectionConfig);
+
+      // TODO: report an error
+      if (document == null)
+      {	
+      }
+
+      // now parse the manager file...
+      Element rootElement = document.getDocumentElement();
+
+      if (rootElement.getTagName() != GSXML.COLLECTION_CONFIG_ELEM)
+      {	// TODO: throw exception
+      }
+
+      System.out.println("Configuring collection");
+
+      NodeList children = rootElement.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++)
+      {	// assume that non-element children are irrelevant
+	if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE)
+	{ continue;
+	}
+
+	String name = children.item(c).getNodeName();
+
+	// the name is a plugin element
+
+	if (name.equals(GSXML.RECOGNISE_ELEM)) {
+	    System.out.println("recognise elem");
+	    
+	    NodeList doc_types = ((Element)children.item(c)).getElementsByTagName(GSXML.DOC_TYPE_ELEM);
+	    for(int i=0; i<doc_types.getLength(); i++) {
+		Element doc_type = (Element)doc_types.item(i);
+		String type = doc_type.getAttribute(GSXML.NAME_ATT);
+		System.err.println("adding recogniser, type "+type);
+		RecogniserInterface ri = this.buildManager.getRecogniserManager().addRecogniser(type);
+		if (ri != null) {
+		    ri.configure(doc_type);
+		}
+	    }
+	} else if (name.equals(GSXML.SEARCH_ELEM)) {
+	  // pick up attributes from the <search> tag now...
+	  NamedNodeMap searchAttributes = children.item(c).getAttributes();
+	  Node searchAttribute = searchAttributes.getNamedItem(GSXML.TYPE_ATT);
+	  String searchType = null;
+	  if (searchAttribute != null) {
+	      searchType = searchAttribute.getNodeValue();
+	  } else {
+	      System.out.println("no "+GSXML.TYPE_ATT+" attribute found for the "+GSXML.SEARCH_ELEM+" element, assuming mg");
+	      searchType = MGIndexer.MG_INDEX_TYPE;
+	  }
+	  
+	  searchAttribute = searchAttributes.getNamedItem(GSXML.NAME_ATT);
+	  String searchName = null;
+	  if (searchAttribute != null) {
+	      searchName = searchAttribute.getNodeValue();
+	  }
+	  if (searchName == null) {
+	      searchName = "idx"; // need to modify this if we have two search elements with no names
+	  }
+	  // create the pertinent indexer...
+	  IndexerInterface indexer = IndexerFactory.makeIndexer(searchType, searchName);
+
+	  if (indexer == null) {
+	    continue;
+	  }
+
+	  // configure the indexer
+	  indexer.configure(children.item(c));
+
+	  // install it into the build manager
+	  this.buildManager.addIndexer(indexer);
+	}
+	else if (name.equals(GSXML.BROWSE_ELEM))
+	{ this.configureBrowsers(children.item(c), collectionConfig);
+	}
+	else if (name.equals(GSXML.NOTIFY_ELEM))
+	{
+	    this.notifyHost = ((Element) children.item(c)).getAttribute(GSXML.NOTIFY_HOST_ATT);
+	}
+	// TODO: other elements - make a factory-method approach here...
+	else
+	{
+	}
+      }
+    }
+    catch (FactoryConfigurationError e) {
+      System.out.println(e);
+    }
+    catch (ParserConfigurationException ex) {
+      System.out.println(ex);
+    }
+    catch (SAXException ex) {
+      System.out.println(ex);
+    }
+    catch (IOException ex)
+    {
+      System.out.println(ex);
+    }
+
+    System.out.println("<<<Obtaining database>>>>");
+  }
+
+  public String getEtcDirectory()
+  { return GSFile.collectionEtcDir(this.collectionHome);
+  }
+
+  public String getImportDirectory()
+  { return GSFile.collectionImportDir(this.collectionHome);
+  }
+
+  public String getBuildDirectory()
+  { return GSFile.collectionBuildDir(this.collectionHome);
+  }
+
+  public String getArchiveDirectory()
+  { return GSFile.collectionArchiveDir(this.collectionHome);
+  }
+
+  public GS3SQLConnection getDatabase()
+  {
+    return this.database;
+  }
+
+  public long getBuildTimestamp()
+  { return this.lastBuildDate.getTime().getTime();
+  }
+
+  public Date getBuildDate()
+  { return this.lastBuildDate.getTime();
+  }
+
+  public void startBuild()
+  { GregorianCalendar today = new GregorianCalendar();
+  
+    if (this.lastBuildDate != null)
+    { // if the build date is different to the last build date, then reset the build
+      // document number
+      if (today.get(Calendar.YEAR) != this.lastBuildDate.get(Calendar.YEAR) ||
+	  today.get(Calendar.MONTH) != this.lastBuildDate.get(Calendar.MONTH) ||
+	  today.get(Calendar.DAY_OF_MONTH) != this.lastBuildDate.get(Calendar.DAY_OF_MONTH))
+      { this.buildDocNo = 1;
+      }
+      else 
+      { System.out.println("Continuing build sequence from " + this.buildDocNo);
+      }
+    }
+    this.lastBuildDate = today;
+  }
+
+  public void endBuild()
+  {
+    // here we write out the build config file
+    // create the xml for the buildConfig
+    XMLConverter converter = new XMLConverter();
+    Document doc = converter.newDOM();
+    Element build_config = doc.createElement(GSXML.COLLECTION_BUILD_ELEM);
+    Element meta_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    build_config.appendChild(meta_list);
+    Element service_list = doc.createElement(GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+    build_config.appendChild(service_list);
+    
+    // ask the indexers and classifiers to add stuff into the service rack list
+    this.buildManager.getIndexerManager().addServiceDescriptions(service_list);
+    this.buildManager.getClassifierManager().addServiceDescriptions(service_list);
+    // get the String
+    String build_config_string = converter.getString(build_config);
+    // write it to the file
+    try {
+      File build_config_file = new File(GSFile.collectionBuildConfigFileBuilding(this.collectionHome));
+      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(build_config_file), "UTF-8"));
+      writer.write(build_config_string, 0, build_config_string.length());
+      writer.close();
+    } catch (Exception e) {
+      System.err.println("CollectionManager.endBuild() Error while trying to output the buildConfig.xml file.");
+      System.err.println(e.getMessage());
+    }
+    
+    // Update build date information
+    try {
+      GS3SQLDelete remove = new GS3SQLDelete("build");
+      //    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("buildKey", "=", "NextSeqNo"));
+      //    rem
+      Statement statement = this.database.createStatement();
+      statement.execute(remove.toString());
+      
+      GS3SQLInsert insert = new GS3SQLInsert("build");
+      insert.addValue("buildKey", "NextSeqNo");
+      insert.addValue("buildValue", Integer.toString(this.buildDocNo));
+      statement.execute(insert.toString());
+    
+      insert = new GS3SQLInsert("build");
+      insert.addValue("buildKey", "lastBuildDate");
+      insert.addValue("buildValue", getDateString(this.lastBuildDate));
+      statement.execute(insert.toString());
+      statement.close();
+    } catch (SQLException e) {
+	System.err.println("CollectionManager.endBuild(): Can't update build information: "+e);
+    }
+    
+    // Do tail of build output
+    Date startDate = this.lastBuildDate.getTime();
+    Date date = new Date();
+
+    long startTime = startDate.getTime();
+    long endTime   = date.getTime();
+
+    long difference = ((endTime - startTime) + 500) / 1000;
+
+    System.out.println("Build completed");
+    System.out.println("---------------");
+    System.out.println("Total Documents: " + this.getCollectionMetadata("gsdl3", "documentCount"));
+    System.out.println("Total Time :     " + (difference / 60) + " min. " + (difference % 60) + " secs.");
+  }
+
+  private static String getDateString(GregorianCalendar date)
+  { StringBuffer dateString = new StringBuffer();
+  
+    int value;
+    dateString.append(date.get(Calendar.YEAR));
+
+    // the use of month is a little odd, hence the following
+    // code.  Calendar.MONTH yields 0 = January, 1 = February,
+    // etc. hence there is a '+1' added to the month to make 
+    // it into January = 1, etc., and the padding is altered
+    // correspondingly.
+    value = date.get(Calendar.MONTH);
+    if (value < 9)
+    { dateString.append("0");
+    }
+    dateString.append(value + 1);
+    value = date.get(Calendar.DAY_OF_MONTH);
+    if (value < 10)
+      dateString.append("0");
+    dateString.append(value);
+
+    return dateString.toString();
+  }
+
+
+  public String getNextDocumentID()
+  { StringBuffer ID = new StringBuffer(getDateString(this.lastBuildDate));
+
+    int value = this.buildDocNo;
+    this.buildDocNo ++;
+    
+    ID.append(":");
+    ID.append(Integer.toString(value));
+    return ID.toString();
+  }
+  
+  public int getDocumentNumber()
+  { this.buildDocNo ++;
+    return this.buildDocNo - 1;
+  }
+
+  /**
+   *  Get the collection metadata item in the given namespace
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   */
+  public String getCollectionMetadata(String namespace, String label)
+  { return this.metadata.getCollectionMetadata(namespace, label).get(0).toString();
+  }
+
+  /**
+   *  Set the collection metadata item in the given namespace
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label
+   *  @param <code>String</code> the value
+   */
+  public void setCollectionMetadata(String namespace, String label, String value)
+  { this.metadata.setCollectionMetadata(namespace, label, value);
+  }
+  
+  public String getCollectionName() {
+      return collectionName;
+  }
+
+/**
+ * @return
+ */
+public String getNotifyHost() {
+    return notifyHost;
+}
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/Create.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/Create.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/Create.java	(revision 12189)
@@ -0,0 +1,116 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+import org.greenstone.gsdl3.util.GSFile;
+
+public class Create
+{
+    String gsdl3_home = null;
+    String gsdl3_srchome = null;
+    String site = null;
+    String site_home = null;
+    String collection = null;
+
+    public Create(String gsdl3_home, String gsdl3_srchome, String site, 
+		  String collection)
+    { 
+	this.gsdl3_home = gsdl3_home;
+	this.gsdl3_srchome = gsdl3_srchome;
+	this.site = site;
+	this.site_home = GSFile.siteHome(gsdl3_home, site);
+	this.collection = collection;
+
+	// check the site's collect directory
+	File site_collect = new File(GSFile.collectDir(this.site_home));
+	if (!site_collect.isDirectory()) {
+	    System.err.println("Site's collect directory "+site_collect.getPath()+" doesn't exist. Please create it and try again");
+	    System.exit(1);
+	}
+
+       	
+    }
+    
+    public void makeCollection() {
+	
+	String coll_home = GSFile.collectionBaseDir(this.site_home, this.collection);
+	File coll_dir_file = new File(coll_home);
+	if (coll_dir_file.exists()) {
+	    System.err.println("Collection directory ("+coll_dir_file.getPath()+") already exists, not creating it.");
+	    System.exit(1);
+	}
+	createDirectory(coll_dir_file);
+	createDirectory(new File(GSFile.collectionImportDir(coll_home)));
+	createDirectory(new File(GSFile.collectionEtcDir(coll_home)));
+
+	File config_template = new File(gsdl3_srchome + File.separator+ "resources"+File.separator + "xml"+File.separator+"collectionConfig.xml");
+	if (!config_template.isFile()) {
+	    System.err.println("Couldn't find the config file template "+config_template.getPath());
+	    System.exit(1);
+	}
+
+	File coll_config = new File(GSFile.collectionConfigFile(coll_home));
+	GSFile.copyFile(config_template, coll_config);
+
+	System.err.println("New collection created in "+coll_home);
+	System.err.println("Put source documents into import, set the collection's configuration in etc/collectionConfig.xml, then run \"gs3-build.(sh|bat) "+ this.site+" "+this.collection+"\" to build the collection");
+    }
+
+    protected void createDirectory(File dir) {
+	dir.mkdir();
+	if (!dir.isDirectory()) {
+	    System.err.println("Couldn't create directory "+dir.getPath());
+	    System.exit(1);
+	}
+    }
+
+    public static void main(String args[])
+    { 
+	String gsdl3_home = System.getProperty("GSDL3HOME");
+	String gsdl3_srchome = System.getProperty("GSDL3SRCHOME");
+	if (gsdl3_home == null) {
+	    System.out.println("Error: Unable to locate GSDL3HOME");
+	    System.exit(1);
+	}
+	if (gsdl3_srchome == null) {
+	    System.out.println("Error: Unable to locate GSDL3SRCHOME");
+	    System.exit(1);
+	}
+
+	int a = 0;
+	String collection = null;
+	String site = null;
+	
+	while (a < args.length) {
+	    if (args[a].equals("-collect")) {
+		if (a < args.length - 1 &&
+		    args[a+1].charAt(0) != '-') {
+		    
+			collection = args[a+1];
+			a ++;
+		}
+	    } else if (args[a].equals("-site")) {
+		if (a < args.length - 1 &&
+		    args[a+1].charAt(0) != '-') {
+		    
+		    site = args[a+1];
+		    a ++;
+		}
+	    }
+	    a ++;
+	}
+	
+	if (site == null) {
+	    
+	    System.err.println("you must specify the site name");
+	    System.exit(0);
+	}
+	if (collection == null) {
+	    System.err.println("you must specify a collection");
+	    System.exit(0);
+	}
+    
+	Create c = new Create(gsdl3_home, gsdl3_srchome, site, collection);
+	c.makeCollection();
+	return;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java	(revision 12189)
@@ -0,0 +1,8 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+public interface FileCrawlObserver
+{
+	public void processFile(File file);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawler.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawler.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/FileCrawler.java	(revision 12189)
@@ -0,0 +1,39 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+public class FileCrawler
+{
+  File root;
+  FileCrawlObserver observer;
+
+  public FileCrawler(File file, FileCrawlObserver observer)
+  { this.root = file;
+    this.observer = observer;
+  }
+
+  public void crawl()
+  { this.crawlDirectory(this.root);
+  }
+
+  public boolean crawlDirectory(File file)
+  {
+    File [] children = file.listFiles();
+
+    if (children == null) {
+      return true;
+    }
+
+    for (int c = 0; c < children.length; c ++) {
+      if (children[c].isDirectory()) {
+	crawlDirectory(children[c]);
+      }
+      else
+      {
+	// TODO: process file
+	this.observer.processFile(children[c]);
+      }
+    }
+    return true;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/GSDataDump.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/GSDataDump.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/GSDataDump.java	(revision 12189)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.gs3build.doctypes.*;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnectionFactory;
+
+public class GSDataDump
+{ 
+  public static void main(String args[])
+  {
+    String database = args[0];
+
+    GS3SQLConnection connection = GS3SQLConnectionFactory.getGS3SQLConnection(database);
+    DocumentList docList = DocumentList.readSQLDocuments(connection);
+    docList.writeDocuments(new File(database));
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java	(revision 12189)
@@ -0,0 +1,356 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.util.GSXML;
+
+public class AZListClassifier extends AbstractClassifier
+{
+  GS3SQLConnection database;
+  MultiMap         alphaMap;
+  List             fields;
+  String           name;
+
+  class AZDocumentItem
+  { public DocumentID documentId;
+    public String     title;
+
+    public AZDocumentItem(DocumentID documentId, String title)
+    { this.documentId = documentId;
+      this.title = title;
+    }
+
+    public String getTitle()
+    { return this.title;
+    }
+
+    public DocumentID getID()
+    { return this.documentId;
+    }
+
+    public String toString()
+    { return this.title;
+    }
+  }
+
+  private void sortList(String key)
+  { List list = this.alphaMap.getAll(key);
+    if (list.size() > 1) {
+      int first, last;
+
+      first = 0;
+      last = list.size() - 1;
+      while (first != last)
+      { int at = (first + last) / 2;
+
+        if (list.get(at).toString().compareTo(list.get(list.size() - 1).toString()) > 0) {
+	  last = at;
+        }
+	else {
+	  first = at + 1;
+	}
+      }
+
+      Object newObject = list.get(list.size()-1);
+      last = list.size() - 1;
+      while (last > first) {
+	list.set(last, list.get(last-1));
+	last --;
+      }
+      list.set(first, newObject);
+    }
+  }
+
+  public AZListClassifier(String name, List fields)
+  { this.fields   = fields;
+    this.alphaMap = new MultiMap();
+    this.name     = name;
+    //  System.out.println("AZList :"+fields.size());
+  }
+
+  public AZListClassifier()
+  { this.fields = new ArrayList();
+  }
+
+  public void configure(List parameters)
+  {
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+    }
+  }
+
+  public void setDatabase(GS3SQLConnection connection)
+  { this.database = connection;
+  }
+
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+			
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      // ...and send them to the classifier
+      Iterator thisValue = values.iterator();
+      while (thisValue.hasNext()) 
+      { String value = thisValue.next().toString();
+
+	String classText = null;
+
+	int c = 0;
+	while (c < value.length() && classText == null)
+	{ if (Character.isDigit(value.charAt(c))) 
+	  { classText = "0-9";
+	  }
+	  else if (Character.isLetter(value.charAt(c))) 
+	  { classText = value.substring(c, c+1).toLowerCase();
+	  }
+  	  c ++;
+	}
+
+	if (classText != null) {
+	    //	  System.out.println(classText + " " + documentID);
+	  this.alphaMap.put(classText, new AZDocumentItem(documentID, value));
+	  this.sortList(classText);
+	}
+      }
+    }
+
+    return true;    
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void endClassifierPass(int pass)
+  {
+  }
+
+  public void completeClassification()
+  { this.writeSQL(this.database);
+  }
+
+  public int writeSQLClassifyNode(GS3SQLConnection connection, String parentId, int orderRef,
+				  String label, String name, String description, int noOfLeafDocs)
+  { GS3SQLAction action;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+  
+    int classifyRef;
+
+    // check for existing node
+    select = new GS3SQLSelect("classifiers");
+    select.addField("ClassifyRef");
+    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ClassifyID", "=", label);
+    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+    select.setWhere(where);
+
+
+    try {
+      Statement statement = connection.createStatement();
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	GS3SQLUpdate update = new GS3SQLUpdate("classifiers");
+	update.setWhere(where);
+	action = update;
+ 
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      else {
+	insert = new GS3SQLInsert("classifiers");
+
+	insert.addValue("ParentID", parentId);
+
+	action = insert;
+	classifyRef = -1;
+      }
+      action.addValue("ClassifyID", label);
+      action.addValue("Name", name);
+      action.addValue("Description", description);
+      action.addValue("ClassifyOrder", Integer.toString(orderRef), GS3SQLField.INTEGER_TYPE);
+      action.addValue("NumLeafDocs", Integer.toString(noOfLeafDocs), GS3SQLField.INTEGER_TYPE);
+
+      // do the update/insert
+      statement.execute(action.toString());
+
+
+      // get the ClassifyRef if we don't already have it (have done a
+      // insert action above)...
+      if (classifyRef == -1) {
+	results = statement.executeQuery(select.toString());
+	if (!results.first()) {
+	  return -1;
+	}
+	
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      
+      statement.close();
+    }  catch (SQLException sqlEx) {
+	System.err.println("AZListClassifier.writeSQLClassifyNode(): "+sqlEx);
+	return -1;
+    }
+  
+
+    return classifyRef;
+  }
+
+  public boolean writeSQL(GS3SQLConnection connection)
+  { GS3SQLAction action;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+ 
+    Iterator keys;
+ 
+    String prefix = this.name;
+    if (prefix==null || prefix.equals("")) {
+	prefix = "CLAZ"+this.fields.get(0).toString();
+	// TODO: write this name back to collectionConfig.xml
+	this.name = prefix;
+    }
+
+    int leafCount = 0;
+    keys = this.alphaMap.keySet().iterator();
+    while (keys.hasNext()) {
+      Object key = keys.next();
+
+      List childDocs = (List) this.alphaMap.getAll(key);
+      if (childDocs != null) {
+	leafCount += childDocs.size();
+      }
+    }
+
+    int parentClassify = this.writeSQLClassifyNode(connection, "", 0, prefix, "", "Classifier", leafCount);
+
+    if (parentClassify < 0) 
+    { return false;
+    }
+
+    try {
+	Statement statement = connection.createStatement();
+    
+    List children;
+    
+    // TODO: cope with change rather than create from scratch...
+    int nodeOrder = 1;
+    keys = this.alphaMap.keySet().iterator();
+    while (keys.hasNext()) {
+      Object key = keys.next();
+
+      List childDocs = (List) this.alphaMap.getAll(key);
+
+      if (childDocs != null) {
+	String className = prefix + "." + key.toString();
+
+	int classifyRef = this.writeSQLClassifyNode(connection, prefix, nodeOrder, className, key.toString(), key.toString(), childDocs.size());
+
+	// note the child documents...
+        Iterator iterator = childDocs.iterator();
+	int childOrder = 1;
+	//St
+	while (iterator.hasNext()) {
+	  AZDocumentItem documentItem = (AZDocumentItem) iterator.next();
+	  DocumentID docId = documentItem.getID();
+
+	  insert = new GS3SQLInsert("classdocuments");
+	  insert.addValue("ClassifyRef", Integer.toString(classifyRef), GS3SQLField.INTEGER_TYPE);
+	  insert.addValue("DocID", docId.toString());
+	  insert.addValue("DocOrder", Integer.toString(childOrder), GS3SQLField.INTEGER_TYPE);	  
+
+	  statement.execute(insert.toString());
+
+	  childOrder ++;
+	}
+
+	nodeOrder ++;
+      }
+    }
+
+   
+    /*
+    else {
+      // TODO: clear 'dead' child classifications
+
+      // delete child documents
+      GS3SQLDelete delete = new GS3SQLDelete("classdocuments");
+      delete.setWhere(where);
+
+      statement.execute(delete.toString());
+    }
+
+    // post the child nodes...
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      if (!childNode.writeSQL(connection)) {
+	return false;
+      }
+    }
+    */
+    statement.close();
+    } catch (SQLException e) {
+	System.err.println("AZListClassifier.writeSQL(): "+e);
+	return false;
+    }
+
+    return true;
+  }
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+
+}
+
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java	(revision 12189)
@@ -0,0 +1,61 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+
+public abstract class AbstractClassifier implements ClassifierInterface
+{ public AbstractClassifier()
+  {
+  }
+
+  public void configure(URL parentURL, org.w3c.dom.Node node)
+  { NodeList children = node.getChildNodes();
+    for (int c = 0; c < children.getLength(); c ++) {
+      Node child = children.item(c);
+      
+      if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	String name = child.getNodeName();
+	
+	if (name.equals("file")) {
+	  NamedNodeMap atts = children.item(c).getAttributes();
+	  Node attribute = atts.getNamedItem("URL");
+	  String urlString = attribute.getNodeValue();
+	  if (urlString == null)
+	    continue;
+	  
+	  try {
+	    URL url = new URL(parentURL, urlString);
+	    //	    this.file = url;
+	    System.out.println(url);
+	  }
+	  catch (java.net.MalformedURLException malEx) {
+	    System.out.println(malEx);
+	  }
+	}
+	else if (name.equals("field")) {
+	  String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	  System.out.println("Adding field " + fieldName);
+	  //	  this.fields.add(fieldName.toString());
+	}
+	else if (name.equals("sort")) {
+	  String sortName = DOMUtils.getNodeChildText(children.item(c));
+	  //	  this.sort = sortName;
+	}
+      }
+    }
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java	(revision 12189)
@@ -0,0 +1,428 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public abstract class AbstractHierarchyNode
+{
+  String prefix;        // a basic prefix used in this hierarchy
+  String descriptor;    // the textual descriptor used on screen or long-hand
+  String name;          // the index number, letter assignment or other item
+                        //   used to identify the position of the item in the
+                        //   hierarchy
+  String id;            // an identifier used by the GLI for maintenance
+                        //   purposes; this plays no active role in the
+                        //   rebuilding process (at the moment)
+  List childNodes;      // the child classification nodes of this node
+  List childDocs;       // the child documents of this node
+  List matches;         // the other metadata values that may be matched
+                        //   against the classifier
+  AbstractHierarchyNode parent; // the parent of the node
+
+  class AbstractHierarchyDocument
+  { DocumentID documentId;
+    String     sortKey;
+
+    public AbstractHierarchyDocument(DocumentID documentId, String sortKey)
+    { this.documentId = documentId;
+      this.sortKey    = sortKey;
+    }
+
+    public DocumentID getID()
+    { return this.documentId;
+    }
+
+    public String toString()
+    { return this.sortKey;
+    }
+  } 
+
+  /**
+   *  Simple node
+   */
+  public AbstractHierarchyNode()
+  { this.descriptor = "";
+    this.name       = "";
+    this.id         = "";
+    this.prefix     = "";
+    this.childNodes = new ArrayList();
+    this.childDocs  = new ArrayList();
+    this.parent     = null;
+    this.matches    = new ArrayList();
+  }
+    
+  public AbstractHierarchyNode(String prefix, String name, String id, String descriptor)
+  { this.descriptor = descriptor;
+    this.name       = name;
+    this.id         = id;
+    this.prefix     = prefix;
+    this.childNodes = new ArrayList();
+    this.childDocs  = new ArrayList();
+    this.parent     = null;
+    this.matches    = new ArrayList();
+    this.matches.add(this.id);
+    this.matches.add(this.name);
+  }
+    
+  public void addChild(AbstractHierarchyNode child)
+  { this.childNodes.add(child);
+    child.setParent(this);
+
+    /**
+    if (this.id == null) {
+      System.out.println(child.id.toString() + " added to root");
+    }
+    else {
+      System.out.println(child.id.toString() + " added to " + this.id);
+    }
+    */
+  }
+
+  public boolean add(AbstractHierarchyNode child)
+  { if (this.id != null && this.id.length() > 0 &&
+        !child.id.startsWith(this.id + "."))
+    { return false;
+    }
+
+    Iterator subNodes = this.childNodes.iterator();
+    while (subNodes.hasNext()) {
+      AbstractHierarchyNode subNode = (AbstractHierarchyNode) subNodes.next();
+      
+      if (subNode.add(child)) {
+	return true;
+      }
+    }
+    
+    this.addChild(child);
+    return true;
+  }
+
+  public void addDocument(DocumentID document, String sortKey)
+  { if (sortKey == null && false) {
+    System.err.println("attempt to classify by an unset key");
+      return;
+    }
+    this.childDocs.add(new AbstractHierarchyDocument(document, sortKey));
+
+    System.out.println("Adding " + document.toString() + " to " + this.name + " " + this.childDocs.size());
+    if (sortKey != null) {
+      int first = 0;
+      int last  = this.childDocs.size() - 1;
+
+      while (first != last) {
+	int at = (first + last) / 2;
+	
+	System.out.println(this.name + " " + at + " " + this.childDocs.size() + " " + this.childDocs.get(at) + " " + sortKey);
+	if (this.childDocs.get(at) == null ||
+	    this.childDocs.get(at).toString() == null) {
+	  last = at;
+	  System.out.println("COntinuing");
+	  continue;
+	}
+
+	if (this.childDocs.get(at).toString().compareTo(sortKey) > 0) {
+	  last = at;
+	}
+	else {
+	  first = at + 1;
+	}
+      }
+
+      Object newItem = this.childDocs.get(this.childDocs.size() - 1);
+      last = this.childDocs.size() - 1;
+      while (last > first) {
+	this.childDocs.set(last, this.childDocs.get(last-1));
+	last --;
+      }
+      this.childDocs.set(first, newItem);
+    }
+
+    //  System.out.println("Added " + document.toString() + " to " + this.name + " " + this.childDocs.size());
+    //    for (int a = 0; a < this.childDocs.size(); a ++) {
+    //System.out.println(this.childDocs.get(a));
+    //}
+  }
+
+  public void setParent(AbstractHierarchyNode parent)
+  { this.parent = parent;
+  }
+    
+  public AbstractHierarchyNode getParent()
+  { return this.parent;
+  }
+
+  public String getParentId()
+  { 
+    if (this.id == null) {
+      return "";
+    }
+
+    int dotAt = this.id.lastIndexOf('.');
+    if (dotAt < 0) {
+      return "";
+    }
+    return this.id.substring(0, dotAt);
+  }  
+
+  public void setDescriptor(String descriptor)
+  { this.descriptor = descriptor;
+  }
+
+  public void setID(String id)
+  { this.id = id;
+    this.matches.add(id);
+  }
+
+  public String getID()
+  { return this.id;
+  }
+
+  public int noOfChildDocs()
+  { return this.childDocs.size();
+  }
+
+  public int noOfLeafDocs()
+  { int total = this.childDocs.size();
+
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      total += childNode.noOfLeafDocs();
+    }
+
+    return total;
+  }
+
+  /**
+   *  Set the name for this hierarchy node - i.e. its brief description
+   *
+   *  @param <code>String</code> the new node name.
+   */
+  public void setName(String name)
+  { this.name = name;
+    this.matches.add(name);
+  }
+
+  /**
+   *  Get the name for this hierarchy node - i.e. its brief description
+   *
+   *  @return <code>String</code> the node's name.
+   */
+  public String getName()
+  { return this.name;
+  }
+
+  /**
+   *  Add another string which can be used to describe this hierarchy node - i.e. one which if
+   *  found in a document would indicate that that document belongs to this hierarchy node.
+   *
+   *  @param <code>String</code> the string to match.
+   */
+  public void addMatch(String match)
+  { this.matches.add(match);
+  }
+
+  /**
+   *  Check if a given string matches any of the values given for this hierarchy node...
+   *
+   *  @param <code>String</code> the descriptive string to be compared against the hierarchy
+   *  @return <code>boolean</code> <code>true</code> whether the string matches this hierarchy
+   */
+  public boolean isMatch (String toMatch)
+  { Iterator thisMatch = this.matches.iterator();
+			
+    while (thisMatch.hasNext())
+    { String thisMatchText = thisMatch.next().toString();
+      
+      if (thisMatchText.equals(toMatch))
+      { return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   *  Take a document, and find the classifications that it matches against in
+   *  the current hierarchy.  
+   *
+   *  @param <code>DocumentID</code> the id of the document being classified
+   *  @param <code>List</code> the values against which the classifier should
+   *         test for the document being a match - i.e. the pertinent document
+   *         property values.
+   *  @param <code>ClassifierObserverInterface</code>
+   *         object modifies the document with information about the
+   *         classifications that it fell within.
+   */
+  abstract public void getClassifications(DocumentID documentID, List values, String sortKey,
+					  ClassifierObserverInterface observer);
+
+  public boolean writeSQL(GS3SQLConnection connection)
+  {
+    int classifyRef;
+    GS3SQLAction action = null;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+
+    Statement statement;
+    // can we connect to the database?
+    try {
+	statement = connection.createStatement();
+    } catch (SQLException e) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	return false;
+    }
+    
+    // Get own full id
+    String fullId = this.id.length() > 0 ? this.prefix+"."+this.id : this.prefix;
+
+    // check for an existing instance of this classifier
+    select = new GS3SQLSelect("classifiers");
+    select.addField("ClassifyRef");
+    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ClassifyID", "=", fullId);
+    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+    select.setWhere(where);
+
+
+    // update or insert the classifier as required
+    try {
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	GS3SQLUpdate update = new GS3SQLUpdate("classifiers");
+	update.setWhere(where);
+	action = update;
+ 
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      else {
+	insert = new GS3SQLInsert("classifiers");
+
+	if (this.id == null || this.id.length() == 0) {
+	  insert.addValue("ParentID", "");
+	}
+	else {
+	  String parentId = this.getParentId();
+
+	  if (parentId.length() > 0) {
+	    insert.addValue("ParentID", this.prefix+"."+this.getParentId());
+	  }
+	  else {
+	    insert.addValue("ParentID", this.prefix);
+	  }
+	}
+
+	action = insert;
+	classifyRef = -1;
+      }
+      action.addValue("ClassifyID", fullId);
+      action.addValue("Name", this.name);
+      action.addValue("Description", this.descriptor);
+
+      String tailId = "0";
+      if (this.id.length() > 0) {
+	int lastDot = this.id.lastIndexOf('.');
+	if (lastDot >= 0) {
+	  tailId = this.id.substring(lastDot+1);
+	}
+	else {
+	  tailId = this.id;
+	}
+      }
+      action.addValue("ClassifyOrder", tailId, GS3SQLField.INTEGER_TYPE);
+      action.addValue("NumLeafDocs", Integer.toString(this.noOfLeafDocs()), GS3SQLField.INTEGER_TYPE);
+
+      statement.execute(action.toString());
+    }
+    catch (SQLException sqlEx) {
+      if (action == null) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx);
+      }
+      else {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx + " " + action.toString());
+      }
+      return false;
+    }
+
+    // get the ClassifyRef if we don't already have it (have done a
+    // insert action above)...
+    if (classifyRef == -1) {
+      try {
+        ResultSet results = statement.executeQuery(select.toString());
+	if (!results.first()) {
+	    statement.close();
+	    return false;
+	}
+	
+	classifyRef = results.getInt("ClassifyRef");
+	
+      }
+      catch (SQLException sqlEx) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx);
+	return false;
+      }
+    }
+    else {
+      // TODO: clear 'dead' child classifications
+
+      // delete child documents
+      GS3SQLDelete delete = new GS3SQLDelete("classdocuments");
+      delete.setWhere(where);
+      try {
+	  statement.execute(delete.toString());
+      } catch (SQLException e) {
+	  System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	  return false;
+      }
+    }
+    
+    // post the child nodes...
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      if (!childNode.writeSQL(connection)) {
+	  System.out.println("Failed to write " );
+	  return false;
+      }
+    }
+
+    // note the child documents...
+    iterator = this.childDocs.iterator();
+    int order = 1;
+    while (iterator.hasNext()) {
+      AbstractHierarchyDocument hierarchyDoc = (AbstractHierarchyDocument) iterator.next();
+      DocumentID docId = hierarchyDoc.getID();
+
+      insert = new GS3SQLInsert("classdocuments");
+      insert.addValue("ClassifyRef", Integer.toString(classifyRef), GS3SQLField.INTEGER_TYPE);
+      insert.addValue("DocID", docId.toString());
+      insert.addValue("DocOrder", Integer.toString(order), GS3SQLField.INTEGER_TYPE);
+      try {
+	  statement.execute(insert.toString());
+      } catch (SQLException e) {
+	  System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	  return false;
+      }
+      order ++;
+    }
+    // close the statment
+    try {
+	statement.close();
+    } catch (SQLException e) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+    }
+
+    return true;
+  }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java	(revision 12189)
@@ -0,0 +1,44 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public interface ClassifierInterface
+{
+  /**
+   *  Initialise the classifier as required.  The parameters given are specific
+   *  to the classifier type, so this interface does not guarantee any interpretations
+   *  of any given parameter.
+   *  
+   *  @param <code>List</code> the parameters with which to configure the classifier.
+   */
+  public void configure(List parameters);
+  public void configure(java.net.URL baseURL, org.w3c.dom.Node node);
+
+  public void setDatabase(org.greenstone.gsdl3.gs3build.database.GS3SQLConnection connection);
+
+  /**
+   *  Classify an individual document - the document should be informed of any
+   *  classification nodes which it is assigned to, so that it can obtain knowledge
+   *  of any "parents".  These are assigned into the "classified" metadata item in
+   *  the "gsdl3" namespace...
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document);
+  
+  public void startClassifierPass(int pass);
+  
+  public int getClassifierPasses();
+  
+  public void endClassifierPass(int pass);
+  
+  /**
+   *  Called at the end of the classification sequence - the classifier should
+   *  save its state at this point...
+   */
+  public void completeClassification();
+
+    public boolean addClassifierDescription(Element classifier_list);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java	(revision 12189)
@@ -0,0 +1,155 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+import java.util.Iterator;
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+
+public class ClassifierManager
+{
+  DocumentList           documents;
+  ClassifierInterface [] list;
+  int                    size;
+  int                    used;
+  GS3SQLConnection       database;
+
+  /**
+   *  Instantiate a classifier manager that will classify the
+   *  documents in a given <code>DocumentList</code> object.
+   *  The list of documents should not change during the
+   *  <code>classifyDocuments</code> function, and the documents
+   *  themselves may be altered at that time.
+   *
+   *  @param <code>DocumentList</code> a provider of documents for
+   *         classification.
+   */
+  public ClassifierManager(DocumentList documentList, GS3SQLConnection database)
+  { this.list = new ClassifierInterface[10];
+    this.size = 10;
+    this.used = 0;
+    this.documents = documentList;
+    this.database  = database;
+  }
+
+  /**
+   *  Add a classifier to the end of the list of classifiers...
+   *
+   *  @param <code>ClassifierInterface</code> the classifier.
+   */
+  public void addClassifier(ClassifierInterface classifier)
+  { this.ensureSize(this.used + 1);
+    this.list[this.used] = classifier;
+    this.used ++;
+
+    classifier.setDatabase(this.database);
+  }
+
+  /**
+   *  Classify an individual document - the document itself may be written
+   *  to in consequence, in which case the document <strong>must</strong>
+   *  be rewritten to disk.  This function does <strong>not</strong> do the
+   *  writing itself.
+   *
+   *  @param <code>DocumentID</code> the document's identifier.
+   *  @param <code>DocumentInterface</code> the document itself.
+   */
+  public void classifyDocument(DocumentID docId, DocumentInterface document)
+  { for (int i = 0; i < this.used; i ++)
+    {	this.list[i].classifyDocument(docId, document);
+    }
+  }
+
+  /**
+   *  Classify all the documents that the document list is able to provide...
+   */
+  public void classifyDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.list[i].getClassifierPasses(); p ++)
+      {	this.list[i].startClassifierPass(p);
+
+	Iterator iterator = this.documents.iterator();
+
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+
+	  if (document.isIndexed()) {
+	    if (!this.list[i].classifyDocument(document.getID(), document)) {
+	      System.out.println("Ending document");
+	    }
+	    
+	    if (document.isChanged()) {
+	      //	      System.out.println("Writing document " + document.getID());
+	      this.documents.storeChangedDocument(document);
+	    }
+	  }
+	}
+	this.list[i].endClassifierPass(p);
+      }
+      this.list[i].completeClassification();
+    }
+  }
+
+  public void ensureSize(int size)
+  { while (size >= this.size)
+    { ClassifierInterface newList [] = new ClassifierInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+
+  public static ClassifierInterface loadClassifier(String classifierName)
+  { System.out.println("Loading classifier " + classifierName);
+    try {
+      ClassifierInterface classifier = 
+	(ClassifierInterface) Class.forName("org.greenstone.gsdl3.gs3build.classifier."+classifierName+"Classifier").newInstance();
+      return classifier;
+    }
+    catch (ClassNotFoundException ex) {
+      System.err.println("Attempt to load classifier " + classifierName + " which does not exist");
+    }
+    catch (InstantiationException instEx) {
+      System.err.println("Unable to create classifier " + classifierName);
+    }
+    catch (IllegalAccessException accessEx) {
+      System.err.println("Unable to instantiate classifier " + classifierName + " due to permissions problems " + accessEx.toString());
+    }
+    return null;
+  }
+    
+  public static ClassifierInterface loadClassifier(String classifierName, List params)
+  { ClassifierInterface classifier = loadClassifier(classifierName);
+    if (classifier != null) {
+      classifier.configure(params);
+    }
+    return classifier;
+  }
+
+  public boolean addServiceDescriptions(Element service_class_list) {
+    if (this.used==0) {
+      // no classiifers
+      return true;
+    }
+    Document owner_doc = service_class_list.getOwnerDocument();
+    Element browse_service = owner_doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    browse_service.setAttribute(GSXML.NAME_ATT, "GS3Browse");
+    service_class_list.appendChild(browse_service);
+    
+    Element classifier_list = owner_doc.createElement(GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+    browse_service.appendChild(classifier_list);
+    boolean success = true;
+    for (int i = 0; i < this.used; i ++) {
+	if (!this.list[i].addClassifierDescription(classifier_list)) {
+	    success = false;
+	}
+    }
+    return success;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java	(revision 12189)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+public interface ClassifierObserverInterface
+{
+  public void recordClassification(String label);
+  public java.util.List getClassifications();
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java	(revision 12189)
@@ -0,0 +1,309 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+public class GS2HierarchyClassifier implements ClassifierInterface
+{ 
+  class GS2HierarchyClassifierObserver implements ClassifierObserverInterface
+  { List              classifications;
+    DocumentInterface document;
+
+    public GS2HierarchyClassifierObserver(DocumentInterface document)
+    { this.document = document;
+    }
+
+    public void recordClassification(String label)
+    { this.document.addDocumentMetadata("gsdl3", "classified", label);
+      this.document.setChanged(true);
+      //      System.out.println("Assigned document " + this.document.getID().toString() + " to " + label);
+    }
+
+    public List getClassifications()
+    { return this.classifications;
+    }
+  }
+
+  class GS2HierarchyNode extends AbstractHierarchyNode
+  { /**
+     *  Simple node
+     */
+    public GS2HierarchyNode()
+    { super();
+    }
+    
+    public GS2HierarchyNode(String prefix, String name, String id, String descriptor)
+    { super(prefix, name, id, descriptor);
+    }
+    
+    /**
+     *  Take a document, and find the classifications that it matches against in
+     *  the current hierarchy.  
+     *
+     *  @param <code>DocumentID</code> the id of the document being classified
+     *  @param <code>List</code> the values against which the classifier should
+     *         test for the document being a match - i.e. the pertinent document
+     *         property values.
+     *  @param <code>ClassifierObserverInterface</code>
+     *         object modifies the document with information about the
+     *         classifications that it fell within.
+     */
+    public void getClassifications(DocumentID documentID, List values, String sortKey,
+				   ClassifierObserverInterface observer)
+    { // Go through the list of values, and if this node matches, record
+      // the fact in the classifier observer...
+      Iterator valueList = values.iterator();
+
+      while (valueList.hasNext())
+      {	if (this.isMatch(valueList.next().toString()))
+	{ observer.recordClassification(this.id);
+	  this.addDocument(documentID, sortKey);
+	}
+      }
+     
+      // recurse into the child nodes for them to do the same...
+      Iterator childList = this.childNodes.iterator();
+
+      while (childList.hasNext())
+      { ((GS2HierarchyNode) childList.next()).getClassifications(documentID, values, sortKey, observer);
+      }
+    }
+  }
+
+  /**
+   *  An inner class to handle Classifier files
+   */
+  class GS2HierarchyHandler extends GS2TextFileHandler
+  { GS2HierarchyNode rootNode = null;
+
+    GS2HierarchyHandler(String hierarchyName, String content)
+    { super(content);
+ 
+      String prefix = "CL"+hierarchyName;
+      this.rootNode = new GS2HierarchyNode(prefix, hierarchyName, "", hierarchyName);
+
+      while (this.hasMoreLines()) {
+	String description;
+	String name, id;
+
+	this.getLine();
+	
+	name = this.getEntry(true);
+	if (name == null || name.length() == 0) {
+	  continue;
+	}
+
+	id = this.getEntry(true);
+	if (id == null || id.length() == 0) {
+	  continue;
+	}
+
+	description = this.getEntry(true);
+	if (description == null || description.length() == 0) {
+	  continue;
+	}
+	
+	// Initialise the tree
+	GS2HierarchyNode node = new GS2HierarchyNode(prefix, name, id, description);
+
+	this.rootNode.add(node);
+      }
+    }
+
+    public GS2HierarchyNode getHierarchy()
+    { return this.rootNode;
+    }
+  }
+
+  private          GS2HierarchyNode hierarchy;
+  List             fields;
+  String           sortBy;
+  GS3SQLConnection database;
+
+  public GS2HierarchyClassifier()
+  { // a 'do nothing' constructor which takes more information from configure...
+    this.fields = new ArrayList();
+    this.database = null;
+  }
+
+  public GS2HierarchyClassifier(File file)
+  {	
+    String documentText = 
+      DocumentLoader.getAsString(file);
+	
+    if (documentText == null) {
+      System.err.println("GS2HierarchyClassifier: Unable to load any content for " + file);
+      return;
+    }
+
+    GS2HierarchyHandler handler = new GS2HierarchyHandler("CL", documentText);
+
+    this.hierarchy = handler.getHierarchy();
+    this.fields = new ArrayList();
+    //    System.out.println("loaded hierarchy for "+ file.toString());
+    this.database = null;
+  }
+
+  public void setDatabase(GS3SQLConnection database)
+  { this.database = database;
+  }
+
+  public void configure(URL url, org.w3c.dom.Node node)
+  {
+  }
+
+  public void configure(List parameters)
+  {
+    File file = null;
+    
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+      else if (param.equals("-hfile")) {
+	// this parameter had better have been expanded previously to a full path,
+	// as we're not going to do that here!!!
+	if (iterator.hasNext()) {
+	  file = new File(iterator.next().toString());
+	}
+      }
+      else if (param.equals("-sort")) {
+	if (iterator.hasNext()) {
+	  String sort = iterator.next().toString();
+	  if (sort != null && sort.length() > 0) {
+	    this.sortBy = sort;
+	  }
+	}
+      }
+    }
+
+    if (file == null) {
+      return;
+    }
+
+    String documentText = 
+      DocumentLoader.getAsString(file);
+	
+    if (documentText == null) {
+      System.err.println("GS2HierarchyClassifier: Unable to load any content for " + file);
+      return;
+    }
+
+    GS2HierarchyHandler handler = new GS2HierarchyHandler(this.fields.get(0).toString(), documentText);
+
+    this.hierarchy = handler.getHierarchy();    
+  }
+
+  /**
+   *  Classify a particular document - the document id will be used to record its
+   *  membership of any classifier nodes, and the document interface will also be
+   *  used to assign back knowledge of any classifiers to which the document is
+   *  assigned.
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>DocumentInterface</code> the document interface
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { if (this.hierarchy == null) {
+      return true;
+    }
+    
+    // the observer records the assignment of documents to classifications
+    GS2HierarchyClassifierObserver classifyObserver = new GS2HierarchyClassifierObserver(document);
+		
+    // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      //      System.out.println("Matching " + values.get(0).toString());
+
+      if (documentID == null) {
+	System.out.println("Bad documentID");
+	continue;
+      }
+
+      // get the sort key for the metadata item if possible
+      String sortKey = null;
+
+      if (this.sortBy != null) {
+	List sortKeys = document.getDocumentMetadataItem(this.sortBy);
+	if (sortKeys != null && sortKeys.size() > 0) {
+	  sortKey = sortKeys.get(0).toString();
+	}
+      }
+
+      // ...and send them to the classifier
+      this.hierarchy.getClassifications(documentID, values, sortKey, classifyObserver);
+    }
+    return true;
+  }
+
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+  
+  public void endClassifierPass(int pass)
+  {
+  }
+  
+  public void completeClassification()
+  { // TODO: store the classifications to file...
+
+    // The hierarchy member may be null if the classifier did not initialise correctly...
+    if (this.hierarchy != null) {
+      this.hierarchy.writeSQL(this.database);
+    }
+  }
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	//classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier.setAttribute(GSXML.NAME_ATT, "name"); // what is teh name???
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java	(revision 12189)
@@ -0,0 +1,344 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.xerces.parsers.SAXParser;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+
+public class HierarchyClassifier extends AbstractClassifier
+{ 
+  class HierarchyClassifierObserver implements ClassifierObserverInterface
+  { List              classifications;
+    DocumentInterface document;
+
+    public HierarchyClassifierObserver(DocumentInterface document)
+    { this.document = document;
+    }
+
+    public void recordClassification(String label)
+    { this.document.addDocumentMetadata("gsdl3", "classified", label);
+    }
+
+    public List getClassifications()
+    { return this.classifications;
+    }
+  }
+
+  class HierarchyNode extends AbstractHierarchyNode
+  { /**
+     *  Simple node
+     */
+    public HierarchyNode()
+    { super();
+    }
+
+    public HierarchyNode(String prefix, String name, String id, String descriptor)
+    { super(prefix, name, id, descriptor);
+    }
+    
+    public void getClassifications(DocumentID documentID, List values, String sortKey,
+				   ClassifierObserverInterface observer)
+    { Iterator valueList = values.iterator();
+
+      while (valueList.hasNext())
+      {	if (this.isMatch(valueList.next().toString()))
+	{ observer.recordClassification(this.name);
+	  this.addDocument(documentID, sortKey);
+	}
+      }
+     
+      // recurse into the child nodes for them to do the same...
+      Iterator childList = this.childNodes.iterator();
+
+      while (childList.hasNext())
+      { ((HierarchyNode) childList.next()).getClassifications(documentID, values, sortKey, observer);
+      }
+    }
+  }
+
+  /**
+   *  An inner class to handle Classifier files
+   */
+  class HierarchyHandler extends DefaultHandler
+  { StringBuffer description;
+    StringBuffer name;
+    StringBuffer path;
+    StringBuffer match;
+    String       prefix;
+    boolean inElement;
+    AbstractHierarchyNode rootNode = null;
+    AbstractHierarchyNode currentNode = null;
+
+    HierarchyHandler(String classifierName)
+    { super();
+ 
+      this.name        = null;
+      this.description = null;
+      this.path        = null;
+      this.match       = null;
+      this.prefix      = classifierName;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("Hierarchy"))
+      {	// create a new node in the hierarchy
+	HierarchyNode node = new HierarchyNode(prefix, "", "", "Classifier");
+
+	this.rootNode = node;
+	this.currentNode = node;
+      }
+      else if (localName.equals("Classification"))
+      {	// create a new node in the hierarchy
+	HierarchyNode node = new HierarchyNode(this.prefix, "", "", "");
+
+	// Initialise the tree
+	if (this.currentNode == null)
+	{ this.rootNode = node;
+	}
+        // Add the node
+	else
+	{ this.currentNode.addChild(node);
+	}
+	this.currentNode = node;
+      }
+      else if (localName.equals("Name"))
+      {	this.name = new StringBuffer();
+      }
+      else if (localName.equals("Path"))
+      {	this.path   = new StringBuffer();
+      }
+      else if (localName.equals("Description"))
+      {	this.description = new StringBuffer();
+      }
+      else if (localName.equals("Matches"))
+      {	this.match = new StringBuffer();
+      }
+      else if (localName.equals("Children"))
+      {	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("Classification"))
+      {	if (this.currentNode.getParent() == null) {
+	  // TODO: Error state
+          // this.rootNodes.add(this.currentNode);
+        }
+	this.currentNode = this.currentNode.getParent();
+      }
+      else if (localName.equals("Name"))
+      { this.currentNode.setName(XMLTools.cleanString(this.name.toString()));
+      System.out.println("Name is " + this.name);
+        this.name = null;
+      }
+      else if (localName.equals("Path"))
+      {	this.currentNode.setID(XMLTools.cleanString(this.path.toString()));
+        this.path = null;
+      }
+      else if (localName.equals("Description"))
+      { this.currentNode.setDescriptor(XMLTools.cleanString(this.description.toString()));
+        this.description = null;
+      }
+      else if (localName.equals("Matches"))
+      { this.currentNode.addMatch(XMLTools.cleanString(this.match.toString()));
+        this.match = null;
+      }
+      else if (localName.equals("Children"))
+      {	
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { String string = new String(c, start, length);
+      if (this.path != null)
+      { this.path.append(string);
+      }
+      else if (this.name != null)
+      { this.name.append(string);
+      }
+      else if (this.description != null)
+      { this.description.append(string);
+      }
+      else if (this.match != null)
+      { this.match.append(string);
+      }
+    }
+
+    public AbstractHierarchyNode getHierarchy()
+    { return this.rootNode;
+    }
+  }
+  
+  private GS3SQLConnection database;
+  private AbstractHierarchyNode    hierarchy;
+  private List             fields;
+  private String           sortBy;
+  private String           name;
+
+  public HierarchyClassifier(String name, URL basefile, List fields, String sortBy)
+  {	
+    try 
+    { SAXParser parser = new SAXParser();
+      HierarchyHandler handler = new HierarchyHandler(name);
+      /*
+      XMLReader reader = XMLReaderFactory.createXMLReader();
+      reader.setContentHandler(handler);
+      reader.setErrorHandler(handler);*/
+      parser.setContentHandler(handler);
+
+      // A Classifier consists of one file only - get it.
+      /*
+      FileReader fileReader = new FileReader(basefile);
+      reader.parse(new InputSource(fileReader));
+      */
+      parser.parse(basefile.toString());
+
+      this.hierarchy = handler.getHierarchy();
+      this.fields = fields;
+      this.sortBy = sortBy;
+      this.name = name;
+      System.out.println("Counting " + this.fields.size() + " fields");
+    }
+    catch (SAXException saxException)
+    { // TODO: log error
+	System.out.println("unable to process hierarchy " + saxException.toString());
+    }
+    catch (java.io.FileNotFoundException fileException)
+    {
+	System.out.println("unable to process hierarchy " + fileException.toString());
+    }
+    catch (java.io.IOException ioException)
+    {
+	System.out.println("unable to process hierarchy " + ioException.toString());
+    }
+  }
+
+  public void configure(List parameters)
+  {
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+    }
+  }
+
+  /**
+   *  Indicate the database to be used for saving the hierarchy to.
+   *
+   *  @param <code>GS3SQLConnection</code> the database
+   */
+  public void setDatabase(GS3SQLConnection database)
+  { this.database = database;
+  }
+
+  /**
+   *  Classify a particular document - the document id will be used to record its
+   *  membership of any classifier nodes, and the document interface will also be
+   *  used to assign back knowledge of any classifiers to which the document is
+   *  assigned.
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>DocumentInterface</code> the document interface
+   *
+   *  @return <code>boolean</code> a return value - currently always 
+   *          <code>true</code> for success
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { // the observer records the assignment of documents to classifications
+    HierarchyClassifierObserver classifyObserver = new HierarchyClassifierObserver(document);
+
+    // if we have no fields, there is nothing to do...
+    if (this.fields == null)
+    { return true;
+    }
+
+    // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+			
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      if (documentID == null) {
+	System.out.println("Bad documentID");
+	continue;
+      }
+
+      // get the sort key for the metadata item if possible
+      String sortKey = null;
+
+      if (this.sortBy != null) {
+	List sortKeys = document.getDocumentMetadataItem(this.sortBy);
+	if (sortKeys != null && sortKeys.size() > 0) {
+	  sortKey = sortKeys.get(0).toString();
+	}
+      }
+
+      // ...and send them to the classifier
+      this.hierarchy.getClassifications(documentID, values, sortKey, classifyObserver);
+    }
+    return true;
+  }
+
+  /**
+   *  @see ClassifierInterface:getClassifierPasses
+   */
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+  
+  public void endClassifierPass(int pass)
+  {
+  }
+  
+  public void completeClassification()
+  { // TODO: store the classifications to file...
+    if (this.hierarchy != null) {
+      this.hierarchy.writeSQL(this.database);
+    }
+  }
+
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java	(revision 12189)
@@ -0,0 +1,31 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.util.GregorianCalendar;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public class BuildIDFactory implements IDFactoryInterface
+{
+	int sequenceNumber;
+
+	public BuildIDFactory(BuildManager manager)
+	{
+		GregorianCalendar today = new GregorianCalendar();
+
+		String oldBuild = manager.getLastBuildSequence();
+		if (oldBuild != null)
+		{ this.sequenceNumber = Integer.parseInt(oldBuild);
+		}
+		else
+		{	this.sequenceNumber = 0;
+		}
+	}
+
+	public DocumentID getDocumentID(DocumentInterface document)
+	{	String value = new String(Integer.toString(this.sequenceNumber));
+		this.sequenceNumber ++;
+		return new DocumentID(value);
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java	(revision 12189)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+import org.greenstone.gsdl3.gs3build.FileCrawlObserver;
+
+public class CollectionCrawlObserver implements FileCrawlObserver
+{
+  BuildManager manager;
+
+  public CollectionCrawlObserver(BuildManager manager)
+  { this.manager = manager;
+  }
+
+  public void processFile(File file)
+  { if (file.getName().equals("collect.cfg")) {
+      GS2CollectionCfg config = GS2CollectionCfg.getGS2CollectionCfg(this.manager, file);
+    }
+    else if (file.toString().endsWith(".txt")) {
+      // Commented out: this is now done on the line above...
+      // GS2HierarchyClassifier classifier = new GS2HierarchyClassifier(file);
+      // manager.getClassifierManager().addClassifier(classifier);
+    }
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java	(revision 12189)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+/**
+ *  A class to consolidate the metadata assigned to the documents of a collection
+ *  as per the extension to Greenstone 2 done by George Buchanan.
+ */
+
+public class CollectionMetadata
+{ METSDescriptive metadata;
+	
+  public CollectionMetadata()
+  { this.metadata = new METSDescriptive("CM0", "collection");
+  }
+
+  /**
+   *  Accrue the metadata assigned to a particular document
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> the field of metadata
+   *  @param <code>String</code> the actual value assigned
+   */
+  public void addDocumentMetadata(DocumentID documentID, String namespace, String field, String value)
+  { // we don't presently use the document ID in this member function
+    this.metadata.addMetadata(namespace, field, value);
+  }
+
+  /**
+   *  Set the metadata stored on a given metadata item - any other values are
+   *  destroyed.
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   *  @param <code>String</code> the actual value assigned
+   */
+  public void setCollectionMetadata(String namespace, String label, String value)
+  { this.metadata.setMetadata(namespace, label, value);
+  }
+
+  /**
+   *  Get the metadata stored on a given metadata item
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   */
+  public List getCollectionMetadata(String namespace, String label)
+  { return this.metadata.getMetadata(namespace, label);
+  }
+
+  /**
+   *  Write the collection metadata
+   *
+   *  TODO: dummy function - complete
+   */
+  public boolean write(PrintWriter writer)
+  { return false;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java	(revision 12189)
@@ -0,0 +1,159 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.File;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.greenstone.gsdl3.gs3build.classifier.ClassifierInterface;
+import org.greenstone.gsdl3.gs3build.classifier.ClassifierManager;
+
+import org.greenstone.gsdl3.gs3build.indexers.*;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public class GS2CollectionCfg
+{
+  BuildManager buildManager;
+  IndexerInterface indexer;
+
+  class GS2CollectCfgHandler extends GS2TextFileHandler
+  { 
+    
+    GS2CollectCfgHandler(String content, File collectFile, GS2CollectionCfg configurator)
+    { super(content);
+ 
+      while (this.hasMoreLines()) {
+	String lineType;
+
+	this.getLine();
+	
+	lineType = this.getEntry(true);
+	if (lineType == null || lineType.length() == 0) {
+	  continue;
+	}
+
+	if (lineType.equals("classify"))
+	{
+	  String classType = this.getEntry(true);
+	  if (classType == null || classType.length() == 0) {
+	    continue;
+	  }
+	  classType = "GS2" + classType;
+	  
+	  List paramList = new ArrayList();
+	  
+	  // read in the parameters - to allow for better abstraction of other parts,
+	  // the handling of this is a bit 'odd', specifically modifying any file
+	  // references here before passing them on...
+	  String parameter = null;
+	  String lastParam = null;
+	  while (this.hasMore())
+	  { String param = this.getEntry(true);
+	    
+	    if (param != null && param.length() > 0) 
+	    { // expand any hfile references...
+	      if (lastParam != null && lastParam.equals("-hfile")) {
+		param = collectFile.getParent() + File.separator + param;
+	      }
+	      
+	      paramList.add(param);
+	    }
+
+	    lastParam = param;
+	  }
+
+	  ClassifierInterface classifier = ClassifierManager.loadClassifier(classType, paramList);
+	  if (classifier != null) {
+	    configurator.addClassifier(classifier);
+	  }
+	  else {
+	    System.err.println("Classifier not loaded");
+	  }
+	}
+	else if (lineType.equals("buildtype")) {
+	  if (this.hasMore()) {
+	    String type = this.getEntry(true);
+	    if (type != null) {
+	      configurator.setBuildType(type);	      
+	    }
+	  }
+	}
+	else if (lineType.equals("indexes")) {
+	  while (this.hasMore()) {
+	    String index = this.getEntry(true);
+	    if (index != null) {
+	      System.out.println("Adding index " + index);
+	      configurator.addIndex(index);
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  public GS2CollectionCfg(BuildManager manager, File file)
+  {
+    String documentText = 
+      DocumentLoader.getAsString(file);
+
+    if (documentText == null) {
+      return;
+    }
+
+    this.buildManager = manager;
+
+    GS2CollectCfgHandler handler = new GS2CollectCfgHandler(documentText, file, this);
+  }
+  
+  void addClassifier(ClassifierInterface classifier)
+  {
+    ClassifierManager manager = this.buildManager.getClassifierManager();
+
+    if (manager != null) {
+      manager.addClassifier(classifier);
+    }
+    else {
+      System.out.println("Unable to install due to missing manager");
+    }
+  }
+
+  void addIndex(String index)
+  {
+    if (this.indexer == null) {
+      this.setBuildType("mg");
+    }
+    this.indexer.configure(IndexerInterface.GS2_INDEX_LABEL, index);
+  }
+
+  void setBuildType(String type)
+  { if (type.equals("mgpp")) {
+      this.indexer = new MGPPIndexer("mgpp");
+    }
+    else {
+      this.indexer = new MGIndexer("mg");
+    }
+    this.buildManager.addIndexer(this.indexer);
+  }
+
+  public static GS2CollectionCfg getGS2CollectionCfg(BuildManager manager, File cfgFile)
+  { GS2CollectionCfg configurator = new GS2CollectionCfg(manager, cfgFile);
+    return configurator;
+  }
+
+  public static GS2CollectionCfg getGS2CollectionCfg(BuildManager manager, String etcDir)
+  {
+    File file = new File(etcDir, "collect.cfg");
+    
+    if (!file.exists()) {
+      return null;
+    }
+
+    return getGS2CollectionCfg(manager, file);
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java	(revision 12189)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public interface IDFactoryInterface
+{
+	public DocumentID getDocumentID(DocumentInterface document);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java	(revision 12189)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+/**
+ *  Facilitates communication between Greenstone and a database system,
+ *  be it gdbm, an SQL database or other product - what lies underneath
+ *  is down to you...
+ *
+ *  As this is a "write only" interface, the document is only stored as
+ *  required.
+ */
+
+import java.sql.*;
+
+public interface DatabaseAdaptorInterface
+{
+    //  public Connection getConnection();
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java	(revision 12189)
@@ -0,0 +1,54 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+
+public class GS3SQLAction
+{ String      table;
+  List        values;
+
+  public GS3SQLAction(String table)
+  {
+    this.table  = table;
+    this.values = new ArrayList();
+  }
+
+  public void addValue(String field, String value, String type)
+  { this.values.add(new GS3SQLValue(field, value, type));
+  }
+  
+  public void addValue(String field, String value)
+  { this.values.add(new GS3SQLValue(field, value));
+  }
+
+  public void addDate(String field, java.sql.Timestamp date)
+  { this.values.add(new GS3SQLValue(field, getDateString(date)));
+  }
+
+  public static String getDateString(java.sql.Timestamp date)
+  { StringBuffer reply = new StringBuffer();
+    Calendar calendar = new GregorianCalendar();
+    calendar.setTime(date);
+
+    reply.append(Integer.toString(calendar.get(Calendar.YEAR)));
+    reply.append("-");
+    reply.append(Integer.toString(calendar.get(Calendar.MONTH)+1));
+    reply.append("-");
+    reply.append(Integer.toString(calendar.get(Calendar.DAY_OF_MONTH)));
+    reply.append(" ");
+    reply.append(Integer.toString(calendar.get(Calendar.HOUR_OF_DAY)));
+    reply.append(":");
+    reply.append(Integer.toString(calendar.get(Calendar.MINUTE)));
+    reply.append(":");
+    reply.append(Integer.toString(calendar.get(Calendar.SECOND)));
+
+    return reply.toString();
+  }
+  
+  public void addField(String field)
+  { this.values.add(new GS3SQLValue(field, null));
+  }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java	(revision 12189)
@@ -0,0 +1,245 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+
+public class GS3SQLConnection extends SQLConnection
+{
+
+    public GS3SQLConnection(java.sql.Connection connection, String database)
+    { 
+	super(connection, database);
+    }
+    
+
+  public void clearCollection(String collectionName)
+  {
+      dropDatabase(collectionName);
+  }
+    
+    public GS3SQLConnection cloneConnection() {
+	GS3SQLConnection conn = GS3SQLConnectionFactory.getGS3SQLConnection(this.database);
+	return conn;
+    }
+  /**
+   *  Initialise a collection for use.
+   *
+   *  @return <code>boolean</code> <code>true</code> if the collection successfully initialised.
+   */
+  public boolean initCollection(String collectionName)
+  {	
+    /*
+    // Check if the collection already exists
+    // if so, abort
+
+    // do a sample query to check if the collection exists
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+    if (this.execute(select.toString())) {
+      return true;
+    }
+
+    // if there is an error, assume that the database doesn't exist...
+    // and reconnect to a "test" database for now...
+    this.connection = GS3SQLConnectionFactory.getConnection("test");
+    */
+
+      // create the new db
+      if (!createDatabase(collectionName)) {
+          System.err.println("Couldn't create database " + collectionName);
+	  return false;
+      }
+      // now connect to it
+      if (!connectToDatabase(collectionName)) {
+          System.err.println("couldn't connect to the database " + collectionName);
+	  return false;
+      }
+    
+      
+    try {
+      Statement statement = this.connection.createStatement();
+
+      // create build history table
+      GS3SQLCreateTable buildTable = new GS3SQLCreateTable("build");
+      buildTable.addAutoPrimaryKey("buildRef");
+      buildTable.addProperty("buildKey", 64);
+      buildTable.addProperty("buildValue");
+      statement.execute(buildTable.toString());
+
+      // create document table
+      GS3SQLCreateTable docTable = new GS3SQLCreateTable("document");
+      docTable.addProperty("DocID", 64);   // The document identifer, unique in collection
+      docTable.setPrimaryKey("DocID");     // ...which is also the primary key
+      docTable.addProperty("DocType", 64); // The document type - used to regenerate the
+                                           // correct Document object type.
+      docTable.addProperty("AccessionDate", GS3SQLField.DATETIME_TYPE);
+      docTable.addProperty("IndexedDate", GS3SQLField.DATETIME_TYPE);
+      docTable.addProperty("ModifiedDate", GS3SQLField.DATETIME_TYPE);
+      statement.execute(docTable.toString());
+      
+      // create structure table
+      GS3SQLCreateTable structureMap = new GS3SQLCreateTable("structure");
+      structureMap.addAutoPrimaryKey("StructureRef");
+      structureMap.addProperty("DocID", 64);          // a 'foreign key' in effect - but not treated
+                                                      // as such for efficiency reasons
+      structureMap.addProperty("StructureID", 64);    // this identifier
+      structureMap.addProperty("StructureType", 64);  // the type of the structure
+      structureMap.addProperty("Label", 128);         // the label of the structure
+      statement.execute(structureMap.toString());
+    
+      // create section table
+      GS3SQLCreateTable sectionMap = new GS3SQLCreateTable("divisions");
+      sectionMap.addAutoPrimaryKey("DivisionRef");
+      sectionMap.addProperty("StructureRef", GS3SQLField.INTEGER_TYPE);
+      sectionMap.addProperty("DocID", 64);        // a 'foreign key' in effect - but not treated
+                                                  // as such for efficiency reasons
+      sectionMap.addProperty("ParentType", 64);   // the type of the parent - document or section
+      sectionMap.addProperty("ParentRef", 64);    // the parent sql reference identifier
+      sectionMap.addProperty("SectionID", 64);    // this identifier
+      sectionMap.addProperty("DivisionType", 64); // the type of the section
+      sectionMap.addProperty("LabelOrder", 64);   // the order of the section
+      sectionMap.addProperty("ShortLabel", 128);  // the short label of the section
+      sectionMap.addProperty("UserLabel", 255);   // the long label of the section
+      statement.execute(sectionMap.toString());
+
+      // a division->metadata reference mapping
+      GS3SQLCreateTable metaRefs = new GS3SQLCreateTable("divisionmetarefs");
+      metaRefs.addProperty("DocID", 64);
+      metaRefs.addProperty("DivisionRef", GS3SQLField.INTEGER_TYPE);
+      metaRefs.addProperty("DivisionType", 16);   // whether a namespace or meta ref
+      metaRefs.addProperty("MetaID", 64); // the metadata reference itself, as a string
+      statement.execute(metaRefs.toString());
+
+      // a division->file reference mapping
+      GS3SQLCreateTable fileRefs = new GS3SQLCreateTable("divisionfilerefs");
+      fileRefs.addProperty("DocID", 64);
+      fileRefs.addProperty("DivisionRef", GS3SQLField.INTEGER_TYPE);
+      fileRefs.addProperty("DivisionType", 16); // whether section, group or file
+      fileRefs.addProperty("FileID", 32);
+      statement.execute(fileRefs.toString());
+      
+      //   create metadata table section table
+      GS3SQLCreateTable metadataSection = new GS3SQLCreateTable("metadata");
+      metadataSection.addAutoPrimaryKey("MetadataRef");
+      metadataSection.addProperty("DocID", 64);   // this table isn't actually used for much
+      metadataSection.addProperty("MetaID", 64);  // in and of itself...
+      metadataSection.addProperty("GroupID", 64); // the 'name' property
+      statement.execute(metadataSection.toString());
+      
+      //   create namespace table
+      GS3SQLCreateTable namespaces = new GS3SQLCreateTable("namespaces");
+      namespaces.addAutoPrimaryKey("NamespaceRef");
+      namespaces.addProperty("MetadataRef", GS3SQLField.INTEGER_TYPE);
+      namespaces.addProperty("DocID", 64);   // these three fields are the primary key
+      namespaces.addProperty("MetaID", 64);
+      namespaces.addProperty("NamespaceID", 64);
+      namespaces.addProperty("NamespaceType", 64);
+      namespaces.addProperty("FileType", 64);
+      namespaces.addProperty("FileLoc");
+      namespaces.addProperty("Creator", 128);
+      statement.execute(namespaces.toString());
+
+      // create metadata grouping table
+      GS3SQLCreateTable metagroups = new GS3SQLCreateTable("mdgroups");
+      metagroups.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE);
+      metagroups.addAutoPrimaryKey("MdGroupRef");
+      metagroups.addProperty("MdGroupID", 32);
+      statement.execute(metagroups.toString());
+
+      // create metadata qualifiers table
+      GS3SQLCreateTable metaqualifiers = new GS3SQLCreateTable("mdqualifiers");
+      metaqualifiers.addProperty("MdGroupRef", GS3SQLField.INTEGER_TYPE);
+      metaqualifiers.addProperty("QualifierName", 64);
+      metaqualifiers.addProperty("QualifierValue", 64);
+      statement.execute(metaqualifiers.toString());
+      
+      // create metadata values table
+      GS3SQLCreateTable metadata = new GS3SQLCreateTable("mdvalues");
+      metadata.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE); // the parent namespace {one candidate primary key}
+      metadata.addProperty("GroupRef", GS3SQLField.INTEGER_TYPE);     // the parent group     {a sub-key or independent primary key}
+      metadata.addProperty("Label", 256);                             // the key of the metadata item
+      metadata.addProperty("Value");                                  // the value of the metadata item
+      statement.execute(metadata.toString());
+    
+      //   create file section table
+      GS3SQLCreateTable filesec = new GS3SQLCreateTable("filesection");
+      filesec.addAutoPrimaryKey("FileSectionRef");
+      filesec.addProperty("DocID", 64);
+      filesec.addProperty("FileSecID", 64);
+      statement.execute(filesec.toString());
+    
+      //   create file groups table
+      GS3SQLCreateTable filegroups = new GS3SQLCreateTable("filegroups");
+      // TODO: some unique identifier
+      filegroups.addAutoPrimaryKey("FileGroupRef");
+      filegroups.addProperty("DocID", 64);
+      filegroups.addProperty("FileGroupID", 64);
+      filegroups.addProperty("ParentRef", GS3SQLField.INTEGER_TYPE);
+      filegroups.addProperty("ParentType", 16);
+      statement.execute(filegroups.toString());
+    
+      //   create file table
+      GS3SQLCreateTable files = new GS3SQLCreateTable("files");
+      // TODO: the unique identifier from file groups table
+      files.addProperty("FileGroupRef", GS3SQLField.INTEGER_TYPE);
+      files.addProperty("FileLocType", 64);
+      files.addProperty("FileLocation");
+      files.addProperty("MIMEType", 64);
+      files.addProperty("FileID", 32);
+      statement.execute(files.toString());
+      
+      //   create admin table...etc.
+
+      // 
+      // END OF METS TABLES
+      //
+
+      //
+      // BEGINNING OF GSDL TABLES
+      //
+      GS3SQLCreateTable classifiers = new GS3SQLCreateTable("classifiers");
+      classifiers.addAutoPrimaryKey("ClassifyRef");
+      classifiers.addProperty("ClassifyID");
+      classifiers.addProperty("ParentID");
+      classifiers.addProperty("ClassifyOrder");
+      classifiers.addProperty("Name");
+      classifiers.addProperty("Description");
+      classifiers.addProperty("NumLeafDocs", GS3SQLField.INTEGER_TYPE);
+      statement.execute(classifiers.toString());
+
+      GS3SQLCreateTable classDocs = new GS3SQLCreateTable("classdocuments");
+      classDocs.addProperty("ClassifyRef", GS3SQLField.INTEGER_TYPE);
+      classDocs.addProperty("DocID");
+      classDocs.addProperty("DocOrder", GS3SQLField.INTEGER_TYPE);
+      statement.execute(classDocs.toString());
+
+      GS3SQLCreateTable classData = new GS3SQLCreateTable("classdata");
+      classData.addProperty("ClassifyRef", GS3SQLField.INTEGER_TYPE);
+      classData.addProperty("Label");
+      classData.addProperty("Value");
+      statement.execute(classData.toString());
+
+      statement.close();
+      //
+      // END OF GSDL TABLES
+      //
+
+    }
+    catch (SQLException ex)
+    {
+      System.out.println(ex.toString());
+      return false;
+    }
+    return true;
+  }
+
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java	(revision 12189)
@@ -0,0 +1,76 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.io.InputStream;
+import java.util.Properties;
+import org.greenstone.gsdl3.util.GlobalProperties;
+
+public class GS3SQLConnectionFactory
+{
+
+    public static Connection getConnection(String database)
+    {
+	try {	    
+	    String server = GlobalProperties.getProperty("mysql.server", "localhost");
+	    String port = GlobalProperties.getProperty("mysql.tcp.port", "3306");
+	    String password = GlobalProperties.getProperty("mysql.gsdl3admin.password");
+	    
+	    Class.forName("com.mysql.jdbc.Driver").newInstance();
+	    String connection_address = "jdbc:mysql://"+server+":"+port+"/"+database+"?user=gsdl3admin";
+	    if (password != null && !password.equals("")) {
+		connection_address += "&password="+password;
+	    }
+	    Connection connection = DriverManager.getConnection(connection_address);
+	    return connection;
+	}
+	catch (Exception ex) {
+	    System.out.println("GS3SQLConnectionFactory.getConnection() Error:");
+	    System.out.println(ex.getMessage());
+	}
+	return null;
+    }
+    
+    public static SQLConnection getSQLConnection(String database)
+    {
+	Connection c = getConnection(database);
+	if (c== null) {
+	    return null;
+	}
+	return new SQLConnection(c, database);
+    }
+
+    
+    public static GS3SQLConnection getGS3SQLConnection(String database)
+    {
+	Connection c = getConnection(database);
+	if (c== null) {
+	    return null;
+	}
+	return new GS3SQLConnection(c, database);
+    }
+
+    /* returns a Connection to the server, */
+    public static Connection getConnection() 
+    {
+	return getConnection("test");
+    }
+
+    /* returns a Connection to the server, */
+    public static SQLConnection getSQLConnection() 
+    {
+	return getSQLConnection("test");
+    }
+
+    public static GS3SQLConnection getGS3SQLConnection()
+    { 
+	return getGS3SQLConnection("test");
+    }
+
+    public static void main(String args[])
+    { 
+	GS3SQLConnection connection = getGS3SQLConnection();
+	connection.initCollection("maya");
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java	(revision 12189)
@@ -0,0 +1,96 @@
+package org.greenstone.gsdl3.gs3build.database;
+  
+import java.util.List;
+import java.util.ArrayList;
+
+
+public class GS3SQLCreateTable
+{ 
+    String tableName;
+    List   properties;
+    List   primaryKey;
+    List   indexed;
+
+    public GS3SQLCreateTable(String tableName)
+    { 
+	this.tableName = tableName;
+	this.properties = new ArrayList();
+	this.primaryKey = new ArrayList();
+	this.indexed    = new ArrayList();
+    }
+    
+    public void addAutoPrimaryKey(String fieldName)
+    {
+	GS3SQLField field = new GS3SQLField(fieldName, GS3SQLField.AUTOINTEGER_TYPE);
+	this.properties.add(field);
+	this.setPrimaryKey(fieldName);
+    }
+
+    public void addProperty(String fieldname)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname);
+	this.properties.add(field);
+    }
+    
+    public void addProperty(String fieldname, String type)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname, type);
+	this.properties.add(field);
+    }
+
+    public void addProperty(String fieldname, int length)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname, length);
+	this.properties.add(field);
+    }
+
+    public void setPrimaryKey(String fieldname)
+    { 
+	this.primaryKey.clear();
+	this.primaryKey.add(fieldname);
+    }
+
+    public void extendPrimaryKey(String fieldname)
+    { 
+	this.primaryKey.add(fieldname);
+    }
+    
+    public void addIndex(String fieldname)
+    { 
+	this.indexed.add(fieldname);
+    }
+
+    public String toString()
+    {	
+	StringBuffer reply = new StringBuffer();
+	
+	reply.append("CREATE TABLE ");
+	reply.append(this.tableName);
+	reply.append(" (");
+	for (int f = 0; f < this.properties.size(); f ++) {
+	    GS3SQLField field = (GS3SQLField) this.properties.get(f);
+	    if (f != 0) {
+		reply.append(",");
+	    }
+	    reply.append(field.toString());
+	}
+
+	for (int p = 0; p < this.primaryKey.size(); p ++) {
+	    if (p == 0){
+		reply.append(", PRIMARY KEY(");
+	    }
+	    else {
+		reply.append(",");
+	    }
+	    reply.append(this.primaryKey.get(p).toString());
+	    
+	    if (p == this.primaryKey.size() - 1) {
+		reply.append(")");
+	    }
+	}
+	// TODO: add primary key, index etc. notations
+	reply.append(");");
+	
+	return reply.toString();
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java	(revision 12189)
@@ -0,0 +1,35 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLDelete
+{ 
+  GS3SQLWhere where;
+  String      table;
+
+  public GS3SQLDelete(String table)
+  { this.table = table;
+    this.where = null;
+  }
+
+  public void setWhere(GS3SQLWhere where)
+  { this.where = where;
+  }
+
+  public String toString()
+  {
+    StringBuffer reply = new StringBuffer();
+    reply.append("DELETE FROM ");
+    reply.append(this.table);
+
+    if (this.where != null)
+    { reply.append(" WHERE ");
+      reply.append(where.toString());
+    }
+
+    return reply.toString();
+  }
+}
+
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java	(revision 12189)
@@ -0,0 +1,74 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLField
+{ String name;
+  int    length;
+  String type;
+
+    public final static String VARCHAR_TYPE = "VARCHAR";
+    public final static String INTEGER_TYPE = "INTEGER";
+    public final static String AUTOINTEGER_TYPE = "AUTOINTEGER";
+    public final static String DATE_TYPE = "DATE";
+    public final static String DATETIME_TYPE = "DATETIME";
+
+    public GS3SQLField(String name)
+    { this.name = name;
+      this.length = 1024;
+      this.type   = VARCHAR_TYPE;
+    }
+
+    public GS3SQLField(String name, int length)
+    { this.name = name;
+      this.length = length;
+      this.type   = VARCHAR_TYPE;
+    }
+
+    public GS3SQLField(String name, String type)
+    { this.name = name;
+      this.type   = type;
+      this.length = 1024;
+    }
+
+    public GS3SQLField(String name, String type, int length)
+    { this.name = name;
+      this.length = length;
+      this.type   = type;
+    }
+
+    public String getName()
+    { return this.name;
+    }
+
+    public int getLength()
+    { return this.length;
+    }
+
+    public String toString()
+    { StringBuffer reply = new StringBuffer();
+
+      reply.append(this.name);
+      reply.append(" ");
+      if (this.type.equals(VARCHAR_TYPE))
+      {
+	if (length > 255) {
+	  reply.append("TINYTEXT");
+	}
+	else {
+	  reply.append("VARCHAR(");
+	  reply.append(Integer.toString(this.length));
+	  reply.append(")");
+	}
+      }
+      else if (this.type.equals(AUTOINTEGER_TYPE))
+      { reply.append("INTEGER AUTO_INCREMENT");
+      }
+      else
+      {
+	reply.append(this.type);
+      }
+      return reply.toString();
+    }
+
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java	(revision 12189)
@@ -0,0 +1,54 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLInsert extends GS3SQLAction
+  { public GS3SQLInsert(String table)
+    { super(table);
+    }
+
+    public String toString()
+    {
+      StringBuffer reply = new StringBuffer();
+      reply.append("INSERT INTO ");
+      reply.append(this.table);
+      reply.append(" ");
+      if (this.values.size() >= 1)
+	reply.append("(");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+        reply.append(value.field);
+      }
+      if (this.values.size() >= 1)
+	reply.append(")");
+      reply.append(" VALUES ");
+      if (this.values.size() >= 1)
+	reply.append("(");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.DATETIME_TYPE) ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+        reply.append(value.value);
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.DATETIME_TYPE) ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+      }
+      if (this.values.size() >= 1)
+	reply.append(")");
+      reply.append(";");
+      return reply.toString();
+    }
+  }
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java	(revision 12189)
@@ -0,0 +1,48 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLSelect extends GS3SQLAction
+{ GS3SQLWhere where;
+  boolean     distinct;
+
+  public GS3SQLSelect(String table)
+  { super(table);
+    this.where  = null;
+    this.distinct = false;
+  }
+    
+  public void setWhere(GS3SQLWhere where)
+  { this.where = where;
+  }
+
+  public void setDistinct(boolean distinct)
+  { this.distinct = distinct;
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+  
+    reply.append("SELECT ");
+
+    if (this.distinct)
+    { reply.append("DISTINCT ");
+    }
+ 
+    for (int i = 0; i < this.values.size(); i ++)
+    { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+      if (i > 0)
+	reply.append(", ");
+      reply.append(value.field);
+    }
+
+    reply.append(" FROM ");
+    reply.append(this.table);
+
+    if (this.where != null)
+    { reply.append(" WHERE ");
+      reply.append(where.toString());
+    }
+    return reply.toString();
+  }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java	(revision 12189)
@@ -0,0 +1,50 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public   class GS3SQLUpdate extends GS3SQLAction
+  { GS3SQLWhere where;
+
+    public GS3SQLUpdate(String table)
+    { super(table);
+      this.where  = null;
+    }
+
+    public void setWhere(GS3SQLWhere where)
+    { this.where = where;
+    }
+
+    public String toString()
+    {
+      StringBuffer reply = new StringBuffer();
+      reply.append("UPDATE ");
+      reply.append(this.table);
+      reply.append(" SET ");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+        reply.append(value.field);
+	reply.append("=");
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+	reply.append(value.value);
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+      }
+      if (this.where != null)
+      { reply.append(" WHERE ");
+	reply.append(where.toString());
+      }
+      reply.append(";");
+      return reply.toString();
+    }
+  }
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java	(revision 12189)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLValue
+{ String field;
+  String value;
+  String type;
+
+  public GS3SQLValue(String field, String value)
+  {
+    this.field = field;
+    this.value = value;
+    this.type  = null;
+  }
+
+  public GS3SQLValue(String field, String value, String type)
+  {
+    this.field = field;
+    this.value = value;
+    this.type  = type;
+  }
+
+  public String getType()
+  {
+    return this.type;
+  }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java	(revision 12189)
@@ -0,0 +1,55 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class GS3SQLWhere implements GS3SQLWhereElement
+{ String condition;
+  List   elements;
+
+  public static final String AND_CONDITION = "AND";
+  public static final String OR_CONDITION = "OR";
+
+  public GS3SQLWhere(GS3SQLWhereElement element)
+  { this.condition = AND_CONDITION;
+    this.elements  = new ArrayList();
+    this.elements.add(element);
+  }
+
+  public GS3SQLWhere()
+  { this.condition = AND_CONDITION;
+    this.elements  = new ArrayList();   
+  }
+
+  public GS3SQLWhere(GS3SQLWhereElement element, String condition, GS3SQLWhereElement other)
+  { this.elements = new ArrayList();
+    this.elements.add(element);
+    this.elements.add(other);
+    this.condition = condition;
+  }
+
+  public void add(GS3SQLWhereElement element)
+  { this.elements.add(element);
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    for (int i = 0; i < this.elements.size(); i ++)
+    {
+      if (i != 0) {
+	reply.append(" ");
+	reply.append(this.condition);
+	reply.append(" ");
+      }
+      reply.append(this.elements.get(i).toString());
+    }
+    return reply.toString();
+  }
+
+  public void setCondition(String condition)
+  { this.condition = condition;
+  }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java	(revision 12189)
@@ -0,0 +1,10 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public interface GS3SQLWhereElement
+{ // empty interface for type safety
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java	(revision 12189)
@@ -0,0 +1,47 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class GS3SQLWhereItem implements GS3SQLWhereElement
+{ String field;
+  String comparison;
+  String value;
+  String type;
+
+  public GS3SQLWhereItem(String field, String comparison, String value, String type)
+  { this.field      = field;
+    this.comparison = comparison;
+    this.value      = value;
+    this.type       = type;
+  }
+
+  public GS3SQLWhereItem(String field, String comparison, String value)
+  { this.field      = field;
+    this.comparison = comparison;
+    this.value      = value;
+    this.type       = GS3SQLField.VARCHAR_TYPE;
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    reply.append(this.field);
+    if (!this.comparison.equals("=")) {
+      reply.append(" ");
+    }
+    reply.append(this.comparison);
+    if (!this.comparison.equals("=")) {
+      reply.append(" ");
+    }
+    if (type.equals(GS3SQLField.VARCHAR_TYPE)) {
+	reply.append("\"" + this.value + "\"");
+    }
+    else {
+	reply.append(this.value);
+    }
+    return reply.toString();
+  }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java	(revision 12189)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.*;
+
+public class MySQLAdaptor implements DatabaseAdaptorInterface
+{
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/SQLConnection.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/SQLConnection.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/database/SQLConnection.java	(revision 12189)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+public class SQLConnection
+{
+    protected Connection connection;
+    protected String database;
+    
+    public SQLConnection(java.sql.Connection connection, String database)
+    { 
+	this.connection = connection;
+	this.database = database;
+    }
+    
+    protected void finalize() {
+	if (this.connection!=null) {
+	    try {
+		this.connection.close();		
+	    } catch (Exception e) {}
+	}	
+	this.connection = null;   
+    }
+    
+    public void close() {
+	finalize();
+    }
+    
+    public Statement createStatement() throws SQLException
+    { 
+	return this.connection.createStatement();
+    }
+    
+    public boolean connectToDatabase(String database) {
+	if (this.connection != null) {
+	    try {
+		this.connection.close();
+	    } catch (Exception e) {}
+	}
+	this.connection = GS3SQLConnectionFactory.getConnection(database);
+	if (this.connection == null) {
+	    return false;
+	}
+	this.database = database;
+	return true;
+    }
+    
+    public boolean dropDatabase(String database) {
+	try {
+	    Statement statement = this.connection.createStatement();
+	    statement.execute("DROP DATABASE "+database+";");
+	    statement.close();
+	}
+	catch (SQLException ex){
+	    System.err.println(ex);
+	    return false;
+	}
+	return true;
+	
+    }
+
+    public boolean createDatabase(String database) {
+	try {
+	    String command = "CREATE DATABASE " + database;
+	    Statement statement = this.connection.createStatement();
+	    statement.execute(command);
+	    statement.close();
+	} catch (Exception e) {
+	    System.err.println(e);
+	    return false;
+	}
+	 
+	return true;
+    }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 12189)
@@ -0,0 +1,488 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Date;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import java.sql.Timestamp;
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.metadata.NamespaceFactory;
+import org.greenstone.gsdl3.gs3build.metadata.StructureIdentifierFactory;
+import org.greenstone.gsdl3.gs3build.metadata.GSDL3Namespace;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSHeader;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSNamespace;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+/**
+ *  Provide a base-line functionality for the <code>DocumentInterface</code>
+ *  class.
+ */
+
+public abstract class AbstractDocument implements DocumentInterface
+{
+  METSFileSet        fileSet;
+  METSDescriptiveSet metadata;
+  METSStructureSet   structureSet;
+  METSHeader         header;
+  DocumentID         id;
+  boolean            isModified;
+  StructureIdentifierFactory structureIdFactory;
+  java.sql.Timestamp firstDate;
+  java.sql.Timestamp indexDate;
+  java.sql.Timestamp modifiedDate;
+
+  /**
+   *  <p>Create a very vanilla document with a given document identifier.</p>
+   *  <p>Most commonly used in dealing with loading files using DocumentFactory
+   *  or similar.</p>
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   */
+  public AbstractDocument(DocumentID id)
+  { this.fileSet      = new METSFileSet();
+    this.metadata     = new METSDescriptiveSet();
+    this.header       = new METSHeader();
+    this.structureSet = new METSStructureSet();
+    this.id           = id;
+    this.structureIdFactory = new StructureIdentifierFactory();
+
+    java.util.Date thisDate = new java.util.Date();
+    this.firstDate = new java.sql.Timestamp(thisDate.getTime());
+    this.indexDate = new java.sql.Timestamp(thisDate.getTime());
+    this.modifiedDate = null; // as a signature that the modified date needs finding...
+  }
+  
+  /**
+   *  Create a basic document from a given <code>URL</code.  This is usually the form
+   *  called through the recognisers.
+   *
+   *  @param <code>URL</code> the URL of the first file in the document package
+   */
+  public AbstractDocument(URL url)
+  { this.fileSet      = new METSFileSet();
+    METSFile metsFile = this.fileSet.addFile(url);
+    this.metadata     = new METSDescriptiveSet();
+    this.header       = new METSHeader();
+    this.structureSet = new METSStructureSet();
+    this.id           = null;
+
+    java.util.Date thisDate = new java.util.Date();
+    this.firstDate = new java.sql.Timestamp(thisDate.getTime());
+    this.indexDate = new java.sql.Timestamp(thisDate.getTime());
+
+    METSStructure structure = new METSStructure("All", "All", "Whole Document");
+    METSDivision documentBody = new METSDivision("All", "All", "All", "Whole Document", "Document");
+    structure.addDivision(documentBody);
+    this.structureSet.addStructure(structure);
+    documentBody.addFileReference(metsFile.getID());
+    documentBody.addMetadataReference("DM1");
+  }
+
+  /**
+   *  Set the identified for the document.  Every document should have
+   *  a document number set on its accession, either through metadata
+   *  placed upon it internally or externally, or by assignment through
+   *  a <code>DocumentIDFactory</code>.  Each identifier should be
+   *  unique.
+   *
+   *  @param <code>DocumentID</code> the document identifier - in XML
+   *         terms, the gsdl3:id element.
+   */
+  public void setID(DocumentID id)
+  { this.id = id;
+    this.isModified = true;
+  }
+
+  /**
+   *  Get the document identifier - this should be unique to the document,
+   *  but care must be taken in the configuration of the collection to
+   *  ensure that this is the case.
+   *
+   *  @return <code>DocumentID</code> the identifer
+   */
+  public DocumentID getID()
+  { return this.id;
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  { return true;
+  }
+
+  /**
+   *  Get the date that this file was modified
+   */
+  public long getFilesDatestamp()
+  { return this.fileSet.getModifiedDatestamp(); 
+  }
+
+  /**
+   *  Get the date that this file was modified
+   */
+  public long getModifiedDatestamp()
+  { if (this.modifiedDate == null) {
+      this.setModifiedDatestamp();
+    }
+    return this.modifiedDate.getTime();
+  }
+
+  /**
+   *  Update/set the date of the most recent file modification
+   */
+  public void setModifiedDatestamp()
+  { this.modifiedDate = new java.sql.Timestamp(this.fileSet.getModifiedDatestamp()); 
+  }
+
+  /**
+   *  Get the date that this document was first indexed
+   */
+  public long getAccessionDate()
+  { return this.firstDate.getTime();
+  }
+
+  /**
+   *  Get the date that this document was last indexed
+   */
+  public long getLastIndexedDate()
+  { return this.indexDate.getTime();
+  }
+
+  /**
+   *  Set the last indexed date for this document;
+   */
+  public void setLastIndexedDate(long timestamp)
+  { this.indexDate = new java.sql.Timestamp(timestamp);
+  }
+
+  /**
+   *  Check if this document is in the database already.
+   *
+   *  In this simple implementation, the first file in the document's "default"
+   *  filegroup is taken to be the canonical file for this document - any document
+   *  of the same type with the same canonical file is taken to be a match.
+   *
+   *  @return <code>boolean</code> - if a matching document is found in the
+   *                                 database.
+   */
+  public String getDuplicateID(GS3SQLConnection connection)
+  { //String query = "SELECT * FROM document INNER JOIN filegroups ON document.docId=filegroups.docId WHERE DocType=\"" + HTML_DOCUMENT_TYPE + "\"";
+
+    // Query for documents using the same file...
+    String query = "SELECT DocID FROM files INNER JOIN filegroups ON files.FileGroupRef=filegroups.FileGroupRef WHERE (filegroups.FileGroupId=\"default\" AND files.FileLocation=\"" + this.fileSet.getFile(0).getLocation().toString() + "\")";
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+
+	List docs = new ArrayList();
+
+	if (results.first()) {
+	    do {
+		String value = results.getString("DocID");		
+		docs.add(value);
+	    } while (results.next());
+	    
+	    Iterator docIterator = docs.iterator();
+	    while (docIterator.hasNext()) {
+		String docId = docIterator.next().toString();
+		String innerQuery = "SELECT * FROM document WHERE DocID=\"" + docId + "\"";
+		results = statement.executeQuery(innerQuery);
+		if (results.first()) {
+		    String docType = results.getString("DocType");
+		    if (docType.equals(this.getDocumentType())) {
+			return docId;
+		    }
+		}
+	    }
+	}
+	statement.close();
+    }
+    catch (java.sql.SQLException sqlEx) {
+      System.err.println("AbstractDocument.getDuplicateID(): "+sqlEx);
+    }
+
+    return "";
+  }
+
+  /**
+   *  Obtain the METS header of this document
+   *
+   *  @return <code>METSHeader</code> the header
+   */
+  public METSHeader getHeader()
+  { return this.header;
+  }	
+
+  /**
+   *  Set the METS header for this document.
+   *
+   *  @param <code>METSHeader</code> the header
+   */
+  public void setHeader(METSHeader header)
+  { this.header = header;
+  }
+
+  /**
+   *  A simple implementation of the isDocumentType function that does <b>not</b> consider
+   *  inheritance - it <code>must</code> be extended as required.
+   */
+  public boolean isDocumentType(String type)
+  { return type.equals(this.getDocumentType());
+  }
+
+  public abstract String getDocumentType();
+  
+  public abstract String getDocumentText();
+
+  public abstract String getSectionText(String sectionId);
+
+  public String getMETSType()
+  { return "document";
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void addDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.addDocumentMetadata(label.getNamespace(), label.getLabel(), value);
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void setDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.setDocumentMetadata(label.getNamespace(), label.getLabel(), value);
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void addDocumentMetadata(String namespace, String label, String value)
+  { this.metadata.addMetadata("default", namespace, label, value);
+    this.isModified = true;
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void setDocumentMetadata(String namespace, String label, String value)
+  { this.metadata.setMetadata("default", namespace, label, value);
+    this.isModified = true;
+  }
+
+  /**
+   *  @see DocumentInterface:removeDocumentMetadata
+   */
+  public void removeDocumentMetadata(String namespace, String label)
+  { this.metadata.removeMetadata("default", namespace, label);
+    this.isModified = true;
+  }
+
+  /**
+   *
+   */
+  public void removeAllMetadata(String namespace, String label)
+  { this.metadata.removeAllMetadata(namespace, label);
+    this.isModified = true;
+  }
+
+  /**
+   *  Post metadata to a file in this document - the appropriate changes
+   *  should be made...
+   */
+  public void postFileMetadata(URL fileLocation, String namespace, String label, String value)
+  {
+    // First get the list of file groups, etc. that this file is associated with...
+    List fileGroups = this.fileSet.findGroups(fileLocation);
+
+    // Next, get the METS divisions associated with each file group...
+    List divisions = this.structureSet.findDivisionsForFiles(fileGroups);
+    
+    // Finally, post the metadata to the metadata group associated with each structure
+    Iterator divisionIter = divisions.iterator();
+    while (divisionIter.hasNext())
+    { METSDivision division = (METSDivision) divisionIter.next();
+
+      // get the open namespace for this division
+      METSNamespace namespaceMetadata = division.findNamespace(namespace, true, this.metadata);
+
+      // then post the metadata to it...
+      namespaceMetadata.addMetadata(label, value);
+    }
+  }
+
+  /**
+   *  Get the metadata structure of the document
+   *
+   *  @return <code>METSDescriptive</code> the metadata holder for the document.
+   */
+  public METSDescriptiveSet getDocumentMetadata()
+  { return this.metadata;
+  }
+
+  /**
+   *  Set the metadata structure for this document
+   *
+   *  @param <code>METSDescriptive</code> the new metadata holder for the document.
+   */
+  public void setDocumentMetadata(METSDescriptiveSet metadata)
+  { this.metadata = metadata;
+    this.isModified = true;
+  }
+
+  /**
+   *  Get the metadata structure of the document
+   *
+   *  @return <code>METSStructureSet</code> the metadata holder for the document.
+   */
+  public METSStructureSet getDocumentStructure()
+  { return this.structureSet;
+  }
+
+  public void setDocumentStructure(METSStructureSet structureSet)
+  { this.structureSet = structureSet;
+  }
+
+  /**
+   *  Get the values associated with a particular metadata value.
+   *
+   *  @param <code>String</code> the namespace to find the values in.
+   *  @param <code>String</code> the label to match to find the values.
+   *
+   *  @return <code>List</code> the values.
+   */
+  public List getDocumentMetadataItem(String namespace, String label)
+  { return this.metadata.getMetadata("default", namespace, label);
+  }
+
+  /**
+   *  Get the values associated with a particular metadata value.
+   *
+   *  @param <code>String</code> the namespace and label separated by a
+   *         colon.
+   *
+   *  @return <code>List</code> the values.
+   */
+  public List getDocumentMetadataItem(String namespaceLabel)
+  { String namespace, label;
+  
+    int colonAt = namespaceLabel.indexOf(':');
+    if (colonAt < 0)
+    { namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
+      label     = namespaceLabel;
+    }
+    else
+    { namespace = namespaceLabel.substring(0, colonAt);
+      label     = namespaceLabel.substring(colonAt+1);
+    }
+    return this.metadata.getMetadata("default", namespace, label);
+  }
+
+  /**
+   *  @see DocumentInterface:getDocumentFiles
+   */
+  public METSFileSet getDocumentFiles()
+  { return this.fileSet;
+  }
+
+  public void setDocumentFiles(METSFileSet fileSet)
+  { this.fileSet = fileSet;
+  }
+
+  /**
+   *  This is just a dummy function that does nothing at this level...
+   */
+  public org.w3c.dom.Document getDOMDocument()
+  { return null;
+  }
+  
+  /**
+   *  @see DocumentInterface:isMETSCompatible
+   */
+  public boolean isMETSCompatible()
+  { return true;
+  }
+
+  /**
+   *  Use a default document writer - this may be overridden for subclasses...
+   *
+   *  @see DocumentInterface:writeMETSObject
+   */
+  public DocumentWriter getMETSWriter()
+  { return new DocumentWriter();
+  }
+
+  /**
+   *  Use a default SQL document writer - this may be overridden for subclasses...
+   *
+   */
+  public DocumentSQLWriter getSQLWriter()
+  { return new DocumentSQLWriter();
+  }
+
+  /**
+   *  Obtain a document from the SQL database
+   */
+    public static AbstractDocument readSQL(GS3SQLConnection connection, ResultSet sqlResult)
+  { try {
+      DocumentID id = new DocumentID(sqlResult.getString("DocID"));
+      String     type = sqlResult.getString("docType");
+
+      // Use a factory method to create the correct subtype...
+      AbstractDocument document = DocumentFactory.createDocument(type, id);
+      // Append the document date information
+      document.indexDate    = sqlResult.getTimestamp("IndexedDate");
+      document.firstDate    = sqlResult.getTimestamp("AccessionDate");
+      document.modifiedDate = sqlResult.getTimestamp("ModifiedDate");
+
+      // Get the individual components of the document
+      METSFileSet fileSet = METSFileSet.readSQL(document, connection);
+      document.setDocumentFiles(fileSet);
+      METSDescriptiveSet descriptiveSet = METSDescriptiveSet.readSQL(document, connection);
+      document.setDocumentMetadata(descriptiveSet);
+      METSStructureSet structureSet = METSStructureSet.readSQL(document, connection);
+      document.setDocumentStructure(structureSet);
+
+      // indicate that the document is not currently modified
+      document.setChanged(false);
+      return document;
+    }
+    catch (SQLException sqlEx) {
+      System.err.println("AbstractDocument.readSQL(): Failure to load document: " + sqlEx);
+    }
+    return null;
+  }
+
+  /**
+   *  
+   */
+  public boolean isChanged()
+  { return this.isModified;
+  }
+
+  public void setChanged(boolean isModified)
+  { this.isModified = isModified;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 12189)
@@ -0,0 +1,95 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+/**
+   A simple base class for Recognisers
+   A new recogniser should set preferred_mime_type, filename_extensions, and document_type in the constructor. 
+   The DocumentFactory class needs to be modified to map the documentType to 
+   the corresponding document
+   And thats all for simple recognisers.
+   More complicated recognisers can just implement from scratch RecogniserInterface
+*/
+abstract public class AbstractRecogniser implements RecogniserInterface
+{
+
+    DocumentList list_repository = null;
+    String preferred_mime_type = "SET THIS IN THE CONCRETE CLASS"; 
+    ArrayList filename_extensions = null;
+    String document_type = "SET THIS IN THE CONCRETE CLASS";
+    
+    /** The constructor should set the variables 
+     * preferredMimeType, filename_extensions and documentType
+     */
+    public AbstractRecogniser() {
+
+    }
+    
+    /** configure by default does nothing */
+    public boolean configure(Element config_elem){
+	return true;
+    }
+    public void setListRepository(DocumentList docList) {
+	this.list_repository = docList;
+    }
+
+    protected boolean isAcceptedFilename(String filename) {
+	if (filename_extensions == null) {
+	    return false; // or should this be true??
+	}
+	for (int i=0; i<filename_extensions.size(); i++) {
+	    if (filename.endsWith((String)filename_extensions.get(i))) {
+		return true;
+	    }
+	}
+	return false;
+    }
+    
+    public boolean parseDocument(METSFile file)
+    {
+	String MIMEType = file.getMIMEType();
+	if (MIMEType == null || 
+	    MIMEType.equals(this.preferred_mime_type)) {
+	    URL location = file.getLocation();
+	    return this.parseDocument(location);
+	}
+	return false;
+    }
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if ( isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting "+document_type+" Document " + filename);
+		AbstractDocument doc = DocumentFactory.createDocument(document_type, url);
+		this.list_repository.addDocument(doc);
+		return true;
+	    }
+	} else {
+	    // Get Mime type remotely, and then proceed if required
+	    String mimeType = HTTPTools.getMIMEType(url);
+	    
+	    if (mimeType == this.preferred_mime_type) {
+		System.out.println("Posting "+document_type+" Document " + url.toString());
+		
+		AbstractDocument doc = DocumentFactory.createDocument(document_type, url);
+		this.list_repository.addDocument(doc);
+		return true;
+	    }
+	}
+	return false;
+    }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java	(revision 12189)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public class DocumentFactory {
+    
+    public static AbstractDocument createDocument(String type, DocumentID id) {
+	
+	if (type.equals(HTMLDocument.HTML_DOCUMENT_TYPE)) {
+	    return new HTMLDocument(id);
+	} else if (type.equals(TextDocument.TEXT_DOCUMENT_TYPE)) {
+	    return new TextDocument(id);
+	} else if (type.equals(JPEGDocument.JPEG_DOCUMENT_TYPE)) {
+	    return new JPEGDocument(id);
+	} else if (type.equals(IndexDocument.INDEX_DOCUMENT_TYPE)) {
+	    return new IndexDocument(id);
+	} else if (type.equals(MetadataDocument.METADATA_DOCUMENT_TYPE)) {
+	    return new MetadataDocument(id);
+	} else if (type.equals(METSDocument.METS_DOCUMENT_TYPE)) {
+	    return new METSDocument(id);
+	} else if (type.equals(GMLDocument.GML_DOCUMENT_TYPE)) {
+	    return new GMLDocument(id);
+	}
+	
+	/*
+	  else if (type.equals(ExtXMLDocument.EXTXML_DOCUMENT_TYPE))
+	  { return new ExtXMLDocument(id);
+	  }*/
+	return null;
+    }
+    public static AbstractDocument createDocument(String type, URL url) {
+	
+	if (type.equals(HTMLDocument.HTML_DOCUMENT_TYPE)) {
+	    return new HTMLDocument(url);
+	} else if (type.equals(TextDocument.TEXT_DOCUMENT_TYPE)) {
+	    return new TextDocument(url);
+	} else if (type.equals(JPEGDocument.JPEG_DOCUMENT_TYPE)) {
+	    return new JPEGDocument(url);
+	} else if (type.equals(IndexDocument.INDEX_DOCUMENT_TYPE)) {
+	    return new IndexDocument(url);
+	} else if (type.equals(MetadataDocument.METADATA_DOCUMENT_TYPE)) {
+	    return new MetadataDocument(url);
+	} else if (type.equals(METSDocument.METS_DOCUMENT_TYPE)) {
+	    return new METSDocument(url);
+	} else if (type.equals(GMLDocument.GML_DOCUMENT_TYPE)) {
+	    return new GMLDocument(url);
+	}
+	
+	/*
+	  else if (type.equals(ExtXMLDocument.EXTXML_DOCUMENT_TYPE))
+	  { return new ExtXMLDocument(id);
+	  }*/
+	return null;
+    }
+    
+  public static DocumentInterface readSQLDocument(GS3SQLConnection connection, DocumentID id)
+  {
+    String query = "SELECT * FROM document WHERE DocID=\""+id.toString()+"\";";
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+	
+	DocumentInterface di = null;
+	if (results.first()) {
+	    di = AbstractDocument.readSQL(connection, results);
+	}
+	statement.close();
+	return di;
+    }
+    catch (SQLException sqlEx) {
+      System.err.println("AbstractDocument.readSQLDocument():"+sqlEx);
+    }
+    return null;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java	(revision 12189)
@@ -0,0 +1,14 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public class DocumentID
+{
+	String id;
+
+	public DocumentID(String id)
+	{	this.id = id;
+	}
+
+	public String toString()
+	{	return this.id;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java	(revision 12189)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+
+public class DocumentIDFactory implements DocumentIDFactoryInterface
+{ private CollectionManager collection;
+
+  public DocumentIDFactory(CollectionManager collection)
+  { this.collection = collection;
+  }
+
+  public DocumentID getNewDocumentID(DocumentInterface document)
+  { DocumentID id;
+
+    String idString = this.collection.getNextDocumentID();
+    id = new DocumentID(idString);
+    
+    return id;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java	(revision 12189)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public interface DocumentIDFactoryInterface
+{
+	public DocumentID getNewDocumentID(DocumentInterface document);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 12189)
@@ -0,0 +1,282 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.List;
+import java.util.Map;
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public interface DocumentInterface
+{
+  /**
+   *  Get the immediate document type of the document.  The type may be derived/inherited
+   *  from other DocumentInterface types, but that cannot be checked here.
+   *
+   *  @return <code>String</code> the document type as a string
+   */
+  public String getDocumentType();
+  
+  /**
+   *  Check if this document is of a particular type, or is derived from a particular
+   *  type - i.e. inheritance is considered as well as the immediate type.
+   *
+   *  @param  <code>String</code> the type to check against.
+   *  @return <code>boolean</code> if the document matches the given type.
+   */
+  public boolean isDocumentType(String documentType);
+  
+  public void setID(DocumentID id);
+  public DocumentID getID();
+
+  /**
+   *  Get the METS type of the document.
+   *
+   *  @return <code>String</code> the document type as a string
+   */
+  public String getMETSType();
+  
+  public void setHeader(METSHeader header);
+  public METSHeader getHeader();
+
+  /**
+   *  Whether the document is indexed or not.
+   *
+   *  @return <code>boolean</code> <code>true</code> by default.
+   */
+  public boolean isIndexed();
+
+  /**
+   *  Get the latest date that a component file was modified...
+   */
+  public long getFilesDatestamp();
+
+  /**
+   *  Get the stored modified date for the files datestamp...
+   */
+  public long getModifiedDatestamp();
+
+  /**
+   *
+   */
+  public void setModifiedDatestamp();
+
+  /**
+   *  Get the date that this document was first indexed
+   */
+  public long getAccessionDate();
+
+  /**
+   *  Get the date that this document was last indexed
+   */
+  public long getLastIndexedDate();
+
+  /**
+   *  Set the last indexed date for this document;
+   */
+  public void setLastIndexedDate(long date);
+
+  /**
+   *  Check if the document matches another in the database
+   */
+  public String getDuplicateID(GS3SQLConnection connection);
+    
+  /**
+   *  The plain text of the document.
+   *
+   *  @return <code>String</code> This value may be <code>null</code>
+   *          for documents which have no textual component - e.g.
+   *          an image file
+   */
+  public String getDocumentText();
+
+  /**
+   *  The document as a dom object
+   */
+  public org.w3c.dom.Document getDOMDocument();
+
+  /**
+   *  Get the text of a section of this document
+   *
+   *  @return <code>String</code> the text as a string - for sections
+   *          that have no textual components, this value may be
+   *          <code>null</code>
+   */
+  public String getSectionText(String sectionId);
+	
+  /**
+   *  The metadata for the document, encoded as a map.
+   *
+   *  The returned Map may have the following properties:
+   *  1) It may be <code>null</code> - e.g. for plain text documents
+   *  2) Any value in the map may be a <code>List</code> object containing
+   *     more than one possible value - e.g. they key 'Author' may associated
+   *     with a List of several people.
+   *  3) Any value in the map may itself be a <code>Map</code> where the
+   *     encoding scheme permits groupings of hierarchical metadata items.
+   *  4) Where the namespace for a metadata item is known, its key will
+   *     include the namespace.  Hence Dublin Core 'Author' would be encoded
+   *     as "dc.Author" as the key.
+   *
+   *  @return <code>METSDescriptive</code> the metadata of the document
+   */
+  public METSDescriptiveSet getDocumentMetadata();
+  
+  /**
+   *  The metadata for a particular given upon the document, encoded as a <code>List</code>
+   *
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> the label of the values to obtain
+   *
+   *  @return <code>List</code> the metadata values
+   */
+  public List getDocumentMetadataItem(String namespace, String label);
+  
+  /**
+   *  The metadata for the document, encoded as a map.
+   *
+   *  @param <code>String</code> the namespace and label of the metadata, separated
+   *                             by a colon.  If no namespace is given, it is 
+   *                             defaulted
+   *
+   *  @return <code>List</code> the metadata values
+   */
+  public List getDocumentMetadataItem(String namespaceLabel);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.  This is a "cheap" form which doesn't have a separate
+   *  namespace element.  Either the data is to be stored in the "open"
+   *  Greenstone metadata namespace, or the namespace is encoded within
+   *  the label.
+   *
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures.  The label may commence
+   *                             with a namespace followed by a colon. 
+   *                             <p>e.g. "dc:title" for Dublin Core Title.</p>
+   *  @param <code>String</code> value of the metadata
+   */
+  public void addDocumentMetadata(MetadataLabel label, String value);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.  This is a "cheap" form which doesn't have a separate
+   *  namespace element.  Either the data is to be stored in the "open"
+   *  Greenstone metadata namespace, or the namespace is encoded within
+   *  the label.
+   *
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures.  The label may commence
+   *                             with a namespace followed by a colon. 
+   *                             <p>e.g. "dc:title" for Dublin Core Title.</p>
+   *  @param <code>String</code> value of the metadata
+   */
+  public void setDocumentMetadata(MetadataLabel label, String value);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void addDocumentMetadata(String namespace, String label, String value);
+
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void setDocumentMetadata(String namespace, String label, String value);
+
+  /**
+   *  Facilitate the removal of document metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   */
+  public void removeDocumentMetadata(String namespace, String label);
+  
+  /**
+   *  Facilitate the removal of all metadata of a given label from the document.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   */
+  public void removeAllMetadata(String namespace, String label);
+  
+  /**
+   *  Post metadata to a file in this document - the appropriate changes
+   *  should be made...
+   *
+   *  @param <code>URL</code> the location of the file...
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void postFileMetadata(URL fileLocation, String namespace, String label, String value);
+
+  /**
+   *  The constituent files for the document: each should be wrapped up
+   *  as a URL
+   *
+   *  @return <code>METSFileSet</code> the files which constitute the document
+   */
+  public METSFileSet getDocumentFiles();
+
+  /**
+   *  Set the constituent files for the document.
+   */
+  public void setDocumentFiles(METSFileSet fileSet);
+
+  /**
+   *  Obtain the structural information on the document
+   *
+   *  @return <code>METSStructureSet</code> the structural information on the
+   *          document.
+   */
+  public METSStructureSet getDocumentStructure();
+  
+  /**
+   *  Indicate whether the document can be stored in a native form in
+   *  a METS wrapper
+   *
+   *  @return <code>boolean</code> <code>true</code> if the document
+   *          is just to be wrapped in a METS shell.
+   */
+  public boolean isMETSCompatible();
+  
+  /**
+   *  Write the document into a METS wrapper - for many document types,
+   *  this will not actually be done by the document itself, but rather
+   *  by the default writer
+   */
+  public DocumentWriter getMETSWriter();
+  
+  /**
+   *  Get the writer to send the document to an SQL database
+   */
+  public DocumentSQLWriter getSQLWriter();
+
+  /**
+   *  Check if the document is changed or not
+   */
+  public boolean isChanged();
+
+  /**
+   *  Set the document modified state
+   */
+  public void setChanged(boolean isModified);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java	(revision 12189)
@@ -0,0 +1,535 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+
+import java.io.PrintWriter;
+import java.io.FileWriter;
+import java.io.File;
+import java.io.IOException;
+
+import java.net.URL;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLSelect;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLWhereItem;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLWhere;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLField;
+
+public class DocumentList
+{
+  DocumentInterface []       list;       // what is currently cached
+  int                        size;       // the maximum number in the cache
+  int                        used;       // the actual number in the cache
+  int                        count;      // the total number of known documents
+  DocumentIDFactoryInterface idFactory;  // A manufacturer of novel document IDs
+  GS3SQLConnection           connection; // used to query the SQL database
+
+  private static final int maxSize = 3;
+
+  public DocumentList(GS3SQLConnection connection)
+  { this.idFactory = null;
+    this.list  = new DocumentInterface[3];
+    this.used  = 0;
+    this.size  = 3;
+    this.count = 0;
+    this.connection = connection;
+  }
+
+  public DocumentList(DocumentIDFactoryInterface idFactory, GS3SQLConnection connection)
+  { this.idFactory = idFactory;
+    this.list  = new DocumentInterface[3];
+    this.used  = 0;
+    this.size  = 3;
+    this.count = 0;
+    this.connection = connection;
+  }
+
+  /**
+   *  Obtain the list of <code>DocumentID</code> objects representing the unique
+   *  document identifiers of documents that refer to the file given as a parameter.
+   *
+   *  @param <code>URL</code> the location of the file to match
+   *
+   *  @return <code>List</code> of <code>DocumentID</code> reference identifiers.
+   */
+  public List getDocumentIdsWithFile(URL fileLocation)
+  { List reply = new ArrayList();
+
+    GS3SQLSelect select = new GS3SQLSelect("files");
+    select.addField("*");
+    GS3SQLWhere  where  = new GS3SQLWhere(new GS3SQLWhereItem("FileLocation", "=", fileLocation.toString()));
+    select.setWhere(where);
+
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(select.toString());
+    
+	select = new GS3SQLSelect("filegroups");
+	select.addField("DocID");
+	select.setDistinct(true);
+      
+	where = new GS3SQLWhere();
+	where.setCondition(GS3SQLWhere.OR_CONDITION);
+	
+	GS3SQLWhereItem whereItem = null;
+
+      	results.first();
+	do {
+	  int fileGroupRef = results.getInt("FileGroupRef");
+	  whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(fileGroupRef), GS3SQLField.INTEGER_TYPE);
+	  where.add(whereItem);
+	}
+	while (results.next());
+	select.setWhere(where);
+	
+	results = statement.executeQuery(select.toString());
+	
+	results.first();
+	do {
+	  String docId = results.getString("DocID");
+	  reply.add(docId);
+	} while (results.next());
+	statement.close();
+    }
+    catch (SQLException sqlEx) {
+	System.err.println("DocumentList.getDocumentIdsWithFile(): "+sqlEx);
+    }
+    
+    return reply;
+  }
+
+  /**
+   *  Get a list of documents that match a given set of patterns,
+   *  within a given URL node.
+   *
+   *  @param <code>List</code> the list of patterns to match
+   *  @param <code>String</code> the partial URL of the root node under which o
+   *         match files.  NB: this is a <code>String</code> as the URL may be
+   *         incomplete and not properly match the strict requirements for <code>URL</code>
+   */
+  public List findDocumentIdsUsingFiles(List fileRefs, String withinNode)
+  {
+    StringBuffer queryBuffer = new StringBuffer("SELECT FileGroupRef FROM files WHERE FileLocation ");
+
+    Iterator files = fileRefs.iterator();
+    while (files.hasNext()) {
+      String file = files.next().toString();
+
+      if (withinNode != null) {
+	queryBuffer.append("REGEXP \"^");
+	queryBuffer.append(withinNode);
+	queryBuffer.append(".*");
+      }
+      else {
+	queryBuffer.append("REGEXP \"");
+      }
+      queryBuffer.append(file);
+      queryBuffer.append("\"");
+
+      if (files.hasNext()) {
+	queryBuffer.append(" OR ");
+      }
+    }
+    queryBuffer.append(";");
+    return this.findDocumentIdsUsingFileQuery(queryBuffer.toString());
+  }
+
+  public List findDocumentIdsUsingFile(String fileRef, String withinNode)
+  { String query = "SELECT FileGroupRef FROM files WHERE FileLocation REGEXP \"^"+withinNode+".*"+fileRef+"\";";
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+  /**
+   *  Return a list of document identifiers against a simple pattern.  No root node is given, so
+   *  any file matching the pattern given will be returned. USE WITH CAUTION!!!
+   *
+   *  @param <code>String</code> a fragment of file pathname to match against.
+   *
+   *  @return <code>List</code> of <code>DocumentID</code> objects.
+   */
+  public List findDocumentIdsUsingFile(String fileRef)
+  { 
+    // Get the simple list of file objects & their file group reference
+    String query = "SELECT FileGroupRef FROM files WHERE FileLocation REGEXP \"" + fileRef +"\";";
+
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+  public List findDocumentIdsUsingFileExact(String fileRef)
+  { 
+    // Get the simple list of file objects & their file group reference
+    String query = "SELECT FileGroupRef FROM files WHERE FileLocation=\"" + fileRef +"\";";
+
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+    private List findDocumentIdsUsingFileQuery(String query) {
+    
+	try {
+	
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+
+	if (!results.first()) {
+	    statement.close();
+	    return null;
+	}
+      
+      // get a list of group ids first and turn it into a query on filegroups
+      StringBuffer queryBuffer = new StringBuffer("SELECT * FROM filegroups WHERE ");
+      boolean first = true;
+      
+      do {
+	int groupRef = results.getInt("FileGroupRef");
+	
+	if (first) {
+	  first = false;
+	}
+	else {
+	  queryBuffer.append(" OR ");
+	}
+	queryBuffer.append("FileGroupRef=" + Integer.toString(groupRef));
+      } while (results.next());
+      queryBuffer.append(";");
+      
+      // make a holder for the actual file section identifiers
+      List divisions = new ArrayList();
+      
+      // expand (or, in fact, contract) through the document
+      // structures...recreating new filegroup queries as necessary
+      while (queryBuffer.length() > 0) {
+	results = statement.executeQuery(queryBuffer.toString());
+	
+	if (!results.first()) {
+	    statement.close();
+	    return null;
+	}
+	
+	queryBuffer = new StringBuffer();
+	do {
+	  String type = results.getString("ParentType");
+	  String parentRef = results.getString("ParentRef");
+	  if (type.equals(METSFileGroup.SECTION_PARENT)) {
+	    divisions.add(parentRef);
+	  }
+	  else {
+	    if (queryBuffer.length() > 0) {
+	      queryBuffer.append(" OR ");
+	    }
+	    queryBuffer.append("FileGroupRef=" + parentRef);
+	  }
+	} while (results.next());
+	
+	if (queryBuffer.length() > 0) {
+	  queryBuffer.insert(0, "SELECT * FROM filegroups WHERE ");
+	  queryBuffer.append(";");
+	}
+      }
+      
+      // ok, now find all the sections in which we are interested...
+      queryBuffer.setLength(0);
+      queryBuffer.append("SELECT DISTINCT DocID FROM filesection WHERE ");
+      Iterator iterator = divisions.iterator();
+      first = true;
+      
+      while (iterator.hasNext()) {
+	String ref = iterator.next().toString();
+	
+	if (first) {
+	  first = false;
+	}
+	else {
+	  queryBuffer.append(" OR ");
+	}
+	
+	queryBuffer.append("FileSectionRef="+ref);
+      }
+      queryBuffer.append(";");
+      
+      // execute the division query
+      results = statement.executeQuery(queryBuffer.toString());
+      
+      if (!results.first()) {
+	  statement.close();
+	  return null;
+      }
+
+      List reply = new ArrayList();
+      do {
+	reply.add(results.getString("DocID"));
+      } while (results.next());
+      
+      statement.close();
+      return reply;
+    }
+    catch (SQLException ex) {
+      System.err.println("DocumentList.findDocumentIdsUsingFileQuery()"+ ex);
+    }
+    return null;
+  }
+
+  /**
+   *  Cache a document into the cache, without writing it to the database.
+   *  Used directly by other parts of DocumentList when they know that the
+   *  document is in the database already, or they are going to write it
+   *  themselves...
+   *
+   *  @param <code>DocumentInterface</code> the document to cache
+   */
+  private void cacheDocument(DocumentInterface document)
+  { // increase cache size, etc. as necessary
+    if (this.used == this.size) {
+      if (this.size >= maxSize) {
+	for (int i = 0; i < this.size - 1; i ++) {
+	  this.list[i] = this.list[i+1];
+	}
+	this.used --;
+      }
+      else {
+	this.ensureSize((this.size * 2) > maxSize ? maxSize : (this.size * 2));
+      }
+    }
+    
+    // insert the document itself
+    this.list[this.used] = document;    
+  }
+
+  /**
+   *  Write the document into the document list (cache) and the database.
+   *
+   *  @param <code>DocumentInterface</code> the document itself
+   */
+  public void addDocument(DocumentInterface document)
+  { // initially, test if the document has a duplicate...
+    String duplicateDocID = document.getDuplicateID(this.connection);
+    if (duplicateDocID.length() > 0) {
+      System.out.println("Found duplicate document ");
+      return;
+    }
+
+    // first cache it...
+    this.cacheDocument(document);
+
+    // set the document identifier, if not already set
+    if (document.getID() == null) {
+      System.out.println("Posting new docuument ID");
+      DocumentID id = this.idFactory.getNewDocumentID(document);
+      document.setID(id);
+    }
+
+    // add to the database as well, if it is modified...
+    if (document.isChanged()) {
+      System.out.println("Document was changed");
+      document.getSQLWriter().writeDocument(document, this.connection);
+    }
+
+    // Remember that we've used one more item from the cache.
+    this.used ++;
+
+    // Note additional document
+    this.count ++;
+  }
+
+  /**
+   *  Note that an individual document is modified, and act accordingly
+   *
+   *  @param <code>DocumentInterface</code> the document
+   */
+  public void storeChangedDocument(DocumentInterface document)
+  { document.getSQLWriter().writeDocument(document, this.connection);
+  }
+
+  /**
+   *  Get an iterator across all the documents, not merely those in
+   *  the cache.  Note that this <code>Iterator</code> does <b>not</b>
+   *  support the <code>remove()</code> function, and will raise an
+   *  <code>UnsupportedOperationException</code> if you attempt to do
+   *  so.
+   *
+   *  @return <code>Iterator</code> the iterator across the documents.
+   */
+  public Iterator iterator()
+  { return new DocumentListIterator(connection);
+  }
+
+  /**
+   *  Get the nth member of the <b>cached</b> document list.
+   *
+   *  @deprecated
+   */
+  public DocumentInterface getDocument(int index)
+  { if (index < 0 || index >= this.used)
+    { return null;
+    }
+    return this.list[index];
+  }
+
+  /**
+   *  Simple "obtain a document" function
+   */
+  public DocumentInterface getDocument(DocumentID documentId)
+  {
+    DocumentInterface document = DocumentFactory.readSQLDocument(connection, documentId);
+    if (document != null) {
+      this.cacheDocument(document);
+    }
+    return document;
+  }
+
+  /**
+   *  Update timestamps on an entire document list - done at the beginning of a build cycle
+   *
+   *  @param <code>The date of the new build cycle</code>
+   */
+  public void updateTimestamps(long buildTimeStamp)
+  { Iterator documents = this.iterator();
+    int item = 0;
+
+    while (documents.hasNext())
+    { DocumentInterface document = (DocumentInterface) documents.next();
+      
+      long thisTimeStamp = document.getFilesDatestamp();
+      long lastTimeStamp = document.getModifiedDatestamp();
+
+      if (thisTimeStamp > lastTimeStamp) {
+	System.out.println("Updating timestamps " + thisTimeStamp + " " + lastTimeStamp);
+
+	DocumentSQLWriter.touchDocument(document.getID(), this.connection, buildTimeStamp, thisTimeStamp);
+      }
+    }
+  }
+
+  /**
+   *  A convenience method to map onto the old Vector source code...
+   */
+  protected void ensureSize(int size)
+  { DocumentInterface [] newList = new DocumentInterface[size];
+    System.arraycopy(this.list, 0, newList, 0, this.size);
+    this.list = newList;
+    this.size = size;
+  }
+
+  /**
+   *  Write the documents into a directory as METS/XML
+   */
+  public void writeDocuments(File directory)
+  { Iterator documents = this.iterator();
+    int item = 0;
+
+    while (documents.hasNext())
+    { DocumentInterface document = (DocumentInterface) documents.next();
+      try
+      { item ++;
+	File localFile = new File(directory, "Doc"+Integer.toString(item)+".xml");
+	FileWriter fileWriter = new FileWriter(localFile);
+	PrintWriter writer = new PrintWriter(fileWriter);
+	document.getMETSWriter().writeDocument(document, writer);
+	writer.close();
+	fileWriter.close();
+      }
+      catch (IOException io)
+      {	
+      }
+    }
+  }
+
+  public void writeSQLDocuments(GS3SQLConnection connection)
+  { for (int i = 0; i < this.used; i ++)
+    { this.list[i].getSQLWriter().writeDocument(this.list[i], connection);
+    }
+  }
+
+  public static DocumentList readSQLDocuments(GS3SQLConnection connection)
+  { DocumentList list = new DocumentList(connection);
+
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+    
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet documents = statement.executeQuery(select.toString());
+	if (documents.first()) { 
+	    do {
+		DocumentInterface document = AbstractDocument.readSQL(connection, documents);
+		list.addDocument(document);
+	    }
+	    while (documents.next());
+	}
+	statement.close();
+    }
+    catch (java.sql.SQLException ex) {
+	System.out.println("DocumentList.writeSQLDocuments(): "+ex);
+	return null;
+    }
+
+    return list;
+  }
+
+  public int getCount()
+  { return this.count;
+  }
+
+  public int size()
+  { return this.used;
+  }
+}
+
+class DocumentListIterator implements Iterator
+{
+  private boolean hasNext;
+  private Statement statement;
+  private ResultSet resultSet;
+  private GS3SQLConnection connection;
+
+  public DocumentListIterator(GS3SQLConnection connection)
+  {
+    this.connection = connection;
+
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+
+    try {
+	this.statement = connection.createStatement();
+      this.resultSet = statement.executeQuery(select.toString());
+      this.hasNext = this.resultSet.first();
+    } catch (SQLException ex) {
+      System.err.println("DocumentListIterator(): "+ex);
+      this.hasNext = false;
+    }
+  }
+
+  public boolean hasNext()
+  { return this.hasNext;
+  }
+
+  public Object next()
+  {
+    // get the 'next' document first
+    DocumentInterface document = AbstractDocument.readSQL(connection, this.resultSet);
+
+    // now actually step forward to the next item, so that we know if we have one!
+    try {
+      this.hasNext = this.resultSet.next();
+
+      if (!this.hasNext) {
+	  this.statement.close(); // be a good citizen & close used statement 
+      }
+    } catch (SQLException ex) {
+	System.err.println("DocumentList.iterator.next(): "+ex);
+      this.hasNext = false;
+    }
+    return document;
+  }
+  
+  public void remove() throws UnsupportedOperationException
+  { throw new UnsupportedOperationException("DocumentList does not support iterator removal of documents");
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java	(revision 12189)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.URL;
+
+public class DocumentLoader
+{
+  public static String getAsString(InputStream in)
+  { StringBuffer reply;
+    byte 	data[] = new byte[1024];
+    int		databytes;
+
+    reply	= new StringBuffer();
+    
+    try
+    {
+      do
+      { databytes = in.read(data);
+        if (databytes > 0)
+	{ reply.append(new String(data, 0, databytes));
+	}
+      } while (databytes >= 0);
+    }
+    catch (IOException io)
+    {
+    }
+    
+    return reply.toString();
+  }
+
+  public static String getAsString(File file)
+  { FileInputStream in;
+    String          reply = null;
+
+    try
+    { in	= new FileInputStream(file);
+      if (in == null)
+	{	return null;
+	}
+      reply = getAsString(in);
+      
+      in.close();
+    }
+    catch (IOException io)
+    { return null;
+    }
+    return reply;
+  }
+
+  public static String getAsString(URL url)
+  { if (url.getProtocol().equals("file"))
+    { File file = new File(url.getPath());
+      return getAsString(file);
+    }
+  
+    return null;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java	(revision 12189)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public class DocumentMetadata
+{
+	String name;
+	String value;
+
+	public DocumentMetadata(String name, String value)
+	{	this.name = name;
+		this.value = value;
+	}
+
+	public String getName()
+	{	return this.name;
+	}
+
+	public String getValue()
+	{	return this.value;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java	(revision 12189)
@@ -0,0 +1,98 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.PrintWriter;
+
+import java.sql.*;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class DocumentSQLWriter
+{
+  public DocumentSQLWriter()
+  {
+  }
+
+  public static boolean touchDocument(DocumentID docID, GS3SQLConnection connection, long touchTime, long modTime)
+  { GS3SQLUpdate update = new GS3SQLUpdate("document");
+    update.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", docID.toString())));
+    update.addDate("IndexedDate", new java.sql.Timestamp(touchTime));
+    update.addDate("ModifiedDate", new java.sql.Timestamp(modTime));
+    try {
+	Statement statement = connection.createStatement();
+	statement.execute(update.toString());
+	statement.close();
+    } catch (SQLException e) {
+	System.err.println("DocumentSQLWriter.touchDocument() Error: "+ e);
+	return false;
+    }
+    System.out.println(update.toString());
+    return true;
+  }
+
+  /**
+   * Just write the document to the database, assuming that it is not already there
+   */
+  public boolean writeDocument(DocumentInterface document, GS3SQLConnection connection)
+  {		
+    // TODO: output document in METS format, with Greenstone-specific metadata
+    // placed in a wrapper inside the descriptive metadata block
+
+    // put the document into the database
+    try {
+      if (document.getID() != null)
+      { //tag = XMLTools.addAttribute(tag, "OBJID", document.getID().toString());
+	GS3SQLSelect select = new GS3SQLSelect("document");
+	select.addField("*");
+	select.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", document.getID().toString())));
+	
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(select.toString());
+	if (!results.first()) { // empty result
+	  GS3SQLInsert insert = new GS3SQLInsert("document");
+	  insert.addValue("DocID", document.getID().toString());
+	  insert.addValue("DocType", document.getDocumentType());
+
+	  insert.addDate("AccessionDate", new java.sql.Timestamp(document.getAccessionDate()));
+	  insert.addDate("IndexedDate", new java.sql.Timestamp(document.getLastIndexedDate()));
+	  insert.addDate("ModifiedDate", new java.sql.Timestamp(document.getModifiedDatestamp()));
+
+	  statement.execute(insert.toString());
+	}
+	else {
+	  /* redundant code - not used... */
+	  //	  GS3SQLUpdate update = new GS3SQLUpdate("document");
+	  //	  update.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", document.getID().toString())));
+	  //	  connection.execute(update.toString());
+	}
+	statement.close();
+      }
+    } catch (SQLException ex) {
+	System.out.println("DocumentSQLWriter.writeDocument() Error:"+ex);
+    }
+    //    output.println(tag);
+    
+    // Write the structural metadata:
+    //   if in "Create METS" mode:
+    //     take all the documents as being in the one group, of the "physical"
+    //     document; particular document plugins may extend this behaviour,
+    //     which would suggest placing it in the AbstractDocument or similar
+    //     class?!
+    if (document.getDocumentMetadata() == null) {
+      System.out.println("Null metadata document");
+    }
+    document.getDocumentMetadata().writeSQL(document, connection);
+
+    document.getDocumentFiles().writeSQL(document, connection);
+
+    document.getDocumentStructure().writeSQL(document, connection);
+
+    // Write the body information
+    // TODO: only output the document if there is some
+    // content/formatting change between the original and
+    // what greenstone actually does; if the original is
+    // good, then place a marker saying "get the document
+    // "from the original file"
+    return true;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java	(revision 12189)
@@ -0,0 +1,65 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.PrintWriter;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+public class DocumentWriter
+{
+    public DocumentWriter()
+    {
+    }
+
+    public boolean writeDocument(DocumentInterface document, PrintWriter output)
+    {		
+	// TODO: output <xml header>
+	String xml_header ="<?xml ";
+	xml_header = XMLTools.addAttribute(xml_header, "version", "1.0");
+	xml_header = XMLTools.addAttribute(xml_header, "encoding", "UTF-8");
+	xml_header = XMLTools.addAttribute(xml_header, "standalone", "no");
+	xml_header = xml_header+"?>";
+	output.println(xml_header);
+
+	// TODO: output document in METS format, with Greenstone-specific metadata
+	// placed in a wrapper inside the descriptive metadata block
+	String tag = XMLTools.getOpenTag("mets", "mets");
+	if (document.getID() != null){
+	    tag = XMLTools.addAttribute(tag,"xmlns:mets","http://www.loc/gov/METS/");
+	    tag = XMLTools.addAttribute(tag,"xmlns:xsi","http://www.w3.org/2001/XMLSchema-instance");
+	    tag = XMLTools.addAttribute(tag,"xmlns:gsdl3","http://www.greenstone.org/gsdlmetadata/1.0/");
+	    tag = XMLTools.addAttribute(tag,"xmlns:xlink","http://www.w3.org/TR/xlink/");
+	    tag = XMLTools.addAttribute(tag,"xsi:schemaLocation","http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/mets.xsd http://www.greenstone.org/gsdlmetadata/1.0/ http://www.greenstone.org/gsdlmetadata/1.0/gsdl_metadata_schema.xsd");
+	    tag = XMLTools.addAttribute(tag, "OBJID", document.getID().toString());
+	}
+	output.println(tag);
+	
+	// Writing the structural metadata:
+	//   if in "Create METS" mode:
+	//     take all the documents as being in the one group, of the "physical"
+	//     document; particular document plugins may extend this behaviour,
+	//     which would suggest placing it in the AbstractDocument or similar
+	//     class?!
+	document.getDocumentMetadata().write(output);
+
+	// TODO: only output the document if there is some
+	// content/formatting change between the original and
+	// what greenstone actually does; if the original is
+	// good, then place a marker saying "get the document
+	// "from the original file"
+	if  (document.isMETSCompatible()){
+	    document.getDocumentFiles().write(output);
+	    document.getDocumentStructure().write(output);
+	}
+	else {
+	    DocumentWriter writer = document.getMETSWriter();
+	    
+	    // ensure that this call isn't recursive - 
+	    // TODO: This is a rather naive check and needs improving
+	    if (writer != this) {
+		writer.writeDocument(document, output);
+	    }
+	}
+	output.write("</mets:mets>\n");
+	return true;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java	(revision 12189)
@@ -0,0 +1,51 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class GMLDocument extends AbstractDocument
+{
+  public static final String GML_DOCUMENT_TYPE = "GML";
+
+  public GMLDocument(DocumentID id)
+  { super(id);
+  }
+
+  public GMLDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return GML_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  public METSDescriptiveSet getDocumentMetadata()
+  { return null;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java	(revision 12189)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+public class GMLRecogniser extends  AbstractRecogniser
+{
+    
+    public GMLRecogniser()   { 
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".gml");
+	this.document_type = GMLDocument.GML_DOCUMENT_TYPE;
+    }
+    
+    
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	if (filename != null) {
+	    if (isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting GML Document " + filename);
+		GMLDocument doc = new GMLDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		// System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java	(revision 12189)
@@ -0,0 +1,35 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class GS2METSRecogniser extends AbstractRecogniser
+{
+    public GS2METSRecogniser() 
+    {
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("docmets.xml");
+	this.document_type = METSDocument.METS_DOCUMENT_TYPE;
+    }
+    
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 12189)
@@ -0,0 +1,354 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.net.MalformedURLException;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.*;
+
+public class HTMLDocument extends AbstractDocument
+{
+  public static final String HTML_DOCUMENT_TYPE = "HTML";
+  Document domDocument;
+
+  public HTMLDocument(DocumentID id)
+  { super(id);
+    this.domDocument = null;
+  }
+
+  /**
+   *  Create the HTMLDocument from a given URL - the URL may in fact be a reference
+   *  to a local file.
+   *
+   *  @param <code>URL</code> The location from which to load the file
+   */
+  public HTMLDocument(URL url)
+  { super(url);
+
+    this.loadDocument(url);
+
+    this._extractDocumentFiles();
+    this._extractDocumentMetadata();
+    // extract the files before resolving the images
+    resolveImages();
+    HTMLDocumentTools docTools = new HTMLDocumentTools(this.domDocument);
+    docTools.setMetsDocument(this);
+    docTools.setUrl(this.fileSet.getFile(0).getLocation());
+    docTools.findSections();
+    METSStructure sectionStruct = docTools.getStructure();
+
+    if (sectionStruct.size() > 0) {
+      METSStructureSet structureSet = this.getDocumentStructure();
+      structureSet.addStructure(sectionStruct);
+    }
+  }
+
+  private void loadDocument(URL url)
+  {
+    //    HTMLDoc htmlDoc;
+
+    HTMLTidy tidyDoc;
+    if (url.getProtocol().equals("file"))
+    { tidyDoc = new HTMLTidy(new File(url.getPath()));
+    }
+    else
+    { //htmlDoc = new HTMLDoc(url);
+      tidyDoc = new HTMLTidy(url);
+    }
+    
+    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
+
+    this.domDocument = tidyDoc.getDocument(); 
+    Runtime.getRuntime().gc();
+    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
+ }
+
+  private void _extractDocumentMetadata()
+  { 
+    NodeList metadata = this.domDocument.getElementsByTagName("META");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String name = element.getAttribute("name");
+      if (name == null || name.length() == 0) {
+	continue;
+      }
+
+      String value = element.getAttribute("content");
+      if (value == null || value.length() == 0) {
+	value = name;
+      }
+
+      this.addDocumentMetadata(new MetadataLabel(name), value);
+    }
+    
+    NodeList titles = this.domDocument.getElementsByTagName("TITLE");
+    StringBuffer title = new StringBuffer();
+    for (int n = 0; n < titles.getLength(); n ++) {
+      Node node = titles.item(n);
+      Element element = (Element) node;
+
+      NodeList childNodes = node.getChildNodes();
+      for (int c = 0; c < childNodes.getLength(); c ++) {
+	Node child = childNodes.item(c);
+	if (child.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	  title.append(child.getNodeValue());
+	}
+      }
+    }
+
+    if (title.length() > 0)
+    { this.addDocumentMetadata(new MetadataLabel("Title"), title.toString());
+    }
+
+    /* Old HTMLDoc based parsing...
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
+    boolean inTitle = false;
+    title = new StringBuffer();
+
+    for (int e = 0; e < codedContent.size(); e ++)
+    { if (codedContent.elementAt(e) instanceof HTMLTag)
+      {	HTMLTag tag = (HTMLTag) codedContent.elementAt(e);
+
+        if (tag.tagName().equals("meta"))
+	{ // check that the name of the metadata item exists
+	  String name = tag.idValue("name");
+	  if (name == null || name.length() == 0) {
+	    continue;
+	  }
+
+	  // get the value, if it exists
+	  String value = tag.idValue("content");
+
+	  // if value does not exist, default it to being the same
+	  // as the name.
+	  if (value == null || value.length() > 0) {
+	    value = name;
+	  }
+	}
+	else 
+	if (tag.tagName().equals("title"))
+	{ inTitle = true;
+	}
+	else if (tag.tagName().equals("/title"))
+	{ inTitle = false;
+	}
+	// cut off when real body content appears - not a perfect
+        // implementation, just cheap & cheerful
+	else if (tag.tagName().equals("/head"))
+	{ break;
+	}
+	else if (tag.tagName().equals("body"))
+	{ break;
+	}
+      }
+      else if (inTitle == true)
+      { title.append(codedContent.elementAt(e).toString());
+      }
+    }
+      */
+  }
+
+  private void _extractDocumentFiles()
+  { URL homeUrl = this.fileSet.getFile(0).getLocation();
+
+    NodeList metadata = this.domDocument.getElementsByTagName("img");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String location = element.getAttribute("src");
+      if (location == null || location.length() == 0) {
+	System.out.println("No name");
+	continue;
+      }
+      try 
+      { // make the url for the image, and then add it to the document list of
+	// 
+	URL imgUrl = new URL(homeUrl, location);
+	METSFile file = this.fileSet.addFile(imgUrl);
+	this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+      }
+      catch (MalformedURLException ex)
+      { // TODO: report exception/failure to resolve...
+      }
+    }
+
+  
+    /**    
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
+    for (int e = 0; e < codedContent.size(); e ++)
+    { if (codedContent.elementAt(e) instanceof HTMLTag)
+      { HTMLTag tag = (HTMLTag) codedContent.elementAt(e);
+
+        if (tag.tagName().equals("img"))
+	{ String location = tag.idValue("src");
+
+	  try 
+	  { // make the url for the image, and then add it to the document list of
+	    // 
+	    URL imgUrl = new URL(homeUrl, location);
+	    METSFile file = this.fileSet.addFile(imgUrl);
+	    this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+	  }
+	  catch (MalformedURLException ex)
+	  { // TODO: report exception/failure to resolve...
+	  }
+	}
+      }
+    }
+    */
+  }
+
+  public String getDocumentType()
+  { return HTML_DOCUMENT_TYPE;
+  }
+
+    // I think this is used for single section documents, while getSectionText
+    // is used for sectioned documents
+    // we will use the domDocument rather than reading it in again to another HTMLDoc.
+  public String getDocumentText()
+  { 
+      XMLConverter converter = new XMLConverter();
+      return converter.getPrettyString(this.domDocument.getDocumentElement());
+      /*  HTMLDoc htmlDoc;
+    URL     url =(URL) this.fileSet.getFile(0).getLocation();
+
+    this.getSectionText("1");
+
+    if (url.getProtocol().equals("file"))
+    { htmlDoc = new HTMLDoc(url, url.getPath());
+    }
+    else
+    { htmlDoc = new HTMLDoc(url);
+    }
+    return htmlDoc.getContent();
+      */ 
+  }
+
+    private void resolveImages() {
+
+	// find the path of the url relative to the collection
+	URL full_path = this.fileSet.getFile(0).getLocation();
+       
+	String base_url;
+	if (full_path.getProtocol().equals("file")) {
+	    base_url = full_path.getPath();
+	    int import_pos = base_url.indexOf("import");
+	    base_url = base_url.substring(import_pos);
+	    base_url = "_httpcollection_/"+base_url;
+	} else {
+	    base_url = full_path.toString();
+	}
+
+	// need to take off the last part
+	base_url = base_url.substring(0, base_url.lastIndexOf("/")+1);
+
+	NodeList metadata = this.domDocument.getElementsByTagName("img");
+	for (int n = 0; n < metadata.getLength(); n ++) {
+	    Node node = metadata.item(n);
+	    Element element = (Element) node;
+	    
+	    String location = element.getAttribute("src");
+	    if (location != null && location.length() > 0 && isRelative(location)) {
+		// modify the source url
+		element.setAttribute("src", base_url+location);
+	    }
+	}
+    }
+
+    private boolean isRelative(String location) {
+
+	if ( location.startsWith("http:") || location.startsWith("file:")) {
+	    return false;
+	}
+	return true;
+    }
+    
+    public Document getDOMDocument()
+    {    
+      if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+      resolveImages();
+    }
+    return this.domDocument;
+  }
+
+  private XPointer getXPointer(METSDivision division)
+  { String fileId = division.getDefaultFileReference();
+
+    if (fileId == null) {
+      System.err.println("Unable to obtain file reference for section " + division.getLabel());
+      return null;
+    }
+
+    METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+    if (fileGroup == null) {
+      System.err.println("Unable to obtain file reference for filegroup " + fileId);
+      return null;
+    }
+
+    METSFile file = fileGroup.getFile(0);
+    if (file == null) {
+      System.err.println("Unable to obtain any files within filegroup " + fileId);
+      return null;
+    }
+
+    URL url = file.getLocation();
+    XPointer xpointer = XPointer.processXPointer(this.domDocument, url);
+
+    return xpointer;
+  }
+
+  private XPointer getXPointer(String sectionId)
+  { if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+      resolveImages();
+    }
+
+    METSDivision division = 
+      this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					      sectionId);
+    if (division == null) {
+      //      System.err.println("Unable to locate section " + sectionId);
+      return null; 
+    }
+
+    return this.getXPointer(division);
+  }
+  
+  public Node getSectionStartNode(METSDivision division)
+  { XPointer xpointer = this.getXPointer(division);
+    return xpointer.getStartNode();
+  }
+
+  public Node getSectionStartNode(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    return xpointer.getStartNode();
+  }
+
+  public String getSectionText(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    if (xpointer == null) {
+      return "";
+    }
+
+    // get the XML content of the xpointer...
+    return xpointer.toString();
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 12189)
@@ -0,0 +1,328 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.w3c.dom.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.AbstractStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFilePos;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileID;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
+import org.greenstone.gsdl3.gs3build.util.*;
+
+public class HTMLDocumentTools
+{
+  Document          document;
+  URL               documentUrl;
+  int               sectionLevel;
+  StringBuffer      sectionLabel;
+  List              sectionSeq;
+  HTMLSection       rootSection;
+  HTMLSection       currentSection;
+  int               sectionSeqNo;
+  METSStructure     metsStructure;
+  AbstractStructure currentMets;
+  AbstractDocument  metsDocument;
+
+  /**
+   *
+   */
+  class HTMLSection
+  { String startLocation;
+    String endLocation;
+    String label;
+    List   children;
+    HTMLSection parent;
+    int         seqNo;
+
+    public HTMLSection(String label, int seqNo)
+    { this.label = label;
+      this.seqNo = seqNo;
+      this.parent = null;
+      this.children = new ArrayList();
+    }
+    
+    public void setStartLocation(String location)
+    { this.startLocation = location;
+    }
+    
+    public void setEndLocation(String location)
+    { this.endLocation = location;
+    }
+
+    public void addChild(HTMLSection child)
+    { this.children.add(child);
+      child.parent = this;
+    }
+    
+    public HTMLSection getParent()
+    { return this.parent;
+    }
+
+    public String getLabel()
+    { return this.label;
+    }
+
+    public String getXPointer()
+    { StringBuffer xPointerBuffer = new StringBuffer();
+    
+      xPointerBuffer.append("xpointer(");
+      xPointerBuffer.append(this.startLocation);
+      xPointerBuffer.append("/range-to(");
+      xPointerBuffer.append(this.endLocation);
+      xPointerBuffer.append("))");
+      
+      return xPointerBuffer.toString();
+    }
+  }
+
+  public HTMLDocumentTools(Document document)
+  { this.document = document;
+    this.sectionLevel = 0;
+    this.sectionSeq   = new ArrayList();
+    this.sectionSeq.add(new Integer(0));
+    this.sectionLabel = new StringBuffer();
+    this.rootSection = new HTMLSection("", 0);
+    this.currentSection = this.rootSection;
+    this.metsStructure  = new METSStructure(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					    "Section", "Section");
+    this.currentMets    = this.metsStructure;
+    this.metsDocument   = null;
+  }
+
+  public void setMetsDocument(AbstractDocument document)
+  { this.metsDocument = document;
+  }
+
+  public void setUrl(URL url)
+  { this.documentUrl = url;
+  }
+
+  public Object findSections()
+  { this.findSections(document.getDocumentElement(), "/1");
+    return this.rootSection;
+  }
+
+  public void findSections(Node node, String location)
+  { // if at an element, then we aren't going to find section information,
+    // but we do need to recurse into the children to find the section markers
+    // there
+    if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      NodeList children = node.getChildNodes();
+      StringBuffer locationBuffer = new StringBuffer();
+      locationBuffer.append(location);
+      locationBuffer.append("/");
+
+      int elementNo = 1;
+      int commentNo = 1;
+      for (int c = 0; c < children.getLength(); c++)
+      {
+	locationBuffer.setLength(location.length()+1);
+	if (children.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  locationBuffer.append(Integer.toString(elementNo));
+	  elementNo ++;
+	}
+	else if (children.item(c).getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+	  locationBuffer.append("comment()[");
+	  locationBuffer.append(Integer.toString(commentNo));
+	  locationBuffer.append("]");
+	  commentNo ++;
+	}
+	this.findSections(children.item(c), locationBuffer.toString());
+      }
+    }
+    // if at a comment node, then one needs to dissect it to discover if
+    // any section markings are included...
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      String commentText = node.getNodeValue();
+      
+      // parse the node now...checking first for any 'section' items in
+      // it - this is a crude check...
+      if (commentText.indexOf("<Section>") >= 0 ||
+	  commentText.indexOf("</Section>") >= 0) {
+	// parse the section roughly using the primitive HTMLDoc form...      
+	HTMLDoc innerDoc     = HTMLDoc.fromString("dummy", commentText);
+	HTMLBlock innerHtml  = innerDoc.getCodedContent();
+	boolean   inMetadata = false;
+	StringBuffer metaValue = new StringBuffer();
+	String       metaLabel = null;
+
+	// Now just go through the content in sequence...
+	for (int e = 0; e < innerHtml.size(); e ++) {
+	  Object element = innerHtml.elementAt(e);
+
+	  if (element instanceof HTMLTag) {
+	    HTMLTag tag = (HTMLTag) element;
+
+	    if (tag.tagName().equals("section")) {
+	      // get the counters sorted out for the sections...
+	      this.sectionSeq.set(this.sectionLevel, new Integer(((Integer) this.sectionSeq.get(this.sectionLevel)).intValue() + 1));
+	      this.sectionLevel ++;
+	      if (sectionSeq.size() == this.sectionLevel) {
+		this.sectionSeq.add(new Integer(0));
+	      }
+	      else {
+		this.sectionSeq.set(this.sectionLevel, new Integer(0));
+	      }
+
+	      // create the label for the new section...
+	      this.sectionLabel.setLength(0);
+	      for (int s = 0; s < sectionLevel; s ++) {
+		if (s > 0) {
+		  this.sectionLabel.append(".");
+		}
+		this.sectionLabel.append(this.sectionSeq.get(s).toString());
+	      }
+
+	      // note the new section in a new data structure...
+	      HTMLSection newSection = new HTMLSection(this.sectionLabel.toString(), this.sectionSeqNo);
+
+	      newSection.setStartLocation(location);
+	      this.currentSection.addChild(newSection);
+	      this.currentSection = newSection;
+
+	      // do the same for mets...
+	      METSDivision newDivision = new METSDivision("DS" + Integer.toString(this.sectionSeqNo), Integer.toString(this.sectionSeqNo),
+							  this.sectionLabel.toString(), this.sectionLabel.toString(), "Section");
+	      this.currentMets.addDivision(newDivision);
+	      this.currentMets = newDivision;
+
+	      this.sectionSeqNo ++;
+	    }
+	    else if (tag.tagName().equals("/section")) {
+	      // cope with poorly formed metadata...
+	      if (inMetadata) {
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+
+	      // note the end position of the section
+	      this.currentSection.setEndLocation(location);
+
+	      // now post that section to the file list...
+	      this.postFilePtr(this.currentSection);
+
+	      // backtrack up the section tree...
+	      this.currentSection = this.currentSection.getParent();
+	      this.currentMets = this.currentMets.getParent();
+	      this.sectionLevel --;
+
+	      // trim the section label
+	      int cutTo = this.sectionLabel.toString().lastIndexOf('.');
+	      if (cutTo >= 0) {
+		this.sectionLabel.setLength(cutTo);
+	      }
+	      else {
+		this.sectionLabel.setLength(0);
+	      }
+
+	      // we can't be in metadata either...
+	      inMetadata = false;
+	    }
+	    else if (tag.tagName().equals("metadata")) {
+	      if (inMetadata) {
+		// post the metadata
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+	      metaValue.setLength(0);
+	      metaLabel = tag.idValue("name");
+	      inMetadata = (metaLabel != null);
+	    }
+	    else if (tag.tagName().equals("/metadata")) {
+	      // post the metadata
+	      this.postMetadata(metaLabel, metaValue.toString());
+	      inMetadata = false;
+	    }
+	  }
+	  else {
+	    if (inMetadata) {
+	      metaValue.append(element.toString());
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  private String makeXPointer(HTMLSection section)
+  { StringBuffer xPointerBuffer = new StringBuffer();
+
+    if (this.documentUrl != null)
+    { xPointerBuffer.append(this.documentUrl.toString());
+      xPointerBuffer.append("#");
+    }
+    xPointerBuffer.append(section.getXPointer());
+
+    return xPointerBuffer.toString();
+  }
+
+  private void postFilePtr(HTMLSection section)
+  { String xPointer = this.makeXPointer(section);
+
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    try {
+      URL url = new URL(xPointer);
+
+      METSFileSet fileSet = this.metsDocument.getDocumentFiles();
+
+      METSFilePos filePos = new METSFilePos(url);
+      METSFile    file = fileSet.createFile(filePos, "text/html");
+      
+      METSFileGroup fileGroup;
+      String fileRef = ((METSDivision) this.currentMets).getDefaultFileReference();
+
+      if (fileRef == null) {
+	fileGroup = fileSet.createGroup();
+	fileSet.addGroup(fileGroup);
+	((METSDivision) this.currentMets).addFileReference(fileGroup.getId());
+      }
+      else {
+	// simple call - the group should be at the 'root' level, immediately below
+	// the FileSet itself...
+	fileGroup = fileSet.getGroup(fileRef);
+      }
+      fileGroup.addFile(file);
+
+      
+    }
+    catch (java.net.MalformedURLException ex) {
+      System.err.println(ex + " " + xPointer);
+    }
+  }
+
+  private void postMetadata(String label, String value)
+  {
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    String metadataRef = ((METSDivision) this.currentMets).getDefaultMetadataReference();
+    METSDescriptive descriptive;
+    if (metadataRef == null) {
+      descriptive = this.metsDocument.getDocumentMetadata().createDescriptive(this.currentSection.getLabel());
+      ((METSDivision) this.currentMets).addMetadataReference(descriptive.getID());
+    }
+    else {
+      descriptive = this.metsDocument.getDocumentMetadata().getDescriptiveById(metadataRef);
+    }
+    descriptive.addMetadata(MetadataLabel.getDefaultNamespace(), label, value);
+  }
+
+  public METSStructure getStructure()
+  { return this.metsStructure;
+  }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java	(revision 12189)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class HTMLRecogniser extends AbstractRecogniser
+{
+    
+    public HTMLRecogniser()
+    { 
+	this.preferred_mime_type = "text/html";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".htm");
+	this.filename_extensions.add(".html");
+	this.document_type = HTMLDocument.HTML_DOCUMENT_TYPE;
+	
+    }
+    
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java	(revision 12189)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class IndexDocument extends AbstractDocument
+{
+  public static final String INDEX_DOCUMENT_TYPE = "GSINDEX";
+
+  public IndexDocument(DocumentID id)
+  { super(id);
+  }
+
+  public IndexDocument(URL url)
+  { super(url);
+  }
+  
+  public String getDocumentType()
+  { return INDEX_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  {
+    return true;
+  }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java	(revision 12189)
@@ -0,0 +1,52 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+public class IndexRecogniser extends AbstractRecogniser
+{
+    
+    public IndexRecogniser(DocumentList listRepository)
+    { 
+	this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("index.txt");
+	this.document_type = IndexDocument.INDEX_DOCUMENT_TYPE;
+    }
+    
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    String leafName;
+	    int leafAt = filename.lastIndexOf(File.separator);
+	    if (leafAt >= 0) {
+		leafName = filename.substring(leafAt+1);
+	    } else {
+		leafName = filename;
+	    }
+
+	    if (leafName.equals("index.txt")) {
+		
+		System.out.println("Posting Index Document " + filename);
+		IndexDocument doc = new IndexDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		//				System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java	(revision 12189)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+
+/**
+ *  Another fairly basic class for representing JPEG documents.
+ */
+
+public class JPEGDocument extends AbstractDocument
+{
+  public static final String JPEG_DOCUMENT_TYPE = "JPEG";
+
+  public JPEGDocument(DocumentID id)
+  { super(id);
+  }
+  
+  public JPEGDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return JPEG_DOCUMENT_TYPE;
+  }
+
+  public String getDocumentText()
+  { // Just a dummy function for JPEGs
+    return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { // Just a dummy function for JPEGs
+    return "";
+  }
+
+  public boolean isMETSCompatible()
+  { return true;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java	(revision 12189)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class JPEGRecogniser extends AbstractRecogniser
+{
+    
+    public JPEGRecogniser()
+    { 
+	this.preferred_mime_type = "image/jpeg";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".jpg");
+	this.filename_extensions.add(".jpeg");
+	this.document_type = JPEGDocument.JPEG_DOCUMENT_TYPE;
+    }
+    
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 12189)
@@ -0,0 +1,249 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+
+import java.net.URL;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTMLDoc;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+
+public class METSDocument extends AbstractDocument {
+
+    public static final String METS_DOCUMENT_TYPE = "METS";
+    Document domDocument;
+    public METSDocument(DocumentID id) {
+    	super(id);
+	//domDocument = null;
+    }
+
+    public METSDocument(URL url) {
+	super(url);
+	
+	if (url.getProtocol().equals("file")) {
+	    this._parseFile(new File(url.getPath()));
+	} else {
+	    System.err.println("Error: METSDocument must currently be a file");
+	    System.err.println("       URL = " + url.getPath());
+	}
+    }
+    
+    private void _parseFile(File file) {
+	
+	String parseFilePath;
+	try {
+	    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	    DocumentBuilder builder = factory.newDocumentBuilder();
+	    domDocument = builder.parse(file);
+
+	    int filePosition = file.getPath().indexOf("import/")+7;
+	    parseFilePath = file.getPath().substring(0, filePosition);
+	    // TODO: get all the types in the tree
+      
+	    // TODO: do a traverse, and thus cope with elements-within-elements if needsbe, but
+	    // this shouldn't happen except in a directly defined situation - actually doing the
+	    // parsing in part inside each node would work well provided one checked for a node
+	    // having already been done...
+	    System.out.println("Read");
+      
+	    // Get METSDmdSec metadata sections
+	    NodeList dmdSecs = domDocument.getElementsByTagName("mets:dmdSec");
+	    
+	    this.metadata = METSDescriptiveSet.parseXML(dmdSecs);
+
+	    // Get METSFileSec sections
+	    NodeList fileSecs = domDocument.getElementsByTagName("mets:fileSec");
+	    this.fileSet = METSFileSet.parseXML(fileSecs, fileSet, parseFilePath);
+	    // Get METSStructureMap Sections
+	    NodeList structMapSecs = domDocument.getElementsByTagName("mets:structMap");
+	    this.structureSet = METSStructureSet.parseXML(structMapSecs);
+
+	    //for (int g = 0; g < fileSecs.getLength(); g ++) {
+		//Schema schema = new Schema(schemas.item(s));
+		//this._parseFileSec(fileSecs.item(g));
+	    //}
+      
+	    // Get document metadata sections
+	    //NodeList dmdSecs = document.getElementsByTagName("mets:dmdSec");
+      
+	    //Schema schema = new Schema(schemas.item(s));
+	    //this.metadata = METSDescriptiveSet.parseXML(fileSecs);
+	} catch (FactoryConfigurationError e) {
+	    System.out.println(e);
+	} catch (ParserConfigurationException ex) {
+	    System.out.println(ex);
+	} catch (SAXException ex) {
+	    System.out.println(ex);
+	} catch (IOException ex) {
+	    System.out.println(ex);
+	}
+    }
+    /*
+    private void _parseFileSec(Node fileSec) {
+	// this is in effect a group without a sense of 'self'...
+	this._parseFileGroup((Element) fileSec, null);
+    }
+
+    private void _parseFileGroup(Element groupTag, METSFileGroup group) {
+	NodeList children = groupTag.getChildNodes();
+
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+
+	    System.out.println(children.item(c));
+    
+	    Element element = (Element) children.item(c);
+
+	    if (element.getNodeName().equals("mets:File")) {
+		if (group != null) {
+		    METSFile file = METSFile.parseXML(element, group);
+		} else {
+		    // TODO: error
+		}
+	    } else if (element.getNodeName().equals("mets:fileGrp")) {
+		// recurse
+		METSFileGroup childGroup = new METSFileGroup(element.getAttribute("ID"));
+
+		this._parseFileGroup(element, childGroup);
+		if (group != null) {
+		    group.addGroup(childGroup);
+		} else { 
+		    this.fileSet.addGroup(childGroup);
+		}
+	    }
+	}
+	}*/
+	
+    public String getDocumentType() {
+	return METS_DOCUMENT_TYPE;
+    }
+	
+    public String getDocumentText() {
+	// TODO: make this more than a dummy function!
+	HTMLDoc metsDoc;
+	URL url = (URL) this.fileSet.getFile(0).getLocation();
+
+	this.getSectionText("1");
+	
+	if (url.getProtocol().equals("file")) {
+	    metsDoc = new HTMLDoc(url,url.getPath());
+	} else {
+	    metsDoc = new HTMLDoc(url);
+	}
+	return metsDoc.getContent();
+    }
+    public Document getDOMDocument() 
+    {
+	if (domDocument == null) {
+	    URL url = (URL) this.fileSet.getFile(0).getLocation();
+	    try {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		domDocument = builder.parse(new File(url.getPath()));
+	    } catch (FactoryConfigurationError e) {
+		System.out.println(e);
+	    } catch (ParserConfigurationException ex) {
+		System.out.println(ex);
+	    } catch (SAXException ex) {
+		System.out.println(ex);
+	    } catch (IOException ex) {
+		System.out.println(ex);
+	    }
+	}
+	return domDocument;
+    }
+
+    private XPointer getXPointer(METSDivision division)
+    {
+	String fileId = division.getDefaultFileReference();
+
+	if (fileId == null) {
+	    System.err.println("Unable to obtain file reference for section " + division.getLabel());
+	    return null;
+	}
+	
+	METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+	if (fileGroup == null) {
+	    System.err.println("Unable to obtain file reference for filegroup " + fileId);
+	    return null;
+	}
+	METSFile file = fileGroup.getFile(0);
+	if (file == null) {
+	    System.err.println("Unable to obtain any files within filegroup " + fileId);
+	    return null;
+	}
+	URL url = file.getLocation();
+	XPointer xpointer = XPointer.processXPointer(domDocument, url);
+	
+	return xpointer;
+    }
+    
+    private XPointer getXPointer(String sectionId)
+    {
+	if (domDocument == null) {
+	    URL url = (URL) this.fileSet.getFile(0).getLocation();
+	    try {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		domDocument = builder.parse(new File(url.getPath()));
+	    } catch (FactoryConfigurationError e) {
+		System.out.println(e);
+	    } catch (ParserConfigurationException ex) {
+		System.out.println(ex);
+	    } catch (SAXException ex) {
+		System.out.println(ex);
+	    } catch (IOException ex) {
+		System.out.println(ex);       
+	    }
+	}
+	
+	METSDivision division = 
+	    this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, sectionId);
+	
+	if(division == null) {
+	    return null;
+	}
+      	return this.getXPointer(division);
+    }
+
+    public Node getSectionStartNode(METSDivision division)
+    {
+	XPointer xpointer = this.getXPointer(division);
+	return xpointer.getStartNode();
+    }
+
+    public Node getSectionStartNode(String sectionId)
+    {
+	XPointer xpointer = this.getXPointer(sectionId);
+	return xpointer.getStartNode();
+    }
+    
+    public String getSectionText(String sectionId) 
+    {
+	XPointer xpointer = this.getXPointer(sectionId);
+	
+	if (xpointer == null) {
+	    return "";
+	}
+
+	//get the XML content of the xpointer
+	return xpointer.toString();
+    }
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java	(revision 12189)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class MetadataDocument extends AbstractDocument
+{
+  public static final String METADATA_DOCUMENT_TYPE = "GSMETADATA";
+
+  public MetadataDocument(DocumentID id)
+  { super(id);
+  }
+
+  public MetadataDocument(URL url)
+  { super(url);
+  }
+  
+  public String getDocumentType()
+  { return METADATA_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  {
+    return false;
+  }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java	(revision 12189)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+//import java.io.*;
+import java.net.URL;
+import java.util.ArrayList;
+
+
+public class MetadataRecogniser extends AbstractRecogniser
+{
+
+    public MetadataRecogniser()
+    { 
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("metadata.xml");
+	this.document_type = MetadataDocument.METADATA_DOCUMENT_TYPE;
+    }
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if (isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting Metadata Document " + filename);
+		MetadataDocument doc = new MetadataDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		// System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 12189)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+/** Interface for Recogniser Objects
+ *
+ * A skeleton class is provided for simple Recognisers (AbstractRecogniser)
+ * 
+*/
+public interface RecogniserInterface
+{
+    public boolean configure(Element config_elem);
+    public void setListRepository(DocumentList docList);
+    public boolean parseDocument(METSFile fileReference);
+    public boolean parseDocument(URL fileReference);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java	(revision 12189)
@@ -0,0 +1,89 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.greenstone.gsdl3.gs3build.FileCrawlObserver;
+
+import java.io.File;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+public class RecogniserManager implements FileCrawlObserver
+{
+  RecogniserInterface list[];
+  int                 used;
+  int                 size;
+    DocumentList docList;
+  public RecogniserManager(DocumentList documentList)
+  { this.list = new RecogniserInterface[10];
+    this.used = 0;
+    this.size = 10;
+    this.docList = documentList;
+  }
+
+  /**
+   *  Add a recogniser to this manager
+   */
+    public RecogniserInterface addRecogniser(String documentType)
+    { 
+	ClassLoader loader = ClassLoader.getSystemClassLoader();
+	java.lang.Class thisClass = null;
+	try {
+	    thisClass = loader.loadClass("org.greenstone.gsdl3.gs3build.doctypes." + documentType + "Recogniser");
+	    Object instance = thisClass.newInstance();
+	    this.addRecogniser((RecogniserInterface) instance);
+	    return (RecogniserInterface)instance;
+	}
+	catch (ClassNotFoundException ex) {
+	    System.err.println(ex);
+	    return null;
+	}
+	catch (InstantiationException instEx) {
+	    System.err.println(instEx);
+	}
+	catch (IllegalAccessException accEx) {
+	    System.err.println(accEx);
+	}
+	return null;
+    }
+    
+  /**
+   *
+   */
+    public void addRecogniser(RecogniserInterface recogniser)
+    {  
+	recogniser.setListRepository(this.docList);
+	this.ensureCapacity(this.used + 1);
+  
+	this.list[this.used] = recogniser;
+	this.used ++;
+    }
+
+  public void processFile(URL url)
+  { boolean result;
+
+    for (int r = 0; r < this.used; r ++)
+    { if (list[r].parseDocument(url)) {
+        break;
+      }
+    }
+  }
+ 
+  public void processFile(File file)
+  { 
+      System.err.println("processing file "+file.getPath());
+      URL url = URLTools.getFileURL(file);
+    
+    if (url != null) 
+    { this.processFile(url);
+    }
+  }
+
+  private void ensureCapacity(int size)
+  { while (size >= this.size)
+    { RecogniserInterface newList [] = new RecogniserInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java	(revision 12189)
@@ -0,0 +1,45 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *  A pretty basic DocumentInterface class - just able to
+ *  provide unsophisticated access to the document as plain
+ *  text, and no attempt at metadata extraction
+ */
+
+public class TextDocument extends AbstractDocument
+{
+  public static final String TEXT_DOCUMENT_TYPE = "Text";
+
+  public TextDocument(DocumentID id)
+  { super(id);
+  }
+
+  public TextDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return TEXT_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return DocumentLoader.getAsString((URL) this.fileSet.getFile(0).getLocation());
+  }
+
+  /**
+   *  Text documents don't actually have more than one section (at present), so
+   *  this is a dummy function...
+   */
+  public String getSectionText(String sectionId)
+  { return null;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java	(revision 12189)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+
+public class TextRecogniser extends AbstractRecogniser
+{
+    
+    public TextRecogniser()
+    { 
+	this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".txt");
+	this.filename_extensions.add(".text");
+	this.document_type = TextDocument.TEXT_DOCUMENT_TYPE;
+    }
+    
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java	(revision 12189)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.expander;
+
+import java.io.File;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public interface ExpanderInterface
+{
+	/**
+	 *  Record the build manager to use for this expander - this is used
+	 *  to pass back any new directories into which expanded files are placed.
+	 */
+	public void configure(BuildManager manager);
+
+	/**
+	 *  Attempt to expand a file.  The returned result is <code>true</code> if the
+	 *  file was actually expanded.
+	 *
+	 *  @param <code>URL</code> the name of the file, or its URL
+	 *  @param <code>File</code> the directory to expand files into - the expander
+	 *         should create its own sub-directory to ensure that no later conflicts
+	 *         emerge.
+	 *
+	 *  @return <code>boolean</code> whether the file was expanded.
+	 */
+	public boolean expandFile(URL fileReference, File expansionDirectory);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java	(revision 12189)
@@ -0,0 +1,81 @@
+package org.greenstone.gsdl3.gs3build.expander;
+
+import java.io.*;
+import java.net.URL;
+import java.util.*;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public class ZipExpander implements ExpanderInterface
+{ BuildManager buildManager;
+
+	public ZipExpander()
+	{
+	}
+
+	public void configure(BuildManager manager)
+	{	this.buildManager = manager;
+	}
+
+	public boolean expandFile(URL url, java.io.File expansionDirectory)
+	{ if (url.toString().startsWith("file://")) 
+		{	String fileName = url.toString().substring(7);
+
+			if (fileName.endsWith(".zip"))
+		  {	System.out.println("Expanding Zip file " + fileName);
+
+				// determine which directory to use
+
+				// get the leaf name
+				String leafName;
+				int leafAt = fileName.lastIndexOf(System.getProperty("file.separator"));
+				
+				if (leafAt >= 0)
+				{ leafName = fileName.substring(leafAt+1);
+				}
+				else
+				{	leafName = fileName;
+				}
+
+			  int extensionAt = leafName.lastIndexOf('.');
+				if (extensionAt >= 0)
+				{	leafName = leafName.substring(0, extensionAt);
+				}
+
+				File targetDir = new File(expansionDirectory, leafName);
+				if (targetDir.exists())
+				{	// find alternative name
+					int  extension = 1;
+					File expandedTarget;
+
+					do
+					{ expandedTarget = new File(targetDir.toString() + Integer.toString(extension));
+					} while (expandedTarget.exists());
+
+					targetDir = expandedTarget;
+				}
+				if (!targetDir.mkdir())
+				{	// TODO: pass/log error - unable to make directory to
+					//       expand file.
+					return false;
+				}
+
+				// Call zip expansion
+				try
+				{
+					Runtime.getRuntime().exec("unzip -d "+targetDir.toString());
+				}
+				catch (IOException io)
+				{ // TODO: handle errors - report to log, etc.
+					return false;
+				}
+
+				// ensure that the build manager knows about the new directory,
+				// and will expand any content in it, etc. in turn
+				this.buildManager.addInputDirectory(targetDir.toString());
+				return true;
+			}
+		}
+		return false;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java	(revision 12189)
@@ -0,0 +1,15 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public interface ExtractorInterface
+{
+  public void configure(String outputDir);
+  public void configure(DocumentList list);
+  public void startPass(int passNumber);
+  public void extractDocument(DocumentID documentID, DocumentInterface document);
+  public void endPass(int passNumber);
+  public int getNumberOfPasses();
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java	(revision 12189)
@@ -0,0 +1,62 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.util.Iterator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public class ExtractorManager
+{
+  public static final String ACCUMULATE_MODE = "accumulate";
+
+  DocumentList          documents;
+  ExtractorInterface [] list;
+  int                   size;
+  int                   used;
+
+  public ExtractorManager(DocumentList documentList)
+  { this.list = new ExtractorInterface[10];
+    this.size = 10;
+    this.used = 0;
+    this.documents = documentList;
+  }
+
+  public void addExtractor(ExtractorInterface extractor)
+  { this.ensureSize(this.used + 1);
+    this.list[this.used] = extractor;
+    this.used ++;
+
+    extractor.configure(this.documents);
+  }
+
+  public void extractDocument(DocumentID docId, DocumentInterface document)
+  { for (int i = 0; i < this.used; i ++)
+    { this.list[i].extractDocument(docId, document);
+    }
+  }
+
+  public void extractDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.list[i].getNumberOfPasses(); p ++)
+      {	this.list[i].startPass(p);
+
+        Iterator iterator = documents.iterator();
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+	  this.list[i].extractDocument(document.getID(), document);
+	}
+	this.list[i].endPass(p);
+      }
+    }
+  }
+
+  public void ensureSize(int size)
+  { while (size >= this.size)
+    { ExtractorInterface newList [] = new ExtractorInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java	(revision 12189)
@@ -0,0 +1,167 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.doctypes.GMLDocument;
+
+public class GMLExtractor implements ExtractorInterface
+{
+  public static final String ACCUMULATE_MODE = "accumulate";
+
+  /**
+   *  An inner class to handle GML files
+   */
+  class GMLHandler extends DefaultHandler
+  { List   files;
+    String label;
+    StringBuffer value;
+    boolean inElement;
+    boolean accumulate;
+
+    GMLHandler()
+    { super();
+    
+      this.label = null;
+      this.value = null;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("FileName"))
+      {	this.value = new StringBuffer();
+      }
+      else if (localName.equals("FileSet"))
+      { this.files = new ArrayList();
+      }
+      else if (localName.equals("Description"))
+      {
+      }
+      else if (localName.equals("Metadata"))
+      { this.label = attributes.getValue("name");
+        this.value = new StringBuffer();
+
+	String mode = attributes.getValue("mode");
+	this.accumulate = mode.equals(ExtractorManager.ACCUMULATE_MODE);	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("FileName"))
+      {	String file = this.value.toString();
+        this.value = null;
+	this.files.add(file);
+      }
+      else if (localName.equals("FileSet"))
+      { // post the existing files item...
+      }
+      else if (localName.equals("Description"))
+      {
+      }
+      else if (localName.equals("Metadata"))
+      { GMLExtractor.postMetadata(this.files, this.label, this.value.toString());
+        this.value = null;
+	this.label = null;
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { if (this.value != null)
+      { String string = new String(c, start, length);
+        this.value.append(string);
+      }
+    }
+  }
+
+  /**
+   *  Construct of extractor
+   */
+  public GMLExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList documentList)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a GML document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  {	
+    if (document.getDocumentType().equals(GMLDocument.GML_DOCUMENT_TYPE))
+    { // Extract the content from the GML file
+      try {
+	XMLReader reader = XMLReaderFactory.createXMLReader();
+	GMLHandler handler = new GMLHandler();
+	reader.setContentHandler(handler);
+	reader.setErrorHandler(handler);
+	  
+	// A GML document consists of one file only - get it from the 'default'
+	// file group
+	FileReader fileReader = new FileReader(document.getDocumentFiles().getFile(0).toString());
+	reader.parse(new InputSource(fileReader));
+      }
+      catch (SAXException saxException)
+      { // TODO: log error
+      }
+      catch (java.io.FileNotFoundException fileException)
+      {
+      }
+      catch (java.io.IOException ioException)
+      {
+      }
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(List files, String value, String label)
+  {
+    
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java	(revision 12189)
@@ -0,0 +1,256 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.IndexDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+public class IndexExtractor implements ExtractorInterface
+{
+  class IndexHandlerException extends Exception
+  { public IndexHandlerException(String value)
+    { super(value);
+    }
+  }
+
+  /**
+   *  An inner class to handle GML files
+   */
+  class IndexHandler extends GS2TextFileHandler
+  { List    labels;
+    URL     base;
+
+    IndexHandler(String content, URL url, DocumentList documentList) throws IndexHandlerException
+    { 
+      super(content);
+
+      this.labels  = new ArrayList();
+      this.base    = url;
+
+      String parentDir;
+      int leaf = this.base.toString().lastIndexOf('/');
+      if (leaf >= 0) {
+	parentDir = this.base.toString().substring(0, leaf+1);
+      }
+      else {
+	parentDir = this.base.toString();
+      }
+
+      // get the first line
+      this.getLine();
+
+      if (!this.hasMore())
+      { throw new IndexHandlerException("No title line");
+      }
+      
+      // get the first totem - it should be "key:"
+      String entry = this.getEntry(true);
+      
+      // now get all the labels
+      while (this.hasMore())
+      { String label = this.getEntry(true);
+        if (label == null || label.length() == 0) {
+	  continue;
+	}
+	
+	this.labels.add(label);
+	System.out.println("Adding label: " + label);
+      }
+
+      while (this.hasMoreLines()) {
+	this.getLine();
+
+	// Get the file pattern itself
+	String filePattern = this.getEntry(true);
+	if (filePattern == null || filePattern.length() == 0) {
+	  continue;
+	}
+
+	// get a list of documents that match the file pattern
+	List documentIds = documentList.findDocumentIdsUsingFile(filePattern);
+	if (documentIds != null) {
+	  Iterator iterator = documentIds.iterator();
+	  while (iterator.hasNext()) {
+	    System.out.println("Matches file " + iterator.next().toString());
+	  }
+	}
+
+	// if no files match this data, then skip this row
+	// TODO: raise a quality error message
+	if (documentIds == null || documentIds.size() == 0) {
+	  continue;
+	}
+
+	// cache up the documents that match for speed improvements...
+	List documents = new ArrayList();
+	Iterator idIterator = documentIds.iterator();
+	while (idIterator.hasNext()) {
+	  String docIdString = idIterator.next().toString();
+	  System.out.println(docIdString);
+	  DocumentID docId   = new DocumentID(docIdString);
+	  DocumentInterface document = documentList.getDocument(docId);
+	  if (document != null) {
+	    documents.add(document);
+	  }
+	}
+
+	// Next, split the row into the separate metadata items
+	int entryNo = 0;
+	while (this.hasMore()) {
+	  String item = this.getEntry(true);
+	  if (item == null || item.length() == 0) {
+	    entryNo ++;
+	    continue;
+	  }
+
+	  String label = null;
+	  if (item.startsWith("<")) {
+	    int labelEnd = item.indexOf('>');
+	    if (labelEnd >= 0) {
+	      label = item.substring(1, labelEnd);
+
+	      item = item.substring(labelEnd+1, item.length());
+
+	      // eliminate any weird whitespace
+	      item.trim();
+
+	      // cope with a solo 'item' label with no following string
+	      if (item.length() == 0) {
+		entryNo ++;
+		continue;
+	      }
+	    }
+	    // starts with a bracketed label
+	  }
+	  else if (entryNo < this.labels.size()) {
+	    label = (String) this.labels.get(entryNo);
+	  }
+
+	  // Actually post the metadata -
+	  // it may be good to have cached all the documents that we're going to change
+	  // in order to minimise rewrites...
+	  if (label != null) {
+	    Iterator docIterator = documents.iterator();
+	    while (docIterator.hasNext()) {
+	      DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	      // Post to document
+	      // TODO: tailor this to posting documents to *sections* as required...
+	      document.addDocumentMetadata(new MetadataLabel(label), item);
+	      System.out.println("Posting " + label + "=" + item + " to " + parentDir + filePattern);
+	    }
+	  }
+	  entryNo ++;
+	}
+	
+	// write out the modified documents
+	// TODO: nicer/more generalised interface for this and related activity in
+	//       extractor manager (actually, enricher manager);
+	Iterator docIterator = documents.iterator();
+	while (docIterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	  System.out.println("Writing modified document " + document.getID());
+	  documentList.storeChangedDocument(document);
+	}
+      }
+    }
+
+  }
+
+  private DocumentList documentList;
+
+  /**
+   *  Construct of extractor
+   */
+  public IndexExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList list)
+  { this.documentList = list;
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a GML document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  { if (document.getDocumentType().equals(IndexDocument.INDEX_DOCUMENT_TYPE))
+    { // Extract the content from the index file
+      
+      // get the file
+      String documentText = 
+	DocumentLoader.getAsString(document.getDocumentFiles().getFile(0).getURL());
+	
+      if (documentText == null) {
+	System.err.println("IndexExtractor: Unable to load any content for " + document.getDocumentFiles().getFile(0).toString());
+	return;
+      }
+
+      try {
+	IndexHandler handler = new IndexHandler(documentText, document.getDocumentFiles().getFile(0).getURL(), this.documentList);
+      }
+      catch (IndexHandlerException ex) {
+      }
+      
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(String file, String value, String label)
+  {
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java	(revision 12189)
@@ -0,0 +1,268 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.apache.xerces.parsers.SAXParser;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.MetadataDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+public class MetaXMLExtractor implements ExtractorInterface
+{
+  /**
+   *  An inner class to handle Metadata files
+   */
+  class MetadataHandler extends DefaultHandler
+  { List         files;
+    String       label;
+    StringBuffer value;
+    URL          url;
+    boolean      inElement;
+    boolean      accumulate;
+    DocumentList documentList;
+    List         documentIds;
+    List         documents;
+
+    MetadataHandler(DocumentList documentList)
+    { super();
+    
+      this.label = null;
+      this.value = null;
+      this.documentList = documentList;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("FileName"))
+      {	this.value = new StringBuffer();
+      }
+      else if (localName.equals("FileSet"))
+      { this.files = new ArrayList();
+      }
+      else if (localName.equals("Description"))
+      { this.documentIds = this.documentList.findDocumentIdsUsingFiles(this.files, this.url.toString());	
+
+	if (documentIds != null && documentIds.size() > 0) {
+	  this.documents = new ArrayList();
+	  
+	  Iterator idIterator = documentIds.iterator();
+	  while (idIterator.hasNext()) {
+	    String docIdString = idIterator.next().toString();
+	    DocumentID docId   = new DocumentID(docIdString);
+	    DocumentInterface document = documentList.getDocument(docId);
+	    if (document != null) {
+	      documents.add(document);
+	    }
+	  }
+	}
+      }
+      else if (localName.equals("Metadata"))
+      { this.label = attributes.getValue("name");
+        this.value = new StringBuffer();
+
+	String mode = attributes.getValue("mode");
+	this.accumulate = mode.equals(ExtractorManager.ACCUMULATE_MODE);	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("FileName"))
+      {	String file = this.value.toString();
+        this.value = null;
+	this.files.add(file);
+      }
+      else if (localName.equals("FileSet"))
+      { // post the existing files item...
+      }
+      else if (localName.equals("Description"))
+      {
+	if (this.documents != null && documents.size() > 0) {
+	  // write out the modified documents
+	  // TODO: nicer/more generalised interface for this and related activity in
+	  //       extractor manager (actually, enricher manager);
+	  Iterator docIterator = documents.iterator();
+	  while (docIterator.hasNext()) {
+	    DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	    //  System.out.println("Writing modified document " + document.getID());
+	    documentList.storeChangedDocument(document);
+	  }
+	}
+      }
+      else if (localName.equals("Metadata"))
+      { MetaXMLExtractor.postMetadata(this.url, this.files, 
+				      this.label, this.value.toString(), 
+				      this.accumulate);
+	/*
+	if (documentIds != null) {
+	  Iterator iterator = documentIds.iterator();
+	  while (iterator.hasNext()) {
+	    System.out.println("Matches file " + iterator.next().toString());
+	  }
+	}
+	*/
+	
+	if (documentIds != null && documentIds.size() > 0) {
+	  Iterator docIterator = this.documents.iterator();
+	  while (docIterator.hasNext()) {
+	    DocumentInterface document = (DocumentInterface) docIterator.next();
+	    
+	    // Post to document
+	    // TODO: tailor this to posting documents to *sections* as required...
+	    if (accumulate) {
+ 	      document.addDocumentMetadata(new MetadataLabel(this.label), this.value.toString());
+	    }
+	    else {
+ 	      document.setDocumentMetadata(new MetadataLabel(this.label), this.value.toString());
+	    }
+	  }	
+	}
+
+	// flatten the metadata items again...
+        this.value = null;
+	this.label = null;
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { if (this.value != null)
+      { String string = new String(c, start, length);
+        this.value.append(string);
+      }
+    }
+
+    public void setUrl(URL url)
+    { this.url = url;
+    }
+  }
+
+  private DocumentList documentList;
+
+  /**
+   *  Construct of extractor
+   */
+  public MetaXMLExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList list)
+  { this.documentList = list;
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a metadata document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  { if (document.getDocumentType().equals(MetadataDocument.METADATA_DOCUMENT_TYPE))
+    { // Extract the content from the metadata file
+      URL url;
+
+      try {
+	SAXParser parser = new SAXParser();
+	MetadataHandler handler = new MetadataHandler(this.documentList);
+        /*
+	XMLReader reader = XMLReaderFactory.createXMLReader();
+	reader.setContentHandler(handler);
+	reader.setErrorHandler(handler);*/
+	parser.setContentHandler(handler);
+	  
+	// Get path of file; we cheat here by assuming that the url is a file - this
+	// really ought to be done better [TODO: fix to handle full paths & URLs]
+	url = document.getDocumentFiles().getFile(0).getURL();
+	String filePath = url.getPath();
+	handler.setUrl(new URL(url, "."));
+
+	// A metadata document consists of one file only - get it from the 'default'
+	// file group
+	/*
+	FileReader fileReader = new FileReader(filePath);
+	reader.parse(new InputSource(fileReader));
+	*/
+	parser.parse(filePath);
+      }
+      catch (SAXException saxException)
+      { // TODO: log error
+	System.err.println(saxException);
+      }
+      catch (java.io.FileNotFoundException fileException)
+      { System.err.println(fileException);
+      }
+      catch (java.io.IOException ioException)
+      { System.err.println(ioException);
+      }
+      /*      catch (java.net.MalformedURLException malEx) {
+	System.err.println("Unable to get parent of URL "+url.toString()+" in metadata extraction.");
+	return;
+      }
+      */
+
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(URL url, List files, String label, String value, boolean accumulate)
+  { String file;
+
+    Iterator fileIter = files.iterator();
+    while (fileIter.hasNext()) {
+      file = fileIter.next().toString();
+
+      System.out.println(url.toString() + " " + file + ": " + label + "=" + value);
+    }
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java	(revision 12189)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.util.GSXML;
+
+public abstract class AbstractIndexer implements IndexerInterface
+{ 
+  class CollectionIndex
+  { String level;
+    String name;
+    List fields;
+
+    public CollectionIndex(String indexName, Node node)
+    { this.fields = new ArrayList();
+      this.name = indexName;
+      this.level = IndexerManager.DEFAULT_LEVEL;
+    
+      NodeList children = node.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++) {
+	Node child = children.item(c);
+	
+	if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  String name = child.getNodeName();
+	  
+	  if (name.equals(GSXML.LEVEL_ELEM)) {
+	    String levelName = DOMUtils.getNodeChildText(children.item(c));
+	    this.level = levelName;
+	  }
+	  else if (name.equals(GSXML.FIELD_ELEM)) {
+	    String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	    this.fields.add(fieldName.toString());
+	  }
+	}
+      }
+      if (this.fields.size() == 0) {
+	this.fields.add(IndexerManager.DEFAULT_FIELD);
+      }
+    }
+
+    public void prepareIndexer(IndexerInterface indexer)
+      { //indexer.addIndex(this.name, this.level, this.fields.get(0).toString());
+    }
+  }
+
+
+  public boolean configure(org.w3c.dom.Node searchNode)
+  {
+    NodeList indexChildren = searchNode.getChildNodes();
+
+    // check for <index> tag children
+    for (int i = 0; i < indexChildren.getLength(); i ++) {
+      String childName = indexChildren.item(i).getNodeName();
+
+      if (childName.equals(GSXML.INDEX_ELEM))
+      { NamedNodeMap atts = indexChildren.item(i).getAttributes();
+	
+	Node attribute = atts.getNamedItem(GSXML.NAME_ATT);
+	String indexName = attribute.getNodeValue();
+	      
+	// create the local indexer representation
+	CollectionIndex index = new CollectionIndex(indexName, indexChildren.item(i));
+	     
+	// prepare it...
+	index.prepareIndexer(this);
+      }
+    }
+    return true;
+  }
+
+  public void tidyup()
+  {
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java	(revision 12189)
@@ -0,0 +1,13 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+public class IndexerFactory
+{ public static final IndexerInterface makeIndexer(String type, String name)
+  { if (type.equals(MGIndexer.MG_INDEX_TYPE)) 
+    { return new MGIndexer(name);
+    }
+    else if (type.equals(MGPPIndexer.MGPP_INDEX_TYPE)) {
+      return new MGPPIndexer(name);
+    }
+    return null;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java	(revision 12189)
@@ -0,0 +1,29 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public interface IndexerInterface
+{
+  public static final String DOCUMENT_LEVEL = "document";
+  public static final String SECTION_LEVEL = "section";
+  public static final String PARAGRAPH_LEVEL = "paragraph";
+
+  public static final String GS2_INDEX_LABEL = "Index";  
+
+  public String  getIndexType();
+  public String  getName();
+  public int     getNumberOfPasses();
+  // TODO: add a function call to provide appropriate XML for the service description
+  public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list);
+  public void    tidyup();
+
+  public boolean configure(String label, String value);
+  public boolean configure(org.w3c.dom.Node node);
+    //  public boolean addIndex(String name, String level, String field);
+
+  public boolean startPass(int passNumber);
+  public boolean indexDocument(DocumentID id, DocumentInterface document);
+  public boolean endPass(int passNumber);
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java	(revision 12189)
@@ -0,0 +1,100 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public class IndexerManager
+{
+  IndexerInterface indexers[];
+  int size;
+  int used;
+  DocumentList documentList;
+
+  public static final String outputDir = "outputDir";
+  public static final String DEFAULT_LEVEL = "document";
+  public static final String DEFAULT_FIELD = "text";
+
+  public IndexerManager(DocumentList documentList)
+  { this.indexers  = new IndexerInterface[10];
+    this.size      = 10;
+    this.used      = 0;
+    this.documentList = documentList;
+  }
+
+  public void addIndexer(IndexerInterface indexer)
+  { this.ensureCapacity(this.used + 1);
+
+    this.indexers[this.used] = indexer;
+    this.used ++;
+  }
+
+  public void indexDocument(DocumentID docId, DocumentInterface document)
+  { if (!document.isIndexed())
+      return;
+	  
+    for (int i = 0; i < this.used; i ++)
+    { this.indexers[i].indexDocument(docId, document);
+    }
+  }
+
+  public void indexDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.indexers[i].getNumberOfPasses(); p ++)
+      {	
+	if (!this.indexers[i].startPass(p)) {
+	  continue;
+	}
+
+	Iterator iterator = this.documentList.iterator();
+
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+
+	  if (document.isIndexed()) {
+	    if (!this.indexers[i].indexDocument(document.getID(), document)) {
+	      System.out.println("Ending document " + document.getID());
+	    }
+	    
+	    // note any changes made to this document...
+	    if (document.isChanged()) {
+	      //System.out.println("Writing document "+document.getID());
+	      this.documentList.storeChangedDocument(document);
+	    }
+	    /**
+	    if (p == 0) {
+	      System.out.println("Writing document "+document.getID());
+	    }
+	    */
+	  }
+	}
+	this.indexers[i].endPass(p);
+      }
+    }
+  }
+
+  public void ensureCapacity(int needSize)
+  { while (needSize >= this.size)
+    { int newSize = this.size * 2;
+      IndexerInterface newIndexers [] = new IndexerInterface[newSize];
+      System.arraycopy(indexers, 0, newIndexers, 0, this.used);
+      this.indexers = newIndexers;
+      this.size = newSize;
+    }
+  }
+
+    public boolean addServiceDescriptions(Element service_class_list) {
+	boolean success = true;
+	for (int i = 0; i < this.used; i ++) {
+	    if (!this.indexers[i].addServiceDescriptions(service_class_list)) {
+		success = false;
+	    }
+	}
+	return success;
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 12189)
@@ -0,0 +1,740 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.w3c.dom.*;
+
+import org.greenstone.mg.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.Processing;
+
+// for debug
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGIndexer //extends AbstractIndexer
+    implements IndexerInterface
+{ 
+    int          pass;
+    int          documentSeqNo;
+    int          sectionSeqNo;
+    boolean      firstDocument;
+    String       outputDirectory;
+    StringBuffer indexBuffer;
+    File         textDirectory;
+    File         indexDirectory;
+    String       indexStem;
+    String       textStem;
+    List         indexes;
+    String       overallName; 
+    
+    MGIndex current_index = null;
+    MGPassesWrapper mgPasses;
+    
+    static final char END_OF_DOCUMENT = (char) 2;
+    //static final char END_OF_SECTION  = (char) 3; // actually this is end of para for mg - don't use this yet
+    static final char END_OF_STREAM   = (char) 4;
+
+    public static final String MG_INDEX_TYPE = "mg";
+    public static final String INDEX_FILE_STEM = "index";
+    
+    class MGIndex
+    { 
+	String name=null;
+	String level=null;
+	List fields=null;
+	boolean error = false;// assume built until we get an error
+	
+	public MGIndex(Element index_element) {
+	    
+	    this.fields = new ArrayList();
+	    this.name = index_element.getAttribute(GSXML.NAME_ATT);
+	    if (this.name.equals("")) {
+		// TODO make this dynamic
+		this.name = "xx";
+	    }
+	    NodeList children = index_element.getChildNodes();
+	    for (int c = 0; c < children.getLength(); c ++) {
+		Node child = children.item(c);
+		
+		if (child.getNodeType() == Node.ELEMENT_NODE) {
+		    String name = child.getNodeName();
+	  
+		    if (name.equals(GSXML.LEVEL_ELEM)) {
+			this.level = DOMUtils.getNodeChildText(children.item(c));
+		    }
+		    else if (name.equals(GSXML.FIELD_ELEM)) {
+			String fieldName = DOMUtils.getNodeChildText(children.item(c));
+			this.fields.add(fieldName);
+		    }
+		}
+	    }
+	}
+
+	public MGIndex(String name, String level, List fields)
+	{ 
+	    this.name  = name;
+	    this.level = level;
+	    this.fields = fields;
+	}
+	
+	// old gs2 style config - can we get rid of this??
+	public MGIndex(String indexLabel)
+	{ 
+	    int colonAt = indexLabel.indexOf(':');
+	    
+	    if (colonAt < 0) {
+		System.err.println("MGIndex(): invalid index specification: "+indexLabel);
+		return;
+	    }
+	    String field_string = indexLabel.substring(colonAt+1);
+	    String [] field_list = field_string.split(",");
+	    this.fields = new ArrayList();
+	    for (int i=0; i<field_list.length; i++) {
+		this.fields.add(field_list[i]);
+	    }
+	    this.level = indexLabel.substring(0, colonAt);
+	    //createIndexName
+	    StringBuffer new_name = new StringBuffer();
+	    new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+	    int c, w;
+	    w = 0;
+	    c = 0;
+	    while (c < field_string.length() && w < 2) {
+		char ch = field_string.charAt(c);
+		
+		ch = Character.toLowerCase(ch);
+		if (Character.isLetter(ch)) {
+		    if (ch != 'a' && ch != 'e' && ch != 'i' &&
+			ch != 'o' && ch != 'u') {
+			new_name.append(ch);
+			w++;
+		    }
+		}
+		c ++;
+	    }
+	    this.name = new_name.toString();
+	}
+
+	
+	public String getLevel()
+	{ 
+	    return this.level;
+	}
+
+	public List getFields()
+	{ 
+	    return this.fields;
+	}
+
+	public String getName()
+	{ 
+// 	    if (this.name==null || this.name.equals("")) {
+// 		createIndexName();
+// 	    }
+	    return this.name;
+	}
+
+	public boolean hasError() {
+	    return this.error;
+	}
+	public void setError(boolean b) {
+	    this.error = b;
+	}
+
+// 	private void createIndexName() {
+// 	    StringBuffer new_name = new StringBuffer();
+// 	    new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+
+// 	    int c, w;
+// 	    w = 0;
+// 	    c = 0;
+// 	    String [] fields_concat = this.fields.toArray
+// 	    while (c < this.field.length() && w < 2) {
+// 		char ch = this.field.charAt(c);
+
+// 		ch = Character.toLowerCase(ch);
+// 		if (Character.isLetter(ch)) {
+// 		    if (ch != 'a' && ch != 'e' && ch != 'i' &&
+// 			ch != 'o' && ch != 'u') {
+// 			new_name.append(ch);
+// 			w++;
+// 		    }
+// 		}
+// 		c ++;
+// 	    }
+// 	    this.name = new_name.toString();
+// 	}
+ 
+
+    } // MGIndex
+
+    public MGIndexer(String name)
+    { 
+	this.indexes = new ArrayList();
+	this.overallName = name; 
+    }
+
+    public String getIndexType()
+    { 
+	return MG_INDEX_TYPE;
+    } 
+
+    public String getName()
+    { 
+	return this.overallName; 
+    }
+    
+    public boolean configure(Node search_node)
+    {
+	NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+	
+	// add a text 'index' - we should be able to turn this off in the config file? actually mg needs a text index
+	ArrayList list = new ArrayList();
+	list.add("text");
+	MGIndex index = new MGIndex("text", "section", list);
+	indexes.add(index);
+	for (int i = 0; i < index_children.getLength(); i ++) {
+	    Element index_elem = (Element)index_children.item(i);
+	    index = new MGIndex(index_elem);
+	    if (index.getName() != null && index.getLevel() != null && index.getFields()!= null) {
+		
+		indexes.add(index);
+	    } else {
+		System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+	    }
+	}
+	// TODO make sure all index names are unique
+	return true;
+    }
+
+    /**
+     *  The output directory should be (collection)/building/text/ for
+     *  normal Greenstone builds.
+     *
+     *  @param <code>String</code> the label to configure
+     *  @param <code>String</code> the value...
+     */ 
+    public boolean configure(String label, String value)
+    { 
+	if (label.equals(IndexerManager.outputDir)) {
+	    this.outputDirectory = value;
+	    this.pass = 0;
+    
+	    // attempt to ensure that the text subdirectory exists
+	    this.textDirectory = new File(outputDirectory, "text");
+	    if (!textDirectory.exists()) {
+		if (!textDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!textDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+
+	    // Sign to the user which mg directory is being used...
+	    System.out.println("Output MG directory is " + this.textStem);
+	}
+	else if (label.equals(IndexerInterface.GS2_INDEX_LABEL)) {
+	    this.indexes.add(new MGIndex(value));
+	}
+
+	return true;
+    }
+
+
+    private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	    // try doing this for all index types
+	    // actually we should only need to do this once ????
+	    if (this.pass == 0) {
+	    //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+		METSDivision division = (METSDivision) structure;
+
+		// get the division metadata block
+		METSDescriptive descriptive;
+		String metadataId = division.getDefaultMetadataReference();
+		if (metadataId == null) {
+		    descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		    division.addMetadataReference(descriptive.getID());
+		}
+		else {
+		    // Get the descriptive item...
+		    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		}
+	    
+		descriptive.addMetadata("gsdl3", "mgseqno", this.overallName + "." + Integer.toString(this.sectionSeqNo));
+	    
+		metsDoc.setChanged(true);
+		//metsDoc.setModified(true);
+		//	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	    } // section level
+	
+	    // append an 'end of section' marker
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+	
+	    // for document-level indexes, always append an 'end of document' tag at the
+	    // end of the document for each section.  Otherwise, each section is followed
+	    // by an end of document character.  This ensures that all indexes use the
+	    // same document numbering...
+	    if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
+		extraBuffer.append(END_OF_DOCUMENT);
+	    }
+	    else {
+		textBuffer.append(END_OF_DOCUMENT);
+		this.documentSeqNo ++;
+	    }
+	    
+	    // produce the body here for metadata output of divisions - in the case of
+	    // text output, that will happen below...
+	    
+	    if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+		// if there is only text, don't do this
+		METSDescriptive descriptive;
+	
+		METSDivision division = (METSDivision) structure;
+
+		String metadataId = division.getDefaultMetadataReference();
+		// are there other metadata refs to get??
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		if (descriptive != null) {
+		    for (int i=0; i<fields.size(); i++) {
+			String field = (String)fields.get(i);
+			if (field.equals("text")) {
+			    continue;
+			}
+			List values = descriptive.getMetadata(namespace, field);
+			if (values != null) {	
+			    Iterator valueIter = values.iterator();
+			    while (valueIter.hasNext()) {
+				String value = valueIter.next().toString();
+				textBuffer.append(value);
+				textBuffer.append(" ");
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	
+	// go through our children as required...
+	Iterator children = structure.getChildIterator();
+	Node startNode;
+	boolean index_text = fields.contains("text");
+	while (children.hasNext()) {
+	    AbstractStructure child = (AbstractStructure) children.next();
+      
+	    // get xpointer for child 
+	    // get start position node
+	    if (metsDoc.getDocumentType() == "METS"){
+		startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+	    } else {
+		startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+	    }
+	    
+	    // while this node isn't the child's start node, produce the 
+	    // HTML node text, if in text field mode...
+	    if (index_text) {
+		while (node != startNode) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+	    
+	    // recurse to child
+	    node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+	} // while next child
+
+	// close a document - the actual closing \B will be done by the main
+	// loop, so only a required \C is printed here...
+	// why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
+	if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+	    if (index_text) {
+		while (node != null) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+      
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+      
+	}
+	return node;
+    }
+
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+    {
+	//  String name, String namespace, String field)
+	StringBuffer extraBuffer = new StringBuffer();
+	Node node = document.getDocumentElement();
+	StringBuffer textBuffer = new StringBuffer();
+	
+	this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
+	textBuffer.append(extraBuffer.toString());
+	return textBuffer.toString();
+    }
+
+    /**
+     *  Index a single document; the document interface can be used to extract individual
+     *  metadata items etc. as required or desired and index those instead or as well as
+     *  the body text of the document.
+     */
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+	
+	if (!this.firstDocument) {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    
+	}
+
+	String docText = null;
+	// set the mgseqno if first pass
+	if (this.pass == 0) {
+	    document.removeAllMetadata("gsdl3", "mgseqno");
+	    document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+	}
+
+	this.sectionSeqNo ++;
+	
+	//long start = System.currentTimeMillis();
+	Document domDocument = document.getDOMDocument();
+	if (domDocument != null) {
+	    System.err.println("dom doc is not null");
+	    METSStructure sections = document.getDocumentStructure().getStructure("Section");
+	    if (sections != null) {
+		System.err.println("sections are not null");
+		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+		//	System.out.println(docText);
+	    }
+	}
+	//long finish = System.currentTimeMillis();
+	//System.err.println("dom doc = "+ Long.toString(finish-start));
+	//start = System.currentTimeMillis();
+	if (docText == null) {
+	    System.err.println("dom doc or sections was null - asking for doc text");
+	    StringBuffer doc_text_buffer = new StringBuffer();
+	    List fields = this.current_index.getFields();
+	    for (int i=0; i<fields.size(); i++) {
+		String field = (String)fields.get(i);
+		if (field.equals("text")) {
+		    doc_text_buffer.append(document.getDocumentText());
+		}  else {
+		    // its a metadata - do namespace properly!!
+		    List values = document.getDocumentMetadataItem("gsdl3", field);
+		    if (values != null) {
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    doc_text_buffer.append(value);
+			}
+		    }
+		}
+	    } // for each field
+	    docText = doc_text_buffer.toString();
+	    sectionSeqNo ++;
+	}
+	//finish = System.currentTimeMillis();
+	//System.err.println("whole doc = "+ Long.toString(finish-start));
+	
+	this.indexBuffer.append(docText);
+	// remember that we're not on the first document, 
+	this.firstDocument = false;
+	this.documentSeqNo ++;
+
+	return true;
+    }
+
+    /**
+     *  Initialise the pass: open required files, check status
+     */
+    public boolean startPass(int passNumber) {
+    	
+	
+	this.pass = passNumber;
+	this.firstDocument = true;
+	this.documentSeqNo = 1;
+	this.sectionSeqNo  = 1;
+	
+	this.mgPasses = new MGPassesWrapper();
+	this.indexBuffer = new StringBuffer();
+	int indexNo = this.pass/2;
+	this.current_index = null;
+	
+	this.current_index = (MGIndex) this.indexes.get(indexNo);
+	if (this.current_index.hasError()) {
+	    // an error has already occurred for this index, don't continue
+	    System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+	    return false;
+	}
+	// attempt to ensure that the text/index subdirectory exists
+	this.indexDirectory = new File(outputDirectory, current_index.getName());
+	if (!indexDirectory.exists()) {
+	    if (!indexDirectory.mkdir()) {
+		return false;
+	    }
+	}
+	else if (!indexDirectory.isDirectory()) {
+	    return false;
+	}
+
+	this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+	if (this.pass == 0) {
+	    // first pass, also set up the textStem
+	    this.textDirectory = this.indexDirectory;
+	    this.textStem = this.indexStem;
+	}
+	mgPasses.setFileName(this.indexStem);
+	if (!Misc.isWindows()) {
+	    mgPasses.setBasePath("/");
+	}
+	int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	
+	mgPasses.setBufferSize(100000);
+ 	
+	switch (mgPass) {
+	case 0:
+	    // -b 100000 -T1
+	    mgPasses.addPass(MGPassesWrapper.TEXT_PASS_1);
+	    
+	    
+	    break;
+	    
+	case 1:
+	    // -b 100000 -T2
+	    mgPasses.addPass(MGPassesWrapper.TEXT_PASS_2);
+	    break;
+	    
+	case 2:
+	    // -b 100000 -2 -m 32 -s 0 -G -t 10 -N1
+	    mgPasses.addPass(MGPassesWrapper.INDEX_PASS_1);
+	    mgPasses.setInvfLevel(MGPassesWrapper.INVF_LEVEL_2);
+	    mgPasses.setStemOptions(MGPassesWrapper.STEMMER_ENGLISH, MGPassesWrapper.NO_STEM_OR_CASE);
+	    mgPasses.setInversionMemLimit(32);
+	    mgPasses.ignoreSGMLTags(true);
+	    break;
+	    
+	case 3:
+	    // -b 100000 -2 -c 3 -G -t 10 -N2
+	    mgPasses.addPass(MGPassesWrapper.INDEX_PASS_2);
+	    mgPasses.setInvfLevel(MGPassesWrapper.INVF_LEVEL_2);
+	    mgPasses.ignoreSGMLTags(true);
+	    break;
+	}
+	
+	mgPasses.init();
+	System.out.println("Starting Pass " + this.pass);
+	return true;
+    }
+  
+    /**
+     *  Complete a pass - reset file counters, close files, etc.
+     */
+    public boolean endPass(int passNumber) {
+	try {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    Thread.sleep(1000); // what for??
+	}
+	catch (InterruptedException ex) {
+	    System.out.println(ex);
+	}
+	mgPasses.finish();
+	try {
+	    Thread.sleep(1000);
+	} catch (Exception e) {}
+	
+	int exit_value = 0;
+	System.out.println("Pass " + this.pass + " completed with " + exit_value);
+	if (exit_value !=0) {
+	    //assume something has gone wrong, don't continue
+	    current_index.setError(true);
+	    return false;
+	}
+	int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	String osextra = "";
+	if (!Misc.isWindows()) {
+	    osextra = " -d / ";
+	}
+	
+	switch (mgPass) {
+	    
+	case 0:
+	    System.out.println("Compressing dictionary");
+	    exit_value = Processing.runProcess("mg_compression_dict -f " + this.textStem + osextra + " -S -H -2 -k 5120");
+	    if (exit_value == 0) {
+		System.out.println("Compressed dictionary successfully written");
+	    } else {
+		System.err.println("Error from mg_compression_dict: " + exit_value);
+		current_index.setError(true);
+		
+		return false;
+	    }
+	    break;
+	    
+	case 2:
+	    System.out.println("Creating perfect hash");
+	    exit_value = Processing.runProcess("mg_perf_hash_build -f " + this.indexStem+osextra);
+	    if (exit_value ==0) {
+		System.out.println("Perfect hashes completed");
+	    } else {
+		System.err.println("Unable to build the perfect hash");
+		current_index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 3:
+	    System.out.println("Writing weights file");
+	    exit_value = Processing.runProcess("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Weights file successfully written");
+	    } else {
+		System.err.println("Unable to create weights file");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating inverted dictionary");
+	    exit_value = Processing.runProcess("mg_invf_dict -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Inverted dictionary file successfully written");
+	    } else {
+		System.out.println("Unable to create inverted dictionary file");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating Stem indexes");
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s1 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 1 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 1");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s2 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 2 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 2");
+		current_index.setError(true);
+		return false;
+	    }
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s3 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 3 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 3");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    break;
+	} // switch
+
+	mgPasses = null;
+	return true;
+    }
+    
+    /**
+     *  Do any tidying up
+     */
+    public void tidyup()
+    {
+    }
+    
+    /**
+     *  Return the number of passes required for this index.
+     */
+    public int getNumberOfPasses()
+    { 
+	//return 2 + this.indexes.size() * 2;
+	return this.indexes.size()*2;
+    }
+
+    public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list)
+    {
+	Document doc = service_rack_list.getOwnerDocument();
+
+	// generate the list of indexes
+	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	boolean found_index = false;
+	String def_index = ""; // the default index will just be the first one created for now.
+	for (int i=1; i<this.indexes.size(); i++) {
+	    // start at 1: 0 will be the text index
+	    MGIndex index = (MGIndex)this.indexes.get(i);
+	    if (!index.hasError()) {
+		Element e = doc.createElement(GSXML.INDEX_ELEM);
+		e.setAttribute(GSXML.NAME_ATT, index.getName());
+		index_list.appendChild(e);
+		if (found_index == false) {
+		    // this is the first index
+		    found_index = true;
+		    def_index = index.getName();
+		}
+	    }
+	}
+	
+	if (!found_index) {
+	    // no indexes were able to be created, so we can't use them or the text
+	    return false;
+	}
+	Element default_index = doc.createElement("defaultIndex");
+	default_index.setAttribute(GSXML.NAME_ATT, def_index);
+	Element base_index_name = doc.createElement("baseIndexPrefix");
+	base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
+	Element index_stem = doc.createElement("indexStem");
+	index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
+	
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	service_rack_list.appendChild(search_service_elem);
+	service_rack_list.appendChild(retrieve_service_elem);
+
+	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGSearch");  
+	search_service_elem.appendChild(index_list);
+	search_service_elem.appendChild(default_index);
+	search_service_elem.appendChild(base_index_name);
+	search_service_elem.appendChild(index_stem);
+	
+	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGRetrieve");
+	retrieve_service_elem.appendChild(default_index.cloneNode(true));
+	retrieve_service_elem.appendChild(base_index_name.cloneNode(true));
+	retrieve_service_elem.appendChild(index_stem.cloneNode(true));
+
+	return true;
+    }
+    
+}
+    
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 12189)
@@ -0,0 +1,732 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import org.greenstone.mgpp.MGPPPassesWrapper;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
+import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.Processing;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.io.File;
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Document;
+
+public class MGPPIndexer extends AbstractIndexer
+{ 
+    int          pass;
+    int          documentSeqNo;
+    int          sectionSeqNo;
+    String       name;
+    boolean      firstDocument;
+    File indexDirectory;
+    File textDirectory;
+    String       indexStem;
+    String       textStem;
+    StringBuffer indexBuffer;
+    String       outputDirectory;
+    //String       outputStem;
+//     String       passExtra;
+//     InputStream	 indexerFeedback;
+//     InputStream  indexerErrors;
+//     OutputStream indexerTextfeed;
+//     Process      mgpp_passes;
+    //String       overallName; 
+    String       currentIndexName;
+    String       currentIndexLevel;
+       String       currentIndexField; 
+    MGPPPassesWrapper mgppPasses;
+    
+
+    static final String documentSeparator = "<Document>";
+    static final String sectionSeparator = "<Section>";
+
+    static final String START_OF_DOCUMENT = "<Document>";
+    static final String END_OF_DOCUMENT = "</Document>";
+    static final String START_OF_SECTION = "<Section>";
+    static final String END_OF_SECTION = "</Section>";
+    
+    
+    public static final String MGPP_INDEX_TYPE = "mgpp";
+    public static final String INDEX_FILE_STEM = "index";
+
+
+    class MGPPIndex
+    {
+	public String name = null;
+	public String doc_level = null; 
+	public ArrayList levels = null;
+	public ArrayList fields = null;
+	boolean error = false;// assume built until we get an error
+	
+	public MGPPIndex(String name) {
+	    this.name = name;
+	    doc_level = "Document";
+	}
+
+	public void setDocLevel(String doc_level) {
+	    this.doc_level = doc_level;
+	}
+
+	public void addLevel(String level) {
+	    this.levels.add(level);
+	}
+
+	// change to allow nested fields
+	public void addField(String field) {
+	    this.fields.add(field);
+	}
+
+	public boolean hasError() {
+	    return this.error;
+	}
+	public void setError(boolean b) {
+	    this.error = b;
+	}
+
+    }
+    
+    
+    public MGPPIndexer(String name)
+    { 
+	this.name = name;
+	//this.passExtra = "";
+    }
+
+    public String getName()
+    { 
+	return this.name;
+    }
+
+    /**
+     *  The output directory should be (collection)/building/text/ for
+     *  normal Greenstone builds
+     */ 
+    public boolean configure(String label, String value)
+    { 
+	if (label.equals(IndexerManager.outputDir)) {
+	    this.outputDirectory = value;
+	    
+	    // attempt to ensure that the text subdirectory exists
+	    this.textDirectory = new File(outputDirectory, "text");
+	    if (!textDirectory.exists()) {
+		if (!textDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!textDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+	    // attempt to ensure that the index subdir exists
+	    this.indexDirectory = new File(outputDirectory, "idx");
+	    if (!indexDirectory.exists()) {
+		if (!indexDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!indexDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+	    // Sign to the user which mg directory is being used...
+	    System.out.println("Output MGPP text directory is " + this.textStem);
+	    System.out.println("Output MGPP index directory is " + this.indexStem); 
+	}
+	this.pass = 0;
+	return true;
+    }
+
+    public String getIndexType()
+    { 
+	return MGPP_INDEX_TYPE;
+    }
+
+    public boolean addIndex(String name, String level, String field)
+    { 
+// 	if (level == "doc_level") {
+// 	    passExtra = " -J " + level;
+// 	}
+// 	else {
+// 	    passExtra = " -K " + level;
+// 	}
+	return true;
+    }
+
+    /**
+     *  Index a single document; the document interface can be used to extract individual
+     *  metadata items etc. as required or desired and index those instead or as well as
+     *  the body text of the document.
+     */
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+	if (this.pass == 0) {
+	    document.removeAllMetadata("gsdl3", "mgseqno");
+	}
+
+	// why do this at the start and not at the end???
+	if (!this.firstDocument) {
+	    // Send a '</Document>' at the end of the doc
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgppPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	}
+
+	String docText = null;
+	
+	//int startSeqNo = this.sectionSeqNo;
+	//this.sectionSeqNo ++;
+	int startSeqNo = this.documentSeqNo;
+
+	Document domDocument = document.getDOMDocument();
+	if (domDocument != null) {
+	    System.err.println("dom doc is not null");
+	    METSStructure sections = document.getDocumentStructure().getStructure("Section");
+	    if (sections != null) {
+		System.err.println("sections are not null");
+		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+		//	System.out.println(docText);
+	    }
+	}
+	if (docText == null) {
+	    System.err.println("dom doc or sections was null - asking for doc text");
+	    //if (this.currentIndexField.equals("text")) {
+		//docText = Character.toString(END_OF_DOCUMENT) + document.getDocumentText(); 
+		docText = document.getDocumentText();
+		//}
+// 	    else {
+// 		StringBuffer textBuffer = new StringBuffer();
+// 		//textBuffer.append(END_OF_DOCUMENT);
+// 		List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
+// 		if (values != null) {
+// 		    Iterator valueIter = values.iterator();
+// 		    while (valueIter.hasNext()) {
+// 			String value = valueIter.next().toString();
+			
+// 			textBuffer.append(value);
+// 			if (valueIter.hasNext()) {
+// 			    //textBuffer.append(END_OF_SECTION);
+// 			    //	      sectionSeqNo ++;
+// 			}
+// 		    }
+// 		}
+// 		else {
+// 		    textBuffer.append("No data");
+// 		}
+// 		docText = textBuffer.toString();
+// 	    }
+	    sectionSeqNo ++;
+	}
+
+	    //try {					
+	    //	this.indexerTextfeed.write(documentSeparator.getBytes(), 0, documentSeparator.getBytes().length);
+	    // }
+// 	    catch (IOException ex) {
+// 		System.out.println("Bad output on end of document" + ex);
+// 		ex.printStackTrace();
+// 		return false;
+// 	    }
+// 	}
+
+	this.indexBuffer.append(START_OF_DOCUMENT);
+	//String docText = document.getDocumentText();
+	this.indexBuffer.append(docText);
+	//int startSeqNo = this.documentSeqNo;
+	
+// 	byte [] bytes = docText.getBytes();
+// 	int pos = 0, end = bytes.length;
+	
+// 	try {
+// 	    while (pos < end) {
+// 		this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
+// 		pos = pos + 512;
+	
+// 		try {
+// 		    while (this.indexerFeedback.available() > 0) {
+// 			byte b[] = new byte[this.indexerFeedback.available()];
+// 			System.out.println("Feedback of " + this.indexerFeedback.available());
+// 			this.indexerFeedback.read(b);
+// 			System.out.println(b);
+// 		    }
+// 		}
+// 		catch (IOException ex) {
+		    
+// 		}
+
+
+// 		try {
+// 		    while (this.indexerErrors.available() > 0) {
+// 			byte b[] = new byte[this.indexerErrors.available()];
+// 			System.out.println("Feedback of " + this.indexerErrors.available());
+// 			this.indexerErrors.read(b);
+// 			System.out.println(new String(b));
+// 		    }
+// 		}
+// 		catch (IOException ex){
+		    
+// 		}
+// 	    }
+// 	}
+// 	catch (IOException ex) {
+// 	    System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
+// 	    ex.printStackTrace();
+// 	    return false;
+// 	}
+	this.firstDocument = false;
+
+	if (this.pass == 0) {    
+	    document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(startSeqNo));
+	}
+	this.documentSeqNo++;
+	
+// 	try {
+// 	    while (this.indexerErrors.available() > 0) {
+// 		char c = (char) this.indexerErrors.read();
+// 		System.out.println(c);
+// 	    }
+// 	    while (this.indexerFeedback.available() > 0) {
+// 		byte b[] = new byte[this.indexerFeedback.available()];
+// 		System.out.println("Feedback of " + this.indexerFeedback.available());
+// 		this.indexerFeedback.read(b);
+// 	    }
+// 	}
+// 	catch (IOException ex) {
+	    
+// 	}
+	return true;
+    }
+
+    /**
+     *  Initialise the pass: open required files, check status
+     */
+    public boolean startPass(int passNumber)
+    { 
+	this.pass = passNumber;
+	this.firstDocument = true;
+	this.documentSeqNo = 1;
+	this.sectionSeqNo = 1;
+
+	this.mgppPasses = new MGPPPassesWrapper();
+	this.indexBuffer = new StringBuffer();
+	
+	MGPPIndex index = null; // do something with this!!
+	
+	// get the parameters for this execution of mg_passes
+	mgppPasses.setFileName((this.pass < 2 ? this.textStem : this.indexStem ));
+	if (!Misc.isWindows()) {
+	    mgppPasses.setBasePath("/");
+	}
+	
+	mgppPasses.setDocumentTag("Document");
+	//mgppPasses.addLevelTag("Section");
+
+	this.currentIndexLevel = "Document";// index.getLevel();
+	this.currentIndexField = "text";//index.getField();
+	this.currentIndexName = "idx"; //index.getName();
+	    
+
+	switch (this.pass) {
+	case 0:
+	    // -T1
+	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_1);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem + " -T1");
+	    break;
+	    
+	case 1:
+	    // -T2
+	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_2);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -T2");
+	    break;
+	    
+	case 2:
+	    // -I1
+	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_1);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I1");
+	    break;
+	    
+	case 3:
+	    //Process p = Runtime.getRuntime().exec("mgpp_perf_hash_build -f " + this.outputStem);
+	    //p.waitFor();
+	    // -I2
+	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_2);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I2");
+	    break;
+	}
+	    
+	//this.indexerFeedback = mgpp_passes.getInputStream();
+	//   this.indexerErrors   = mgpp_passes.getErrorStream();
+	//   this.indexerTextfeed = mgpp_passes.getOutputStream();
+	//	}
+	//catch (IOException ex) {
+	//   System.out.println(ex);
+	//   ex.printStackTrace();
+	//   return false;
+	//}/	
+	//catch (InterruptedException ex) {
+	//   System.out.println(ex);
+	//   ex.printStackTrace();
+	//   return false;
+	//}
+	mgppPasses.init();
+	System.out.println("Pass " + this.pass);
+	return true;
+    }
+
+    /**
+     *  Complete a pass - reset file counters, close files, etc.
+     */
+    public boolean endPass(int passNumber)
+    { 
+	// TODO: end pass
+	Process p;
+	MGPPIndex index = null; // do something with this!!
+	try {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgppPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    Thread.sleep(1000); // what for??
+	}
+	catch (InterruptedException ex) {
+	    System.out.println(ex);
+	}
+	mgppPasses.finish();
+	try {
+	    Thread.sleep(1000);
+	} catch (Exception e) {}
+	
+	int exit_value = mgppPasses.exitValue();
+	System.out.println("Pass " + this.pass + " completed with " + exit_value);
+	if (exit_value !=0) {
+	    //assume something has gone wrong, don't continue
+// 	    if (index != null) {
+// 		index.setError(true);
+// 		return false;
+// 	    }
+	}
+	
+	String osextra = "";
+	if (!Misc.isWindows()) {
+	    osextra = " -d / ";
+	}
+
+	switch (this.pass) {
+	case 0:
+	    //System.exit(1);
+	    System.out.println("Compressing dictionary");
+	    exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.textStem + " -S -H -2 -k 5120"+ osextra);
+	    
+	    if (exit_value == 0) {
+		System.out.println("Compressed dictionary successfully written");
+	    } else {
+		System.err.println("Error from mgpp_compression_dict: " + exit_value);
+		//index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 2:
+	    System.out.println("Creating perfect hash");
+	    exit_value = Processing.runProcess("mgpp_perf_hash_build -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Perfect hashes completed");
+	    } else {
+		System.err.println("Unable to build the perfect hash");
+		//index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 3:
+	    System.out.println("Writing weights file");
+	    exit_value = Processing.runProcess("mgpp_weights_build -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Weights file successfully written");
+	    } else {
+		System.err.println("Unable to create weights file");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating inverted dictionary");
+	    exit_value = Processing.runProcess("mgpp_invf_dict -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Inverted dictionary file successfully written");
+	    } else {
+		System.out.println("Unable to create inverted dictionary file");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating Stem indexes");
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s1 -f " + this.indexStem +osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 1 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 1");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s2 -f " + this.indexStem + osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 2 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 2");
+		//index.setError(true);
+		return false;
+	    }
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s3 -f " + this.indexStem + osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 3 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 3");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    break;
+	} // switch
+	
+	return true;
+    }
+	
+    /**
+     *  Do any tidying up
+     */
+    public void tidyup()
+    {
+    }
+    
+    /**
+     *  Return the number of passes required for this index.
+     */
+    public int getNumberOfPasses()
+    { 
+	return 4;
+    }
+
+    public boolean addServiceDescriptions(Element service_rack_list) {
+	Document doc = service_rack_list.getOwnerDocument();
+
+	// generate the list of indexes
+	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	Element e = doc.createElement(GSXML.INDEX_ELEM);
+	e.setAttribute(GSXML.NAME_ATT, "idx");
+	index_list.appendChild(e);
+	String def_index = "idx";
+	
+// 	boolean found_index = false;
+// 	String def_index = ""; // the default index will just be the first one created for now.
+// 	for (int i=0; i<this.indexes.size(); i++) {
+// 	    MGIndex index = (MGIndex)this.indexes.get(i);
+// 	    if (!index.hasError()) {
+// 		Element e = doc.createElement(GSXML.INDEX_ELEM);
+// 		e.setAttribute(GSXML.NAME_ATT, index.getName());
+// 		index_list.appendChild(e);
+// 		if (found_index == false) {
+// 		    // this is the first index
+// 		    found_index = true;
+// 		    def_index = index.getName();
+// 		}
+// 	    }
+// 	}
+	
+// 	if (!found_index) {
+// 	    // no indexes were able to be created, so we can't use them or the text
+// 	    return false;
+// 	}
+
+	Element f = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	
+	Element default_index = doc.createElement("defaultIndex");
+	default_index.setAttribute(GSXML.NAME_ATT, def_index);
+
+	Element base_index_name = doc.createElement("baseIndexPrefix");
+	base_index_name.setAttribute(GSXML.NAME_ATT, "dtx");  //overallName);
+
+	Element index_stem = doc.createElement("indexStem");
+	index_stem.setAttribute(GSXML.NAME_ATT, "index");
+
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element default_level = doc.createElement("defaultLevel");
+	default_level.setAttribute(GSXML.NAME_ATT, "Document");
+
+	Element level_list = doc.createElement("levelList");
+	Element level = doc.createElement("level");
+	level.setAttribute(GSXML.NAME_ATT, "Document");
+	level_list.appendChild(level);
+
+	Element field_list = doc.createElement("fieldList");
+	Element field = doc.createElement("field");
+	field.setAttribute(GSXML.NAME_ATT, "ZZ");
+	field_list.appendChild(field);
+
+	service_rack_list.appendChild(search_service_elem);
+	service_rack_list.appendChild(retrieve_service_elem);
+
+	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPSearch");  
+	search_service_elem.appendChild(index_list);
+	search_service_elem.appendChild(default_index);
+	search_service_elem.appendChild(level_list);
+	search_service_elem.appendChild(default_level);
+search_service_elem.appendChild(field_list); // do we need this??
+	search_service_elem.appendChild(base_index_name);
+	search_service_elem.appendChild(index_stem);
+
+	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPRetrieve");
+	retrieve_service_elem.appendChild(default_level.cloneNode(true));
+	retrieve_service_elem.appendChild(base_index_name.cloneNode(true));
+	retrieve_service_elem.appendChild(index_stem.cloneNode(true));
+
+	return true;
+    }
+ 
+ 
+  private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			  AbstractStructure structure, StringBuffer textBuffer,
+			  StringBuffer extraBuffer, String namespace) 
+			  //String name, String namespace, String field)
+  {
+    // send out the ctrl-c...if this is
+    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	// try doing this for all index types
+	if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+	    METSDivision division = (METSDivision) structure;
+
+	    // get the division metadata block
+	    METSDescriptive descriptive;
+	    String metadataId = division.getDefaultMetadataReference();
+	    if (metadataId == null) {
+		descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		division.addMetadataReference(descriptive.getID());
+	    }
+	    else {
+		// Get the descriptive item...
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+	    }
+	    
+	    descriptive.addMetadata("gsdl3", "mgseqno", this.name + "." + Integer.toString(this.sectionSeqNo));
+	    
+	    metsDoc.setChanged(true);
+	    //metsDoc.setModified(true);
+	    //	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	} // section level
+	
+	// append an 'end of section' marker
+	//textBuffer.append(END_OF_SECTION);
+	this.sectionSeqNo ++;
+	
+	// for document-level indexes, always append an 'end of document' tag at the
+	// end of the document for each section.  Otherwise, each section is followed
+	// by an end of document character.  This ensures that all indexes use the
+	// same document numbering...
+	if (this.currentIndexLevel == null ||
+	    this.currentIndexLevel.equals(IndexerInterface.DOCUMENT_LEVEL)) {
+	    extraBuffer.append(END_OF_DOCUMENT);
+	}
+	else {
+	    textBuffer.append(END_OF_DOCUMENT);
+	    this.documentSeqNo ++;
+	}
+	
+	// produce the body here for metadata output of divisions - in the case of
+      // text output, that will happen below...
+      if (!this.currentIndexField.equals("text")) 
+      { METSDescriptive descriptive;
+	
+	METSDivision division = (METSDivision) structure;
+
+	String metadataId = division.getDefaultMetadataReference();
+	    
+	descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+	if (descriptive != null) {
+	  List values = descriptive.getMetadata(namespace, this.currentIndexField);
+	  
+	  if (values != null) {	
+	    Iterator valueIter = values.iterator();
+	    while (valueIter.hasNext()) {
+	      String value = valueIter.next().toString();
+	      
+	      textBuffer.append(value);
+	      if (valueIter.hasNext()) {
+		  //textBuffer.append(END_OF_SECTION);
+	      }
+	    }
+	  }
+	}
+      }
+    }
+
+    // go through our children as required...
+    Iterator children = structure.getChildIterator();
+    Node startNode;
+    while (children.hasNext()) {
+      AbstractStructure child = (AbstractStructure) children.next();
+      
+      // get xpointer for child 
+      // get start position node
+      if (metsDoc.getDocumentType() == "METS"){
+	  startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+      } else {
+	  startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+      }
+	    //Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+      
+      // while this node isn't the child's start node, produce the HTML node text, if
+      // in text field mode...
+      if (this.currentIndexField.equals("text")) {
+	while (node != startNode) {
+	  XPointer.printNode(node, textBuffer, false);
+
+	  // print buffer to node
+	  node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+	}
+      }
+      
+      // recurse to child
+      node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+    } // while next child
+
+    // close a document - the actual closing \B will be done by the main
+    // loop, so only a required \C is printed here...
+    if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+      while (node != null) {
+	if (this.currentIndexField.equals("text")) {
+	  XPointer.printNode(node, textBuffer, false);
+	}
+	node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+      }
+      
+      //textBuffer.append(END_OF_SECTION);
+      this.sectionSeqNo ++;
+      
+    }
+    return node;
+  }
+
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+    //  String name, String namespace, String field)
+  { StringBuffer extraBuffer = new StringBuffer();
+    Node node = document.getDocumentElement();
+    StringBuffer textBuffer = new StringBuffer();
+
+    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
+    textBuffer.append(extraBuffer.toString());
+    return textBuffer.toString();
+  }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java	(revision 12189)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class AbstractIdentifierFactory
+{
+  private String idPrelude;
+  private int    idSeqNo;
+
+  public AbstractIdentifierFactory(String idPrelude)
+  { this.idPrelude = idPrelude;
+    this.idSeqNo   = 1;
+  }
+
+  public String getNextIdentifier()
+  { int id = this.idSeqNo ++;
+
+    return idPrelude + Integer.toString(id);
+  }
+
+  public void noteIdentifier(String identifier)
+  { if (!identifier.startsWith(this.idPrelude)) 
+      return;
+  
+    identifier = identifier.substring(this.idPrelude.length());
+
+    int value = Integer.parseInt(identifier);
+    if (value >= this.idSeqNo) {
+      this.idSeqNo = value + 1;
+    }
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java	(revision 12189)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class AbstractMetadataGroup
+{ private String ID;
+
+  public AbstractMetadataGroup(String ID)
+  { this.ID = ID;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java	(revision 12189)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+
+/**
+ *  A helpful 'superclass' of METSStructure and METSDivision which allows for some
+ *  simple generalisation of the two classes.
+ */
+
+public abstract class AbstractStructure
+{ Map               children;
+  AbstractStructure parent;
+  String            ID;     // the unique identifier for this division (unique within the
+                            // document, not necessarily unique within the collection).
+
+  public AbstractStructure(String ID)
+  { this.children = new LinkedHashMap();
+    this.parent   = null;
+    this.ID       = ID;
+  }
+
+  public AbstractStructure getParent()
+  { return this.parent;
+  }
+
+  public void setParent(AbstractStructure parent)
+  { this.parent = parent;
+  }
+
+  /**
+   *  Get the identifer of this structure
+   *
+   *  @param <code>String</code> the identifier.
+   */
+  public String getID()
+  { return this.ID;
+  }
+
+  public abstract String getStructureType();
+
+  /**
+   *  Add a child division, or sub-division to the document.  For example,
+   *  sections within a chapter.
+   *
+   *  @param <code>METSDivision</code> the child division.
+   */
+  public void addDivision(METSDivision child)
+  { this.children.put(child.getLabel(), child);
+    child.setParent(this);
+  }
+
+  public Iterator getChildIterator()
+  { return this.children.values().iterator();
+  }
+    
+  public int size()
+  { return this.children.size();
+  }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java	(revision 12189)
@@ -0,0 +1,18 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  TODO: complete a class to create file identifiers
+ */
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class FileIdentifierFactory extends AbstractIdentifierFactory
+{
+    public static final String FILE_IDENTIFIER_PRELUDE = "FILE";
+        
+    public FileIdentifierFactory()
+    { 
+	super(FILE_IDENTIFIER_PRELUDE);
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java	(revision 12189)
@@ -0,0 +1,113 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+/**
+ *  Implement a simple, unordered, namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class GSDL3Namespace extends SimpleNamespace
+{
+    public static String GSDL3_NAMESPACE_ID = "gsdl3";
+    private static String GSDL3_METADATA_TAG = "Metadata";
+    
+    public GSDL3Namespace()
+    { 
+	super(GSDL3_NAMESPACE_ID);
+    }
+    
+    public GSDL3Namespace(METSLocation location)
+    { 
+	super(GSDL3_NAMESPACE_ID, location);
+    }
+    
+    public GSDL3Namespace(Element mdWrapTag)
+    {	
+	super(GSDL3_NAMESPACE_ID);
+	
+	NodeList childNodes = mdWrapTag.getChildNodes();
+	for (int c = 0; c < childNodes.getLength(); c ++){
+	    // a metadata node
+	    if (childNodes.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE){
+		String metadataLabel = childNodes.item(c).getNodeName();
+
+		// cull any leading "GSDL3" tag and its colon...
+		if (metadataLabel.startsWith(GSDL3_NAMESPACE_ID + ":")){
+		    metadataLabel = metadataLabel.substring(this.name.length() + 1);
+		}
+		
+		// TODO: throw error: bad metadata type
+		if (metadataLabel.equals(GSDL3_METADATA_TAG) == false){
+		    continue;
+		}
+		
+		// Get the label - if the name isn't set, abort.
+		// TODO: throw error: incomplete metadata tag
+		metadataLabel = ((Element) childNodes.item(c)).getAttribute("name");
+		if (metadataLabel == null) {
+		    continue;
+		}
+		
+		// now build the associated value.
+		String metadataValue = "";
+		
+		Element childElement = (Element) childNodes.item(c);
+		for (int i = 0; i < childElement.getChildNodes().getLength(); i ++) {
+		    metadataValue = metadataValue + childElement.getChildNodes().item(i).toString();
+		}
+		
+		this.addMetadata(metadataLabel, metadataValue);
+	    }
+	}
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, String label)
+    {	
+	if (this.location == null){
+	    // just place the metadata in a simple wrapper
+	    Iterator values = this.metadataMap.getAll(label).iterator();
+	    
+	    while (values.hasNext()){
+		String tag;
+		String value = values.next().toString();
+		
+		tag = XMLTools.getOpenTag(GSDL3_NAMESPACE_ID, GSDL3_METADATA_TAG);
+		tag = XMLTools.addAttribute(tag, "name", label);
+		writer.print(tag);
+		writer.print(value);
+		
+		writer.println(XMLTools.getCloseTag(this.name, GSDL3_METADATA_TAG));
+	    }
+	}
+	else {
+	    String tag = XMLTools.getOpenTag("mets", "mdRef");
+	    tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	    tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", this.getName());
+	    tag = XMLTools.makeSingleton(tag);
+	    writer.println(tag);
+	}
+	return true;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java	(revision 12189)
@@ -0,0 +1,11 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  METS Administrative metadata
+ *
+ *  May contain zero or more instances of administrative metadata
+ */
+
+public class METSAdmin
+{
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java	(revision 12189)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSAgent
+{
+	String name;
+	String note;
+
+	String role;
+	String otherrole;
+	String type;
+	String othertype;
+
+	public METSAgent(String name, String note, String role, String type)
+	{	this.name = name;
+		this.note = note;
+
+		this.setRole(role);
+		this.setType(type);
+	}
+
+	/**
+	 *  Check if the role matches one of the standard offerings
+	 */
+	private boolean validateRole(String role)
+	{	return true;
+	}
+
+	/**
+	 *  Check if the type matches one of the standard offerings
+	 */
+	private boolean validateType(String role)
+	{	if (role.equals("INDIVIDUAL") ||
+		    role.equals("ORGANIZATION")) {
+			return true;
+		}
+		return false;
+	}
+
+	public void setRole(String otherRole)
+	{	// automatically use 'OTHER' if either the role or type of agent don't
+		// validate against the standard list
+		if (this.validateRole(role)) {
+			this.role = role;
+			this.otherrole = null;
+		}
+		else {
+			this.role = "OTHER";
+			this.otherrole = role;
+		}
+	}
+
+	public void setType(String otherType)
+	{	// automatically use 'OTHER' if either the role or type of agent don't
+		// validate against the standard list
+		if (this.validateType(type)) {
+			this.type = type;
+			this.othertype = null;
+		}
+		else {
+			this.type = "OTHER";
+			this.othertype = type;
+		}
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDate.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDate.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDate.java	(revision 12189)
@@ -0,0 +1,153 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSDate
+{
+	String era;
+	String year;
+	String month;
+	String day;
+	String hour;
+	String minute;
+	String second;
+	String timezone;
+
+	public void METSDate(String string)
+	{	String date;
+
+		int timeAt = string.indexOf('T');
+
+		// if we've got a time signature, get it
+		if (timeAt >= 0) {
+		  // get the date for later use
+			date = string.substring(0, timeAt);
+
+			// extract the time string
+			String time = string.substring(timeAt+1);
+
+			// check for time zone usage - currently '+' and '-' only...
+			int zoneAt = time.indexOf('+');
+			if (zoneAt == -1) {	
+				zoneAt = time.indexOf('-');
+			}
+			if (zoneAt >= 0) {
+				this.timezone = time.substring(zoneAt);
+				time = time.substring(0, zoneAt);
+			}
+
+			// finally, divide into hours, minutes etc.
+			int hourAt = time.indexOf(':');
+			if (hourAt >= 0) {
+				this.hour = time.substring(0, hourAt);
+
+				time = time.substring(hourAt+1);
+				int minuteAt = time.indexOf(':');
+				if (minuteAt >= 0)
+				{	this.minute = time.substring(0, minuteAt);
+					this.second = time.substring(minuteAt+1);
+				}
+				else
+				{ this.minute = time;
+					this.second = null;
+				}
+			}
+		}
+		else 
+		{ this.timezone = null;
+			this.hour     = null;
+			this.minute   = null;
+			this.second   = null;
+
+			date = string;
+		}
+
+		// get the era
+		int prefixEnds = 0;
+		while (!Character.isDigit(date.charAt(prefixEnds)))
+		{ prefixEnds ++;
+		}
+
+		if (prefixEnds != 0) {
+			this.era = date.substring(0, prefixEnds);
+			date = date.substring(prefixEnds);
+		}
+		else {
+			this.era = null;
+		}
+
+		// finally, get the date
+		
+		// now try dividing up by '-'
+		int yearAt = date.indexOf('-');
+		if (yearAt >= 0) {
+			this.year = date.substring(0, yearAt);
+			date = date.substring(yearAt+1);
+
+			int monthAt = date.indexOf('-');
+			if (monthAt >= 0) {
+				this.month = date.substring(0, monthAt);
+				this.day = date.substring(monthAt+1);
+			}
+			else {
+				this.month = date;
+				this.day   = null;
+			}
+		}
+		else {
+			// try dividing by YYYYMMDD - any part of which may be missing...
+			if (date.length() >= 4) {
+				this.year = date.substring(0, 4);
+				if (date.length() >= 6) {
+					this.month = date.substring(4, 6);
+					this.day = date.substring(6);
+				}
+				else {
+					this.month = date.substring(4);
+					this.day   = null;
+				}
+			}
+			// we just have a day in this case...
+			else {
+				this.year = date;
+				this.month = null;
+				this.day = null;
+			}
+		}
+	}
+
+	public String toString()
+	{	if (this.year == null) {
+			return "";
+		}
+
+		StringBuffer buffer = new StringBuffer();
+
+		if (this.era != null) {
+			buffer.append(era);
+		}
+		buffer.append(year);
+		if (this.month != null) {
+			buffer.append("-");
+			buffer.append(this.month);
+			if (this.day != null) {
+				buffer.append("-");
+				buffer.append(this.day);
+
+				if (this.hour != null) {
+					buffer.append("T");
+					buffer.append(this.hour);
+					buffer.append(":");
+					buffer.append(this.minute);
+					if (this.second != null) {
+						buffer.append(":");
+						buffer.append(this.second);
+
+						if (this.timezone != null) {
+							buffer.append(this.timezone);
+						}
+					}
+				}
+			}
+		}
+		return buffer.toString();
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java	(revision 12189)
@@ -0,0 +1,494 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.PrintWriter;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Support the descriptive metadata of a given document
+ *
+ *  The descriptive metadata can occur multiple times, so this 'wrapper' is
+ *  in fact a holder for a number of individual metadata sets.  Individual
+ *  metadata groups are found in METSNamespace items.
+ *
+ *  
+ */
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+public class METSDescriptive
+{ 
+    MultiMap map;
+    String   ID;
+    String   name;
+    
+    public METSDescriptive(String ID, String name)
+    { 
+	this.map  = new MultiMap();
+	this.ID   = ID;
+	this.name = name;
+    }
+    
+    public String getID()
+    { 
+	return this.ID;
+    }
+    
+    public String getName()
+    { 
+	return this.name;
+    }
+    
+    public void addNamespace(METSNamespace namespace)
+    { 
+	this.map.put(namespace.getName(), namespace);
+    }
+    
+    public METSNamespace getNamespace(String namespace)
+    {
+	return (METSNamespace) this.map.get(namespace);
+    }
+    
+    /**
+     *  Get the version of a metadata namespace that can be used for
+     *  adding new values to the document.  Remember, each namespace may
+     *  occur more than once, but only one of the occurrences can be 'open'.
+     *
+     *  @param <code>String</code> the name of the namespace
+     */
+    public METSNamespace getOpenNamespace(String namespace)
+    { 
+	if (this.map.getAll(namespace) == null) {
+	    return null;
+	}
+	
+	Iterator namespaces = ((List) this.map.getAll(namespace)).iterator();
+	
+	while (namespaces.hasNext()) {
+	    METSNamespace namespaceData = (METSNamespace) namespaces.next();
+	    
+	    if (namespaceData.isEditable()){
+		return namespaceData;
+	    }
+	}
+	return null;
+    }
+    
+    /**
+     *  Add a piece of metadata to this document.  Existing values are preserved.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata   
+     *  @param <code>String</code> the value of the metadata
+     */
+    public void addMetadata(String namespace, String label, String value)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData == null ||
+	    namespaceData.isEditable() == false) {
+	    namespaceData = NamespaceFactory.initNamespace(namespace);
+	    this.addNamespace(namespaceData);
+	}
+	namespaceData.addMetadata(label, value);
+    }
+    
+    /**
+     *  Add a piece of metadata to this document.  Existing values in editable
+     *  parts of the document's metadata are destroyed.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata   
+     *  @param <code>String</code> the value of the metadata
+     */
+    public void setMetadata(String namespace, String label, String value)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData == null ||
+	    namespaceData.isEditable() == false) {
+	    namespaceData = NamespaceFactory.initNamespace(namespace);
+	    this.addNamespace(namespaceData);
+	}
+	namespaceData.setMetadata(label, value);
+    }
+    
+    /**
+     *  Remove the metadata for a particular label.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata
+     */
+    public void removeMetadata(String namespace, String label)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData != null && namespaceData.isEditable() == true) {
+	    namespaceData.removeMetadata(label);
+	}
+    }
+    
+    /**
+     *  Fetch a <code>List</code> of the values matching the given label in
+     *  the requisite namespace.
+     *
+     *  @param <code>String</code> namespace
+     *  @param <code>String</code> label of metadata to match
+     *
+     *  @return <code>List</code> the corresponding values found.
+     */
+    public List getMetadata(String namespace, String label)
+    {
+	// Simple case - no metadata
+	if (!this.map.containsKey(namespace)){
+	    return null;
+	}
+	
+	// if there's just one instance of the requisite namespace,
+	// then again it is a simple case
+	if (this.map.getAll(namespace).size() == 1){
+	    return this.getNamespace(namespace).getMetadata(label);
+	}
+	
+	// otherwise, step through and accumulate
+	List accumulatedValues = new ArrayList();
+	Iterator namespaceIterator = this.map.getAll(namespace).iterator();
+	while (namespaceIterator.hasNext()){
+	    METSNamespace localNamespace = (METSNamespace) namespaceIterator.next();
+	    
+	    List localList = localNamespace.getMetadata(label);
+	    if (localList != null && localList.size() > 0){
+		accumulatedValues.addAll(localList);
+	    }
+	}
+	
+	// if the result of the accumulation is empty, return a null item
+	if (accumulatedValues.size() == 0){
+	    return null;
+	}
+	
+	return accumulatedValues;
+    }
+    
+    protected void parse_xmlData(METSNamespace namespace, Element element) 
+    {
+	if (namespace==null) {
+	    System.err.println("namespace is null!!!!");
+	}
+	NodeList children = element.getChildNodes();
+	
+	for (int c = 0; c < children.getLength(); c++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child  = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    // if (childName.equals("gsdl2:Metadata")) {
+	    if (childName.equals("gsdl3:Metadata")) {
+		String metaName = child.getAttribute("name");
+		Text text = (Text) child.getFirstChild();
+		String metaValue = text.getData();
+		
+		addMetadata(namespace.getName(),metaName, metaValue);
+	    }
+	}
+    }
+    
+    public void parse_mdWrap(Element element)
+    {
+	// deal with mets:mdWrap
+	
+	METSNamespace namespace = NamespaceFactory.parseXML(element);
+	
+	NodeList children = element.getChildNodes();
+	
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child  = (Element) children.item(c);
+	    String childName = child.getNodeName();
+
+	    if (childName.equals("mets:xmlData")) {
+		parse_xmlData(namespace, child);
+	    } else {
+		System.err.println("Warning: unrecognised tag " + childName 
+				   + "in mets:mdWrap");
+	    }
+	}
+    }
+
+    /**
+     *  Parse an XML Element as a METS Descriptive Metadata section
+     *
+     *  @param <code>Element</code> the XML element which represents the dmdSec itself
+     */
+    public static METSDescriptive parseXML(Element element) 
+    {
+	// Note: no parsing of attributes required, we just move onto
+	// the metadata namespaces/sections themselves
+
+	// deal with mets:dmdSec
+	String ID = element.getAttribute("ID");
+	String label = element.getAttribute("GROUPID");
+	
+	METSDescriptive thisDescriptive = new METSDescriptive(ID, label);
+
+	//thisDescriptive = new METSDescriptive(ID, label);
+	
+	NodeList children = element.getChildNodes();
+
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    if (childName.equals("mets:mdRef")) {
+		//for External Descriptive Metadata
+		METSNamespace namespace = NamespaceFactory.parseXML(element);
+		System.err.println("*** mets::mdRef does not appear to be implemented yet");
+	    } else if (childName.equals("mets:mdWrap")) {
+		thisDescriptive.parse_mdWrap(child);
+	    } else {
+		// TODO: raise an error!
+		System.out.println("Error: METSDescriptive: unrecognised tag "+childName);
+	    }
+	}
+	return thisDescriptive;
+    }
+    /**
+     *  Write the document metadata to a <code>PrintWriter</code> in METS
+     *  XML format.
+     *  
+     *  @param <code>PrintWriter</code> the writer to use for output
+     */
+    public void write(PrintWriter output)
+    { 
+	String tag = XMLTools.getOpenTag("mets", "dmdSec");
+	tag = XMLTools.addAttribute(tag, "ID", this.ID);
+	tag = XMLTools.addAttribute(tag, "GROUPID", this.name);
+	output.println(tag);
+	
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()){
+	    Object nextKey = keys.next();
+	    if (nextKey == null){
+		continue;
+	    }
+	    String key = nextKey.toString();
+	
+	    // get every copy of the current namespace name - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+	    
+	    // namespaces will write themselves...
+	    while (namespaces.hasNext()){
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+	
+		namespace.write(output);
+	    }
+	}
+	
+	tag = XMLTools.getCloseTag("mets", "dmdSec");
+	output.println(tag);
+    }
+    
+    public void copy_metadata(METSDescriptive dst) 
+    {
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()) {
+	    Object nextKey = keys.next();
+	    if (nextKey == null) {
+		continue;
+	    }
+	    String key = nextKey.toString();
+	    
+	    // get every copy of the current namespace name - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+      
+	    while (namespaces.hasNext()) {
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+		
+		// for each metadata name 
+
+		Iterator mdnIter = namespace.getMetadataNames();
+		while (mdnIter.hasNext()) {
+
+		    String md_name = mdnIter.next().toString();
+		    
+		    List values = getMetadata(key, md_name);
+		    
+		    if (values != null) {	
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    dst.addMetadata("gsdl3",md_name,value);
+			}
+		    }
+		}
+	    }
+	}
+    }
+
+
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	// check if this node is in the
+	int sqlId = -1;
+	
+	//    System.out.println("Writing " + connection.toString());
+	
+	GS3SQLSelect select = new GS3SQLSelect("metadata");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetaID", "=", this.ID);
+	GS3SQLWhereItem item = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    GS3SQLAction action;
+	    
+	    if (resultSet.first()) {
+		// the node already exists - no need to do anything as such
+		GS3SQLUpdate update = new GS3SQLUpdate("metadata");
+		update.setWhere(where);
+		action = update;
+	    }
+	    else { // Set result set to null, just in case first() didn't work above...
+		// It is a new node and needs writing
+		action = new GS3SQLInsert("metadata");
+		
+		action.addValue("DocID", document.getID().toString());
+		action.addValue("MetaID", this.ID);
+	    }
+	    
+	    // always set the group identifier
+	    action.addValue("GroupID", this.name);
+	    
+	    // execute the action as required
+	    statement.execute(action.toString());
+	    
+	    // get the resultSet again
+	    // now execute the select statement again to get the new identifier
+	    // 'MetadataRef'
+	    // System.out.println(select.toString());
+	    resultSet = statement.executeQuery(select.toString());
+	    
+	    if (resultSet.first()) {
+		// get the reference for this item...
+		sqlId = resultSet.getInt("MetadataRef");
+	    }
+	    statement.close();
+	}
+	catch (SQLException sql){
+	    System.err.println("METSDescriptive.writeSQL(): "+sql);
+	}
+	if (sqlId == -1) {
+	    return false;
+	}
+	
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()){
+	    Object nextKey = keys.next();
+	    if (nextKey == null){
+		continue;
+	    }
+	    String key = nextKey.toString();
+	    
+	    // get every current namespace - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+	    
+	    // namespaces will write themselves...
+	    while (namespaces.hasNext()){
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+		
+		if (!namespace.writeSQL(sqlId, connection)){
+		    return false;
+		}
+	    }
+	}
+	return true;
+    }
+    
+    public static METSDescriptive readSQL(DocumentInterface document, GS3SQLConnection connection,
+					  ResultSet resultSet)
+    {
+	try {
+	    String ID   = resultSet.getString("MetaID");
+	    String name = resultSet.getString("GroupID");
+	    
+	    // create the metadata block object
+	    METSDescriptive descriptive = new METSDescriptive(ID, name);
+	    
+	    // get its metadata reference to retrieve namespaces
+	    int metadataRef = resultSet.getInt("MetadataRef");
+	    
+	    // query the database for matching namespaces for this metadata block
+	    GS3SQLSelect select = new GS3SQLSelect("namespaces");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(metadataRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    select.setWhere(where);
+	    
+	    Statement statement = connection.createStatement();
+	    ResultSet namespaceSet = statement.executeQuery(select.toString());
+	    
+	    // parse through the namespaces, calling the namespace class to create instances as it will
+	    if (namespaceSet.first()) {
+		do {
+		    METSNamespace namespace = METSNamespace.readSQL(connection, namespaceSet);
+		    if (namespace != null) {
+			descriptive.addNamespace(namespace);
+		    }
+		    else {
+			System.err.println("Null namespace output");
+		    }
+		}
+		while (namespaceSet.next());
+	    }
+	    statement.close();
+	    return descriptive;
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSDescriptive.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return null;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java	(revision 12189)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class METSDescriptiveSection
+{ 
+    METSLocation location;
+    String       type;
+    String       otherType;
+    
+    public METSDescriptiveSection(String type)
+    {	
+	this.location  = null;
+	this.type      = type;
+	this.otherType = null;
+    }
+	
+    public METSDescriptiveSection(METSLocation location, String type)
+    {	
+	this.location  = location;
+	this.type      = type;
+	this.otherType = null;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java	(revision 12189)
@@ -0,0 +1,266 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSDescriptiveSet
+{ 
+    Map children;
+    Map childrenById;
+    DescriptiveIdentifierFactory identifierFactory;
+    
+    class DescriptiveIdentifierFactory extends AbstractIdentifierFactory
+    { 
+	public DescriptiveIdentifierFactory()
+	{ 
+	    super("DM");
+	}
+    }
+    
+    public METSDescriptiveSet()
+    { 
+	this.children = new HashMap();
+	this.childrenById = new HashMap();
+	
+	this.identifierFactory = new DescriptiveIdentifierFactory();
+	
+	this.createDescriptive("default");
+    }
+    
+    /**
+     *  Create a new descriptive child, adding it automatically to this descriptive set.
+     *
+     *  @param <code>String</code> a name to describe the descriptive set.
+     */
+    public METSDescriptive createDescriptive(String name)
+    { 
+	METSDescriptive descriptive = new METSDescriptive(this.identifierFactory.getNextIdentifier(), name);
+	
+	this.children.put(name, descriptive);
+	this.childrenById.put(descriptive.getID(), descriptive);
+	return descriptive;
+    }
+    
+    /**
+     *  Add a descriptive item to this set.
+     *
+     *  @param <code>METSDescriptive</code> the descriptive set
+     */
+    public void addDescriptive(METSDescriptive metadata)
+    { 
+	// insert the item into both maps...
+	this.children.put(metadata.getName(), metadata);
+	this.childrenById.put(metadata.getID(), metadata);
+	
+	// when a descriptive block is added, check that it hasn't been used before...
+	//this.identifierFactory.noteIdentifier(metadata.getID());
+	this.identifierFactory.noteIdentifier(metadata.getName());
+    }
+    
+    /**
+     *  Get a descriptive item by its name(GROUPID in METS file)
+     */
+    public METSDescriptive getDescriptive(String name)
+    { 
+	
+	return (METSDescriptive) this.children.get(name);
+    }
+    
+    /**
+     *  Get a descriptive item by its identifier
+     */
+    public METSDescriptive getDescriptiveById(String id)
+    { 
+	return (METSDescriptive) this.childrenById.get(id);
+    }
+    
+    public void addNamespace(String descriptiveName, METSNamespace namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	if (descriptive != null){
+	    descriptive.addNamespace(namespace);
+	}
+    }
+    
+    public METSNamespace getNamespace(String descriptiveName, String namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	if (descriptive != null){
+	    return descriptive.getNamespace(namespace);
+	}
+	return null;
+    }
+    
+    public METSNamespace getOpenNamespace(String descriptiveName, String namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    return descriptive.getOpenNamespace(namespace);
+	}
+	return  null;
+    }
+    
+    public void addMetadata(String descriptiveName, String namespace, String label, String value)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.addMetadata(namespace, label, value);
+	}
+    }
+    
+    public void setMetadata(String descriptiveName, String namespace, String label, String value)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.setMetadata(namespace, label, value);
+	}
+    }
+    
+    public void removeMetadata(String descriptiveName, String namespace, String label)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.removeMetadata(namespace, label);
+	}
+    }
+    
+    public void removeAllMetadata(String namespace, String label)
+    { 
+	Iterator descriptiveIter = this.children.values().iterator();
+	
+	while (descriptiveIter.hasNext()) {
+	    METSDescriptive descriptive = (METSDescriptive) descriptiveIter.next();
+	    
+	    descriptive.removeMetadata(namespace, label);
+	}
+    }
+    
+    public List getMetadata(String descriptiveName, String namespace, String label)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    return descriptive.getMetadata(namespace, label);
+	}
+	return null;
+    }
+    
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDescriptive group = (METSDescriptive) groups.next();
+	    
+	    group.write(writer);
+	}
+    }
+    
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDescriptive group = (METSDescriptive) groups.next();
+	    
+	    if (!group.writeSQL(document, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    public static METSDescriptiveSet parseXML(NodeList dmdSecs) 
+    {
+	METSDescriptiveSet set = new METSDescriptiveSet();
+	
+	for (int g = 0; g < dmdSecs.getLength(); g ++) {
+	    METSDescriptive metadata = METSDescriptive.parseXML((Element) dmdSecs.item(g));
+	    
+	    set.addDescriptive(metadata);
+	}
+    
+	System.err.println("******** Away to do transfer!!!!!!");
+	  
+	METSDescriptive top_level_metadata = set.getDescriptive("1");
+	METSDescriptive tl2_metadata = set.getDescriptive("1");
+	METSDescriptive opo_metadata = set.getDescriptive("1.1");
+	METSDescriptive default_metadata = set.getDescriptive("default");
+	
+	if (top_level_metadata != null) {
+	    if (default_metadata != null) {
+		top_level_metadata.copy_metadata(default_metadata);
+	    }
+	    else {
+		System.err.println("Warning: Could not find default metadata");
+	    }
+	}
+	else {
+	    System.err.println("Warning: Could not find top level metadata for groupID=1");
+	}
+	return set;
+    } 
+    
+    public static METSDescriptiveSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    {
+	METSDescriptiveSet set = new METSDescriptiveSet();
+	
+	GS3SQLSelect select = new GS3SQLSelect("metadata");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+
+	// start going through the matching metadata blocks
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSDescriptive descriptive = METSDescriptive.readSQL(document, connection, resultSet);
+		    if (descriptive != null) {
+			set.addDescriptive(descriptive);
+		    }
+		    else {
+			System.out.println("Null descriptive");
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSDescriptiveSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	
+	
+	return set;
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java	(revision 12189)
@@ -0,0 +1,681 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSDivision extends AbstractStructure
+{
+    String       order;       // see the METS documentation for the meaning of the different
+    String       orderLabel;  // label types.
+    String       userLabel;
+    String       type;        // e.g. section, chapter, physical or logical, etc.
+    List         metadataRefs;
+    List         adminRefs;
+    List         fileRefs;
+    
+    public static final String DIVISION_TYPE  = "Division";
+    public static final String DIVISION_TAG   = "div";
+    public static final String FPTR_TAG       = "fptr";
+    
+    public METSDivision(String ID, String order, String orderLabel, String userLabel, String type)
+    { 
+	super(ID);
+	
+	this.order = order;
+	this.orderLabel = orderLabel;
+	this.userLabel = userLabel;
+	this.type = type;
+	this.fileRefs = new ArrayList();
+	this.metadataRefs = new ArrayList();
+    }
+    
+    public String getStructureType()
+    { 
+	return DIVISION_TYPE;
+    }
+    
+    public String getLabel()
+    { 
+	return this.orderLabel;
+    }
+    
+    /**
+     *  Find all the divisions that match a given list of file group
+     *  identifiers...
+     *
+     *  @param <code>List</code> the list of identifiers to find
+     *  @param <code>List</code> the list into which to place any
+     *         matching divisions.
+     */
+    public void findDivisionsForFiles(List listOfFileIdentifiers, List resultList)
+    { 
+	Iterator iterator = this.fileRefs.iterator();
+	
+	// get the references on this division, and mark if necessary
+	while (iterator.hasNext()) {
+	    String reference = iterator.next().toString();
+	    
+	    Iterator fileIterator = listOfFileIdentifiers.iterator();
+	    while (fileIterator.hasNext()){
+		String fileRef = fileIterator.next().toString();
+		
+		if (fileRef.equals(reference)){
+		    resultList.add(this);
+		}
+	    }
+	}
+	
+	// get the child groups, and check each in turn...
+	Iterator groups = this.children.values().iterator();
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    group.findDivisionsForFiles(listOfFileIdentifiers, resultList);
+	}    
+    }
+    
+    /**
+     *  Get the namespace from the given set of descriptive blocks...
+     */
+    public METSNamespace findNamespace(String namespaceName, boolean isOpenNamespace, 
+				       METSDescriptiveSet descriptiveSet)
+    { 
+	METSNamespace namespace = null;
+	
+	// Special case - no existing metadata references
+	if (this.metadataRefs.size() == 0) {
+	    // Create new Descriptive and add it to the descriptive set
+	    METSDescriptive descriptive = new METSDescriptive("DM"+this.ID, this.orderLabel);
+	    descriptiveSet.addDescriptive(descriptive);
+	    
+	    // Create the namespace and add it to the Descriptive
+	    namespace = NamespaceFactory.initNamespace(namespaceName);
+	    if (namespace != null){
+		descriptive.addNamespace(namespace);
+	    }
+	    
+	    return namespace;
+	}
+	
+	Iterator descriptiveIterator = this.metadataRefs.iterator();
+	
+	while (descriptiveIterator.hasNext()){
+	    String descriptiveRef = descriptiveIterator.next().toString();
+	    
+	    METSDescriptive descriptive = descriptiveSet.getDescriptiveById(descriptiveRef);
+	    
+	    if (isOpenNamespace) {
+		namespace = descriptive.getOpenNamespace(namespaceName);
+	    }
+	    else {
+		namespace = descriptive.getNamespace(namespaceName);
+	    }
+	    
+	    if (namespace != null)
+		break;
+	}
+	
+	// Special case - no such namespace in any descriptive list...
+	if (namespace == null){
+	    // take the first descriptive area...
+	    METSDescriptive descriptive = descriptiveSet.getDescriptiveById(this.metadataRefs.get(0).toString());
+	    
+	    // create the namespace, and add it to the METSDescriptive
+	    namespace = NamespaceFactory.initNamespace(namespaceName);
+	    if (namespace != null){
+		descriptive.addNamespace(namespace);
+	    }
+	}
+	
+	return namespace;
+    }
+    
+    /**
+     *  Add a new file reference to the structure
+     *
+     *  @param <code>METSFileID</code> the file to be referenced.
+     */
+    public void addFileReference(METSFileID fileRef)
+    { 
+	this.addFileReference(fileRef.toString());
+    }
+    
+    public void addFileReference(String fileRef)
+    { 
+	this.fileRefs.add(fileRef);
+    }
+    
+    /**
+     *  Add a new metadata reference to the structure
+     */
+    public void addMetadataReference(String metadataRef)
+    { 
+	// TODO: correct the type
+	this.metadataRefs.add(metadataRef);
+    }
+
+    /**
+     *  Get all the metadata references
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    public List getMetadataReferences()
+    { 
+	return this.metadataRefs;
+    }
+
+    /**
+     *  Get all the default (first) metadata reference...
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    //TODO: check for this first *open* metadata item...
+    public String getDefaultMetadataReference()
+    { 
+	if (this.metadataRefs.size() == 0) {
+	return null;
+    }
+    return this.metadataRefs.get(0).toString();
+    }
+
+    /**
+     *  Get all the default (first) first filegroup reference...
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    public String getDefaultFileReference()
+    { 
+	if (this.fileRefs.size() == 0) {
+	return null;
+    }
+    return this.fileRefs.get(0).toString();
+    }
+
+    /**
+     *  Get a child or descendant division
+     *
+     *  @param <code>String</code> the path to the child division
+     *  @return <code>METSDivision</code> the division found.
+     */
+    public METSDivision getDivision(String divisionLabel)
+    { 
+	// if the division ID equals this, then process it
+	if (divisionLabel.equals(this.orderLabel)) {
+	    return this;
+	}
+
+	// if we just shouldn't be here, then error...
+	if (!divisionLabel.startsWith(this.orderLabel + ".")) {
+	    return null;
+	}
+
+	// get the next division in the division path...
+	String subdivisionPath = divisionLabel.substring(this.orderLabel.length() + 1);
+	String nextDivision = getTopDivisionName(subdivisionPath);
+	String nextLabel;
+
+	// first try the current division name with an appended '.' and the next child id...
+	METSDivision child = (METSDivision) this.children.get(this.orderLabel + "." + nextDivision);
+	if (child == null){
+	    // if that didn't work, try the next division on its own
+	    child = (METSDivision) this.children.get(nextDivision);
+	    
+	    // if neither worked, then abort...
+	    if (child == null) {
+		return null;
+	    }
+	    else {
+		// set the next identifier
+		nextLabel = METSDivision.getSubDivisionName(divisionLabel);
+	    }
+	} 
+	else {
+	    // set the next identifier...
+	    nextLabel = divisionLabel;
+	}
+	
+	return child.getDivision(nextLabel);
+    }
+    
+    /**
+     *  A convenience method for splitting a division name - this gives the name
+     *  without the leading division label...  To be used when division identifiers
+     *  are given as '.' separated paths and division names do not contain the
+     *  full path of their parents.
+     *
+     *  @return <code>String</code>
+     */
+    public static String getSubDivisionName(String divisionName)
+    { 
+	int at = divisionName.indexOf('.');
+
+	if (at >= 0){
+	    return divisionName.substring(at+1);
+	}
+	else {
+		return null;
+	}
+    }
+
+    /**
+     *  Get the parent division name.  Division identifiers
+     *  concatenate the ancestor's names with intervening
+     *  '.' characters.
+     *
+     *  @return <code>String</code> the division name
+     */
+    public static String getParentName(String divisionName)
+    { 
+	int at = divisionName.lastIndexOf('.');
+
+	if (at >= 0) {
+	    return divisionName.substring(0, at);
+	} 
+	return "";
+    }
+    
+    /**
+     *  Get the top division name.
+     *
+     *  @return <code>String</code> the division name
+     */
+    public static String getTopDivisionName(String divisionName)
+    { 
+	int at = divisionName.indexOf('.');
+
+	if (at >= 0){
+	    return divisionName.substring(0, at);
+	}
+	else {
+	    return divisionName;
+	}
+    }
+
+    /*  Parse an XML Element as a METSDivision section
+     *
+     *  @param <code>Element</code> the XML element which represents
+     *  the mets:div itself 
+     */
+    
+    public static METSDivision parseXML(Element element) 
+    {
+ 	METSDivision thisDivision = null;
+ 
+ 	String divId = element.getAttribute("ID");
+ 	String divType = element.getAttribute("TYPE");
+ 	String divOrder = element.getAttribute("ORDER");
+ 	String divOrderLabel = element.getAttribute("ORDERLABEL");
+ 	String divUserLabel = element.getAttribute("LABEL");
+ 	String divMetaRef = element.getAttribute("DMDID");
+ 	
+ 	thisDivision = new METSDivision(divId, divOrder,divOrderLabel, divUserLabel, divType);
+ 		
+ 	thisDivision.addMetadataReference(divMetaRef);
+ 
+ 	NodeList children = element.getChildNodes();
+ 
+ 	for (int c = 0; c < children.getLength(); c++){
+ 	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+ 	    Element child = (Element) children.item(c);
+ 	    String childName = child.getNodeName();
+ 
+ 	    if (childName.equals("mets:fptr")) {
+ 		
+ 		METSDivision.fptr_parseXML(thisDivision, child);
+ 	    } else if (childName.equals("mets:div")) {
+ 		METSDivision division = METSDivision.parseXML(child);
+ 		thisDivision.addDivision(division);
+ 	    } else {
+ 		System.err.println("Error: mets:div expected mets:div or mets:fptr, but got " + childName);
+ 	    } 
+ 	}
+ 	return thisDivision;
+    }
+ 
+ 
+    /**
+     * Parse an XML element as a  METSFptr Section
+     * @param <code>Element</code> the XML element which represents the mets:fptr itself
+     */
+    private static void fptr_parseXML(METSDivision thisDivision, Element element) 
+    {
+ 	String fileRef = element.getAttribute("FILEID");
+ 	thisDivision.addFileReference(fileRef);
+    }
+ 
+    /**
+
+
+    /**
+    *  Write the METS file in an XML to a text-output sink.
+    *
+    *  @param <code>PrintWriter</code> the destination of the output
+    */
+    public boolean write(PrintWriter writer)
+    { 
+	// Get the main tag for the division
+	String tag = XMLTools.getOpenTag("mets", DIVISION_TAG);
+	tag = XMLTools.addAttribute(tag, "ID", this.ID.toString());
+	tag = XMLTools.addAttribute(tag, "TYPE", this.type);
+	tag = XMLTools.addAttribute(tag, "ORDER", this.order);
+	if (this.orderLabel != null){
+	    tag = XMLTools.addAttribute(tag, "ORDERLABEL", this.orderLabel);
+	}
+	if (this.userLabel != null){ 
+	    tag = XMLTools.addAttribute(tag, "LABEL", this.userLabel);
+	}
+	
+	// Add a metadata reference attribute
+	if (this.metadataRefs.size() > 0){
+	    String metaList = this.writeList(this.metadataRefs);
+	    tag = XMLTools.addAttribute(tag, "DMDID", metaList);
+	}
+	
+	// write the central tag
+	writer.print("   ");  //indentation
+	writer.println(tag);
+
+	// write the list of file pointers for this structural element
+	if (this.fileRefs.size() > 0) {
+	    String fileList = this.writeTagList("mets", "fptr", "FILEID", this.fileRefs);
+	    writer.print("      ");  //indentation
+	    writer.println(fileList);
+	}
+	
+	// write out any children in turn
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.write(writer)) {
+		return false;
+	    }
+	}
+    
+	// close the div element
+	writer.print("   ");  //indentation
+	writer.println(XMLTools.getCloseTag("mets", DIVISION_TAG));
+
+	return true;
+    }
+
+    /**
+     *  A convenience method for writing a <code>List</code> as a series of tags...
+     */
+    private String writeTagList(String namespace, String tagname, 
+				String attribute, List list)
+    { 
+	StringBuffer listString = new StringBuffer();
+	
+	Iterator iterator = list.iterator();
+	while (iterator.hasNext()){
+	    String item = iterator.next().toString();
+	    
+	    String tag = XMLTools.getOpenTag(namespace, tagname);
+	    tag = XMLTools.addAttribute(tag, attribute, item);
+	    tag = XMLTools.makeSingleton(tag);
+	    
+	    listString.append(tag);
+	}
+	return listString.toString();
+    }
+
+    /**
+     *  A convenience method for writing a <code>List</code> as a space-separated <code>String</code>.
+     */
+    private String writeList(List list)
+    { 
+	StringBuffer listString = new StringBuffer();
+	
+	Iterator iterator = list.iterator();
+	while (iterator.hasNext()){
+	    String item = iterator.next().toString();
+	    
+	    if (listString.length() > 0) {
+		listString.append(" ");
+	    }
+	    
+	    listString.append(item);
+	}
+	return listString.toString();
+    }
+    
+    public boolean writeSQL(DocumentID docId, int parentRef, boolean parentIsStructure, 
+			    GS3SQLConnection connection)
+    { 
+	int sqlRef = -1;
+	
+	// set the "where" to find the reference for this division
+	GS3SQLSelect select = new GS3SQLSelect("divisions");
+	select.addField("DivisionRef");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(parentRef), GS3SQLField.INTEGER_TYPE);
+	GS3SQLWhereItem parentItem = new GS3SQLWhereItem("ParentType", "=", parentIsStructure ? METSStructure.STRUCTURE_TYPE : METSDivision.DIVISION_TYPE);
+	GS3SQLWhereItem item = new GS3SQLWhereItem("SectionID", "=", this.ID.toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(parentItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	Statement statement = null;
+	ResultSet resultSet = null;
+	
+	// Do the actual writing
+	GS3SQLAction action;
+	
+	try {
+	    statement = connection.createStatement();
+	    resultSet = statement.executeQuery(select.toString());
+	    if (!resultSet.first()) {
+		
+		GS3SQLInsert insert = new GS3SQLInsert("divisions");
+		insert.addValue("DocID", docId.toString());
+		insert.addValue("SectionID", this.ID.toString());
+		insert.addValue("ParentRef", Integer.toString(parentRef));
+		insert.addValue("ParentType", parentIsStructure == true ? METSStructure.STRUCTURE_TYPE : METSDivision.DIVISION_TYPE);
+		
+		action = insert;
+	    }
+	    else {
+		sqlRef = resultSet.getInt("DivisionRef");
+		
+		GS3SQLUpdate update = new GS3SQLUpdate("divisions");
+		GS3SQLWhere  updateWhere = 
+		    new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE));
+		update.setWhere(updateWhere);
+		action = update;
+	    }
+	}
+	catch (SQLException sql) {
+	    System.err.println(sql + " " + select.toString());
+	    return false;
+	}
+	
+	action.addValue("DivisionType", this.type.toString());
+	action.addValue("LabelOrder", this.order);
+	action.addValue("ShortLabel", this.orderLabel);
+	action.addValue("UserLabel", this.userLabel);
+	
+	try {
+	    statement.execute(action.toString());
+	} catch (SQLException e) {
+	    System.err.println("METSDivision.writeSQL(): "+e);
+	    return false;
+	}
+	
+	// if doing a fresh item, get the new structure reference...
+	if (sqlRef == -1){
+	    // get the new structure reference
+	    try {
+		// read in the structure reference
+		resultSet = statement.executeQuery(select.toString());
+		if (resultSet.first()) {
+		    sqlRef = resultSet.getInt("DivisionRef");
+		}
+	    }
+	    catch (SQLException sqlex) {
+		System.err.println("METSDIVISION.writeSQL(): Unable to retrieve reference for Division " + sqlex);
+		return false;
+	    }
+	}
+	
+	// delete the old file/metadata references
+	try {
+	    GS3SQLWhere referenceWhere = 
+		new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(sqlRef), 
+						    GS3SQLField.INTEGER_TYPE));
+	    
+	    GS3SQLDelete delete = new GS3SQLDelete("divisionfilerefs");
+	    delete.setWhere(referenceWhere);
+	    
+	    statement.execute(delete.toString());
+	    
+	    delete = new GS3SQLDelete("divisionmetarefs");
+	    delete.setWhere(referenceWhere);
+	    statement.execute(delete.toString());
+	
+	    // write the new file references
+	    if (this.fileRefs.size() > 0){
+		Iterator iterator = this.fileRefs.iterator();
+		
+		while (iterator.hasNext()) {
+		    GS3SQLInsert fileinsert = new GS3SQLInsert("divisionfilerefs");
+		    fileinsert.addValue("DocID", docId.toString());
+		    fileinsert.addValue("DivisionRef", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE);
+		    fileinsert.addValue("DivisionType", "Group");
+		    fileinsert.addValue("FileID", iterator.next().toString());
+		    statement.execute(fileinsert.toString());
+		}
+	    }
+	    
+	    // write the metadata references
+	    if (this.metadataRefs.size() > 0){
+		Iterator iterator = this.metadataRefs.iterator();
+		
+		while (iterator.hasNext()) {
+		    GS3SQLInsert metainsert = new GS3SQLInsert("divisionmetarefs");
+		    metainsert.addValue("DocID", docId.toString());
+		    metainsert.addValue("DivisionRef", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE);
+		    metainsert.addValue("DivisionType", "Group");
+		    metainsert.addValue("MetaID", iterator.next().toString());
+		    statement.execute(metainsert.toString());
+		}
+	    }
+	    statement.close();
+	} catch (SQLException e) {
+	    System.err.println("METSDIVISION.writeSQL(): "+e);
+	    return false;
+	}
+	// write out any children in turn
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.writeSQL(docId, sqlRef, false, connection)) {
+		return false;
+	    }
+	}
+	return true;
+    }
+    
+    public static METSDivision readSQL(GS3SQLConnection connection, ResultSet resultSet)
+    {
+	METSLocation metsLocation = null;
+	
+	GS3SQLSelect select = null;
+	try {
+	    // read the basic information
+	    String ID   = resultSet.getString("SectionID");
+	    String type = resultSet.getString("DivisionType");
+	    String order = resultSet.getString("LabelOrder"); 
+	    String orderLabel = resultSet.getString("ShortLabel");
+	    String userLabel  = resultSet.getString("UserLabel");
+      
+	    // construct the division object
+	    METSDivision division = new METSDivision(ID, order, orderLabel, userLabel, type);
+      
+	    // obtain a list of child divisions
+	    int divisionRef = resultSet.getInt("DivisionRef");
+
+	    select = new GS3SQLSelect("divisions");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(divisionRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSDivision.DIVISION_TYPE);
+	    where.add(whereItem);
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet childSet = statement.executeQuery(select.toString());
+      
+	    // circulate through to obtain further children
+	    if (childSet.first())
+		{
+		    do {
+			METSDivision childDivision = METSDivision.readSQL(connection, childSet);
+			division.addDivision(childDivision);
+		    }
+		    while (childSet.next());
+		}
+	    select = new GS3SQLSelect("divisionfilerefs");
+	    select.addField("*");
+	    where = new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(divisionRef), 
+							GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    ResultSet fileSet = statement.executeQuery(select.toString());
+
+	    if (fileSet.first()){
+		do {
+		    String reference = fileSet.getString("FileID");
+		    division.fileRefs.add(reference);
+		}
+		while (fileSet.next());  
+	    }
+	    
+	    select = new GS3SQLSelect("divisionmetarefs");
+	    select.addField("*");
+	    where = new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(divisionRef), 
+							GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    ResultSet metaSet = statement.executeQuery(select.toString());
+
+	    if (metaSet.first()){
+		do {
+		    String reference = metaSet.getString("MetaID");
+		    division.metadataRefs.add(reference);
+		}
+		while (metaSet.next());  
+	    }
+        
+	    statement.close();
+	    return division;
+	}
+	catch (SQLException sqlEx){
+	    System.out.println(sqlEx + " " +select.toString());
+	}    
+	return null;
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java	(revision 12189)
@@ -0,0 +1,75 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A METS document representation - the METS objects are represented
+ *  by hand-coded classes rather than JAXB products due to the fact
+ *  that we want wrappers to provide higher-level abstractions than
+ *  the basic METS XML mark-up.
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+
+import java.io.File;
+import java.net.URL;
+
+public class METSDocument
+{
+	// TODO: add use of 'ID'
+	String objectID;
+	String label;
+	String type;
+	// TODO: add use of 'PROFILE';
+
+	METSHeader      header;
+	METSDescriptiveSet descriptiveSet;
+	METSAdmin       admin;
+	METSFileSet     fileSet;
+
+	public METSDocument(String identifier)
+	{
+		this.objectID = identifier; // In Greenstone land, this should be a unique identifier within
+		                            // the current collection
+		this.type = "document"; // Types in METS are just strings - we currently use
+		                        // 'Document' as a standard representation.
+                            // Individual options should be defined in a published
+														// METS Profile - in Greenstone we currently 'cheat'
+														// and bodge the system
+		this.header      = new METSHeader();
+		this.descriptiveSet = new METSDescriptiveSet();
+		this.fileSet     = new METSFileSet();
+	}
+
+	/**
+	 *  Set the document identifier for this document
+	 *
+	 *  @param <code>String</code> the identifier
+	 */
+	public void setDocId(String id)
+	{	this.objectID = id;
+	}
+
+	public String getDocId()
+	{	return this.objectID;
+	}
+
+	public METSHeader getHeader()
+	{	return this.header;
+	}	
+
+	public void setHeader(METSHeader header)
+	{	this.header = header;
+	}
+
+	public void addFile(METSFile file)
+	{	this.fileSet.addFile(file);
+	}
+
+	public void addFile(URL url)
+	{	this.fileSet.addFile(url);
+	}
+
+	public void addFile(METSFile file, String toGroup)
+	{	this.fileSet.addFile(file, toGroup);
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFile.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFile.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFile.java	(revision 12189)
@@ -0,0 +1,352 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import java.net.URL;
+import java.net.URLConnection;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  An individual METS component file for a document.  Many features of METS are not
+ *  currently fully supported/used here, but member variables exist for all of the
+ *  standard members.
+ */
+
+public class METSFile
+{
+    METSFileID    id;
+    METSFilePos   location;
+    String        MIMEType;
+    int           sequenceNo;
+    long          size;
+    METSDate      created;
+    long          checkSum;
+    List          adminRefs;
+    List          describeRefs;
+    METSFileGroup group;
+    
+    /**
+     *  Create a new METSFile object, with the given properties.  The remaining
+     *  fields are filled in with default values.
+     *
+     *  @param <code>String</code> the id of the file
+     *  @param <code>METSFilePos</code> the location of the file
+     *  @param <code>String</code> the MIME type of the file
+     */
+    public METSFile(METSFileID id,  METSFilePos location, String mimeType)
+    {
+	this.id           = id;        // required
+	this.location     = location;  // the location
+	this.MIMEType     = mimeType;  // the MIME type;
+	this.sequenceNo   = -1;       // -1 indicates unset
+	this.size         = -1;        // -1 indicates unset
+	this.checkSum     = -1;        // -1 indicates unset
+	this.created      = null;
+	this.adminRefs    = new ArrayList();
+	this.describeRefs = new ArrayList();
+	this.group        = null;
+    }
+    
+    public boolean equals(URL url)
+    { 
+	return this.location.equals(url);
+    }
+    
+    /**
+     *  Set the parent group
+     *
+     *  @param <code>METSFileGroup</code> the new holding file group
+     */
+    public void setGroup(METSFileGroup group)
+    { 
+	this.group = group;
+    }
+    
+    /**
+     *  Get the parent group of this file
+     *
+     *  @return <code>METSFileGroup</code> the holding file group
+     */
+    public METSFileGroup getGroup()
+    { 
+	return this.group;
+    }
+    
+    /**
+     *  Get the identifier of this file
+     *
+     *  @return <code>METSFileID</code> the file identifier
+     */
+    public METSFileID getID()
+    { 
+	return this.id;
+    }
+    
+    /**
+     *  Set the identifier of this file
+     *
+     *  @param <code>METSFileID</code> the file identifier
+     */
+    public void setID(METSFileID id)
+    { 
+	this.id = id;
+    }
+    
+    /**
+     *  @return <code>String</code> the MIME (content) type of the file
+     */
+    public String getMIMEType()
+    { 
+	return this.MIMEType;
+    }
+    
+    /**
+     *  @return <code>URL</code> the location of the file.  This will often be in
+     *          "file://" form.
+     */
+    public URL getLocation()
+    { 
+	return this.location.getLocation();
+    }
+    
+    /**
+     *  Take a url and make a METSFile object out of it - works out
+     *  details such as the Mime type, identifiers and other necessary
+     *  information to call the constructor for METSFile
+     *
+     *  @param <code>URL</code> the url of the object - this may be a 
+     *                          reference to a file on the local filestore
+     *
+     *  @return <code>METSFile</code> the METS file object for the corresponding
+     *                                document component.
+     */
+    public static METSFile makeMETSFile(URL url, String mimeType)
+    {	
+	METSFile reply = new METSFile(new METSFileID(), new METSFilePos(url), mimeType);
+	return reply;
+    }
+    
+    public static METSFile makeMETSFile(URL url)
+    { 
+	String mimeType;
+	
+	if (url.getProtocol().equals("file")){
+	    // TODO: Work out the MIME type
+	    mimeType = URLConnection.getFileNameMap().getContentTypeFor(url.toString().substring(7));
+	    if (mimeType == null) {
+		mimeType = "text/plain";
+	    }
+	} else { 
+	    // TODO: look up the MIME type through the pertinent connection
+	    mimeType = HTTPTools.getMIMEType(url);
+	}
+	// TODO: make the identifier...
+	
+	return makeMETSFile(url, mimeType);
+    }
+    
+    /**
+     *  Write the METS file in an XML to a text-output sink
+     *
+     *  @param <code>PrintWriter</code> the destination of the output
+     */
+    public void write(PrintWriter writer)
+    { 
+	String tag = XMLTools.getOpenTag("mets", "file");
+	tag = XMLTools.addAttribute(tag, "MIMETYPE", this.MIMEType);
+	tag = XMLTools.addAttribute(tag, "ID", this.id.toString());
+	writer.print("      ");   //indentation
+	writer.println(tag);
+	
+	tag = XMLTools.getOpenTag("mets", "FLocat");
+	tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	
+	tag = XMLTools.makeSingleton(tag);
+	writer.print("         ");  //indentation
+	writer.println(tag);
+	
+	writer.print("      ");   //indentation
+	writer.println(XMLTools.getCloseTag("mets", "file"));
+    }
+    
+    
+    public boolean writeSQL(int groupReference, GS3SQLConnection connection)
+    { 
+	// check if this file is in the table already...
+	GS3SQLAction action;
+	
+	GS3SQLSelect select = new GS3SQLSelect("files");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(groupReference),
+							GS3SQLField.INTEGER_TYPE);
+	GS3SQLWhere  where  = new GS3SQLWhere(whereItem);
+	whereItem = new GS3SQLWhereItem("FileID","=", this.id.toString());
+	where.add(whereItem);
+	select.setWhere(where);
+
+	Statement statement = null;
+	
+	// if not, then make an insert action
+	try {
+	    statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(select.toString());
+	    if (!results.first()){
+		GS3SQLInsert insert = new GS3SQLInsert("files");
+		
+		insert.addValue("FileGroupRef", Integer.toString(groupReference), GS3SQLField.INTEGER_TYPE);
+		insert.addValue("FileID", this.id.toString());
+		
+		action = insert;
+	    }
+	    else {
+		GS3SQLUpdate update = new GS3SQLUpdate("files");
+		
+		update.setWhere(where);
+		action = update;
+	    }
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFile.writeSQL(): "+ex);
+	    return false;
+	}
+	action.addValue("FileLocType", this.location.getType());
+	action.addValue("FileLocation", this.location.getLocation().toString());
+	action.addValue("MIMEType", this.MIMEType);
+	
+	try {
+	    statement.execute(action.toString());
+	    statement.close();
+	    
+	} catch (SQLException e) {
+	    System.err.println("METSFile.writeSQL():"+e);
+	    return false;
+	}
+	return true;
+    }
+    
+    public static METSFile readSQL(GS3SQLConnection connection, ResultSet parentSet)
+    { 
+	try {
+	    String locType = parentSet.getString("FileLocType");
+	    String location = parentSet.getString("FileLocation");
+	    String mimeType = parentSet.getString("MIMEType");
+	    String id = parentSet.getString("FileID");
+	
+	    METSFileID metsID = new METSFileID(id);
+	    METSFilePos metsFilePos = new METSFilePos(locType, location);
+	    
+	    METSFile reply = new METSFile(metsID, metsFilePos, mimeType);
+	    return reply;
+	}
+	catch (SQLException ex){
+	    System.out.println("METSFile.readSQL(): "+ex);
+	}
+	catch (java.net.MalformedURLException urlEx){
+	    System.out.println("METSFile.readSQL(): "+urlEx);
+	}
+	return null;
+    }
+    
+    public static METSFile parse_file(Element element, METSFileGroup group, String parseFilePath)
+    {
+	NodeList children = element.getChildNodes();
+
+	METSFile thisFile = null;
+	
+	for (int c = 0; c < children.getLength(); c++){
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+		continue;
+	    }  
+	    Element child = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    if (childName.equals("mets:file")){
+		METSFile.parse_file(child, group, parseFilePath);
+	    } else if (childName.equals("mets:FLocat")) {
+		String mimeType = element.getAttribute("MIMETYPE");
+		String fileId = element.getAttribute("ID");   
+		
+		METSFilePos thisFilePos = METSFile.parse_flocateXML(child, parseFilePath, fileId);
+	   	thisFile = new METSFile(new METSFileID(fileId), thisFilePos, mimeType);    
+		group.addFile(thisFile);
+	    } else {
+		System.err.println("Warning: unrecognised tag " +childName);
+	    }
+	}
+	return thisFile;
+    }
+    
+    public static METSFilePos parse_flocateXML(Element element, String parseFilePath, String fileId)
+    {
+	METSFilePos thisFilePos = null;
+	
+	// get most of the information from the child FLocat node
+	String locationType = element.getAttribute("LOCTYPE");
+	String href = element.getAttribute("xlink:href");
+	//String fileId = element.getAttribute("ID");
+	try {
+	    thisFilePos = new METSFilePos(locationType, href);
+	    
+	    if (fileId.startsWith("default.")) {
+		//retrieve the string after ..import/
+		int importPosition = href.indexOf("import/")+7;
+		href.substring(importPosition);
+		if (fileId.equals("default.1")) {
+		    href = "file:" + parseFilePath+ "doctxt.xml";
+		} else {
+		    href = "file:" + parseFilePath + href.substring(importPosition);
+		}
+	    }
+            thisFilePos = new METSFilePos(locationType, href);
+	} catch (java.net.MalformedURLException ex) {
+	    // TODO: raise error
+	    System.err.println("METSFile.parse_flocateXML(): "+ex);
+	}
+	return thisFilePos;
+    }
+    
+    
+    /**
+     *  Get the location of this file as a URL
+     */
+    public URL getURL()
+    { 
+	return this.location.getLocation();
+    }
+    
+    /**
+     *  Get modified file date
+     */
+    public long getModifiedDatestamp()
+    { 
+	return this.location.getModifiedDatestamp();
+    }
+    
+    /**
+     *  Overridden <code>toString</code> for convenience - returns the location of the file.
+     *
+     *  @return <code>String</code> the location of the file as a string
+     */
+    public String toString()
+    {
+	return this.location.toString();
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java	(revision 12189)
@@ -0,0 +1,408 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.net.URL;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+import java.io.PrintWriter;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+/**
+ *  Child groups are indicated by colons.  E.g. Chapter1:Section1 would indicate
+ *  the group "Section1" inside the "Chapter1" group...
+ */
+
+public class METSFileGroup
+{ 
+    List   children;
+    List   childGroups;
+    String id;
+    
+    public static final String SECTION_PARENT = "Section";
+    public static final String GROUP_PARENT = "Group";
+    
+    public METSFileGroup(String id)
+    { 
+	this.children = new ArrayList();
+	this.childGroups = new ArrayList();
+	this.id = id;
+    }
+    
+    /**
+     *  Get the modified datestamp that applies for this file group.  An empty group
+     *  will return <code>0</code>.
+     *
+     *  @return <code>long</code> the date modified as milliseconds from the 1st January 1970
+     */
+    public long getModifiedDatestamp()
+    { 
+	long reply = 0; // empty groups reply '0'
+	
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    long fileStamp = file.getModifiedDatestamp();
+	    if (fileStamp > reply) {
+		reply = fileStamp;
+	    }
+	}
+	
+	childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    long fileStamp = fileGroup.getModifiedDatestamp();
+	    if (fileStamp > reply) {
+		reply = fileStamp;
+	    }
+	}
+	
+	return reply;
+    }
+    
+    /**
+     *  Find all the occurrences of a given file, and place them
+     *  in a <code>List</code>.
+     */
+    public void findGroups(URL findFile, List resultList)
+    { 
+	// find in this particular group
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    if (file.equals(findFile))
+		{ resultList.add(this.id);
+		}
+	}
+	
+	// iterate over the child groups
+	childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    
+	    fileGroup.findGroups(findFile, resultList);
+	}
+    }
+    
+    /**
+     *  Get a sub group of this METSFileGroup.
+     *
+     *  @param <code>String</code> the name of the subgroup
+     *  @return <code>METSFileGroup</code> the child group, which will
+     *          be <code>null</code> if the group is not known.
+     */
+    public METSFileGroup getSubgroup(String id)
+    { 
+	String childId;
+	
+	int dotAt = id.indexOf(':');
+	if (dotAt == -1){
+	    childId = null;
+	}
+	else{ 
+	    childId = id.substring(dotAt+1);
+	    id = id.substring(0, dotAt);
+	}
+	
+	// iterate over the child groups
+	Iterator childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup group = (METSFileGroup) childIter.next();
+	    
+	    if (group.id.equals(id)) {
+		if (childId == null) {
+		    return group;
+		}
+		else {
+		    return group.getSubgroup(childId);
+		}
+	    }
+	}
+	return null;
+    }
+    
+    /**
+     *  Add a filegroup to this group of files.
+     *
+     *	@param <code>METSFileGroup</code> the filegroup.
+     */
+    public void addGroup(METSFileGroup group)
+    { 
+	this.childGroups.add(group);
+    }
+    
+    /**
+     *  Add a file to this group of files.
+     *
+     *	@param <code>METSFile</code> the file in a METS File object.
+     */
+    public void addFile(METSFile file)
+    { 
+	this.children.add(file);
+	file.setGroup(this);
+	if (!file.getID().isDefined()){
+	    file.setID(new METSFileID(this.id + "." + Integer.toString(this.children.size())));
+	}
+    }
+    
+    /**
+     *  Get the nth child of the group
+     *
+     *  @param <code>int</code> the index into the group
+     *  @return <code>METSFile</code> the corresponding child.  This may be <code>null</code>
+     *          particularly if the index given falls outside of the valid range of child
+     *          indexes.
+     */
+    public METSFile getFile(int index)
+    { 
+	if (index < 0 || index >= this.children.size()){
+	    return null;
+	}
+	
+	return (METSFile) this.children.get(index);
+    }
+    
+    /**
+     *  Get all the files in the group as a <code>List</code>
+     *
+     *  @return <code>List</code> the file childen of the group.
+     */
+    public List getFiles()
+    { 
+	return this.children;
+    }
+    
+    /**
+     *  Get all the subgroups as a <code>List</code>
+     */
+    public List getSubgroups()
+    { 
+	return this.childGroups;
+    }
+    
+    /**
+     *  Get the number of files in the group
+     *
+     *  @return <code>int</code> the count
+     */
+    public int noOfFiles()
+    { 
+	return this.children.size();
+    }
+    
+    /**
+     *  Get the number of subgroups.
+     */
+    public int noOfSubgroups()
+    { 
+	return this.childGroups.size();
+    }
+    
+    /**
+     *  @return <code>String</code> the name of this group
+     */
+    public String getId()
+    { 
+	return this.id;
+    }
+    
+    public static METSFileGroup parse_groupXML(Element element, METSFileSet set, String parseFilePath, METSFileGroup group)
+    {
+ 	//deal with mets:fileGrp
+ 	NodeList children = element.getChildNodes();
+ 	
+ 	for (int c = 0; c < children.getLength(); c++) {
+ 	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+ 	    Element child = (Element) children.item(c);
+ 	    String childName = child.getNodeName();
+	    
+ 	    if (childName.equals("mets:fileGrp")) {
+ 		String groupId = child.getAttribute("ID");
+ 		METSFileGroup childGroup = new METSFileGroup(groupId);
+		
+ 		if (group != null){
+ 		    group.addGroup(childGroup);
+ 		} else {
+ 		    set.addGroup(childGroup);
+ 		}
+ 		METSFile file = METSFile.parse_file(child, childGroup, parseFilePath);
+ 	    }
+ 	}
+ 	return group;
+    }
+
+    /**
+     *  Write the object in an XML METS format.
+     *
+     *  @param <code>PrintWriter</code> the printwriter being used to write the object
+     */
+    public void write(PrintWriter writer)
+    { 
+	String tag = XMLTools.getOpenTag("mets","fileGrp");
+	tag = XMLTools.addAttribute(tag, "ID", this.id);
+	writer.print("   ");   //indentation
+	writer.println(tag);
+	
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    file.write(writer);
+	}
+	writer.print("   ");   //indentation
+	writer.println(XMLTools.getCloseTag("mets","fileGrp"));
+    }
+    
+    public boolean writeSQL(DocumentInterface document, String parentRef, boolean parentIsSection, 
+			    GS3SQLConnection connection)
+    { 
+	int sqlId = -1;
+	// check if this node is in the database already
+	GS3SQLSelect select = new GS3SQLSelect("filegroups");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileGroupID", "=", this.id);
+	GS3SQLWhereItem whereDoc  = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(whereDoc);
+	select.setWhere(where);
+	
+	ResultSet results = null;
+	Statement statement = null;
+	
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(select.toString());
+	    if (!results.first()) {
+		GS3SQLInsert insert = new GS3SQLInsert("filegroups");
+		
+		insert.addValue("DocID", document.getID().toString());
+		insert.addValue("FileGroupID", this.id);
+		insert.addValue("ParentRef", parentRef, GS3SQLField.INTEGER_TYPE);
+		insert.addValue("ParentType", parentIsSection ? SECTION_PARENT : GROUP_PARENT);
+		
+		statement.execute(insert.toString());
+	    }
+	    else {
+		// TODO: update the data for this file group...
+	    }
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFileGroup.writeSQL(): "+ex);
+	    return false;
+	}
+	
+	// get the filegroup reference now
+	try {
+	    results = statement.executeQuery(select.toString());
+	    if (results.first()) {
+		sqlId = results.getInt("FileGroupRef");
+	    } else {
+		statement.close();
+		return false;
+	    }
+	    statement.close();
+	} catch (SQLException sqlex) {
+	    System.err.println("METSFileGroup.writeSQL(): "+sqlex);
+	    return false;
+	}
+	// iterate over the child groups
+	Iterator childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    
+	    if (!fileGroup.writeSQL(document, Integer.toString(sqlId), false, connection)){
+		System.err.println("METSFileGroup.writeSQL(): Couldn't write FileGroup");
+		return false;
+	    }
+	}
+	
+	// iterate over the child files
+	childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();	    
+	    if (!file.writeSQL(sqlId, connection)){
+		System.err.println("METSFileGroup.writeSQL(): Couldn't write File");
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSFileGroup readSQL(DocumentInterface document, GS3SQLConnection connection,
+					ResultSet resultSet)
+    {
+	try {
+	    String ID         = resultSet.getString("FileGroupID");
+	    String parentType = resultSet.getString("ParentType");
+	    String parentRef  = resultSet.getString("ParentRef");
+	    
+	    // create the metadata block object
+	    METSFileGroup group = new METSFileGroup(ID);
+	    
+	    // get its metadata reference to retrieve divisions
+	    int groupRef = resultSet.getInt("FileGroupRef");
+	    
+	    // query the database for matching groups which are children of this one
+	    GS3SQLSelect select = new GS3SQLSelect("filegroups");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(groupRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSFileGroup.GROUP_PARENT);
+	    where.add(whereItem);
+	    select.setWhere(where);
+	    
+	    Statement statement = connection.createStatement();
+	    
+	    // parse through the child groups
+	    ResultSet childSet = statement.executeQuery(select.toString());
+	    if (childSet.first()) {
+		do {
+		    METSFileGroup fileGroup = METSFileGroup.readSQL(document, connection, childSet);
+		    if (fileGroup != null) {
+			group.addGroup(fileGroup);
+		    }
+		}
+		while (childSet.next());
+	    }
+	    // now scan for file members
+	    select = new GS3SQLSelect("files");
+	    select.addField("*");
+	    whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(groupRef), GS3SQLField.INTEGER_TYPE);
+	    where = new GS3SQLWhere(whereItem);
+	    select.setWhere(where);
+	    ResultSet childFileSet = statement.executeQuery(select.toString());
+	    if (childFileSet.first()) {
+		do {
+		    METSFile file = METSFile.readSQL(connection, childFileSet);
+		    if (file != null) {
+			group.addFile(file);
+		    }
+		}
+		while (childFileSet.next());
+	    }
+	    statement.close();
+	    return group;
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSFileGroup.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return null;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java	(revision 12189)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A type-safe wrapper for METS File identifiers.
+ */
+
+public class METSFileID
+{	
+    String id;
+    
+    public METSFileID()
+    {	
+	this.id = null;
+    }
+    
+    public METSFileID(String id)
+    {	
+	this.id = id;
+    }
+    
+    public String toString()
+    {	
+	return this.id;
+    }
+    
+    public boolean isDefined()
+    {	
+	return this.id != null;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java	(revision 12189)
@@ -0,0 +1,27 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ *  Notes: Greenstone doesn't currently use/support the SEQ or AREA 
+ *  file pointer types hence, one can use only a simple file reference
+ */
+
+public class METSFilePointer
+{
+	public static final String SIMPLE = "";
+	public static final String SEQUENCE = "seq";
+	public static final String AREA = "area";
+
+	String type;
+	String ID;
+	String fileId;
+	List   children;
+
+	public METSFilePointer(String ID, String fileId)
+	{	this.ID = ID;
+		this.fileId = fileId;
+		this.type = METSFilePointer.SIMPLE;
+	}	
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java	(revision 12189)
@@ -0,0 +1,41 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.File;
+import java.net.URL;
+
+public class METSFilePos
+{
+  METSLocation location;
+
+  public METSFilePos(File file) throws java.net.MalformedURLException
+  { this.location = new METSLocation(file);
+  }
+
+  public METSFilePos(URL url)
+  { this.location = new METSLocation(url);
+  }
+
+  public METSFilePos(String locationType, String location) throws java.net.MalformedURLException
+  { this.location = new METSLocation(locationType, location);
+  }
+  
+  public URL getLocation()
+  { return this.location.getLocation();
+  }
+
+  public String getType()
+  { return this.location.getType();
+  }
+
+  public long getModifiedDatestamp()
+  { return this.location.getModifiedDatestamp();
+  }
+
+  public String toString()
+  { return this.location.getLocation().toString();
+  }
+
+  public boolean equals(URL filePos)
+  { return this.location.equals(filePos);
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java	(revision 12189)
@@ -0,0 +1,13 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSFileRef
+{	String label;
+
+	public METSFileRef(String label)
+	{	this.label = label;
+	}
+
+	public String toString()
+	{	return this.label;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java	(revision 12189)
@@ -0,0 +1,344 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A class to hold all the alternative file structures
+ *  for a particular METS Document
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.File;
+import java.io.PrintWriter;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSFileSet
+{ 
+    Map    fileGroups;
+    String reference;
+    FileIdentifierFactory identifierFactory;
+    FileGroupIdentifierFactory groupIdentifierFactory;
+    
+    class FileGroupIdentifierFactory extends AbstractIdentifierFactory
+    {
+	public static final String FILEGROUP_PRELUDE = "FG";
+	
+	public FileGroupIdentifierFactory()
+	{
+	    super("FILEGROUP_PRELUDE");
+	}
+    }
+    
+    public METSFileSet()
+    { 
+	this.fileGroups = new HashMap();
+	this.fileGroups.put("default", new METSFileGroup("default"));
+	this.reference = null;
+	this.identifierFactory = new FileIdentifierFactory();
+	this.groupIdentifierFactory = new FileGroupIdentifierFactory();
+    }
+    
+    public long getModifiedDatestamp()
+    { 
+	long reply = 0;
+	
+	Iterator groupsIter = this.fileGroups.values().iterator();
+	while (groupsIter.hasNext()) {
+		METSFileGroup group = (METSFileGroup) groupsIter.next();
+		long groupStamp = group.getModifiedDatestamp();
+		if (groupStamp > reply){
+		    reply = groupStamp;
+		}
+	    }
+	return reply;
+    }
+    
+    
+    /**
+     *  Get a new FileGroup to lie within this fileset.  The group is not initialised with
+     *  a parent, etc. - all that is done is the allocation of a unique group identifier.
+     *
+     *  @return <code>METSFileGroup</code> the new group.
+     */
+    public METSFileGroup createGroup()
+    { 
+	METSFileGroup group = new METSFileGroup(this.groupIdentifierFactory.getNextIdentifier());
+	return group;
+    }
+    
+    public METSFile createFile(METSFilePos filePos, String type)
+    { 
+	METSFile file = new METSFile(new METSFileID(this.identifierFactory.getNextIdentifier()), filePos, type);
+	return file;
+    }
+    
+    /**
+     *  Get the group that corresponds to the given name...
+     *
+     *  @param <code>String</code> the name of the group.
+     *  @return <code>METSFileGroup</code> the group object - this will be
+     *          <code>null</code> if the group is not found
+     */
+    public METSFileGroup getGroup(String name)
+    { 
+	if (!this.fileGroups.containsKey(name)){
+	    return null;
+	}
+	return (METSFileGroup) this.fileGroups.get(name);
+    }
+    
+    /**
+     *  Get the Nth file from the default group...
+     *
+     *  @param <code>int</code> the index into the default group to use...
+     *  @return <code>METSFile</code> the file.
+     */
+    public METSFile getFile(int index)
+    { 
+	METSFileGroup group = this.getGroup("default");
+
+	if (group == null) {
+	    return null;
+	}
+	return group.getFile(0);
+    }
+    
+    /**
+     *  Add a file to the default file group
+     */
+    public void addFile(METSFile file)
+    { 
+	METSFileGroup defaultGroup = (METSFileGroup) this.fileGroups.get("default");
+	defaultGroup.addFile(file);
+    }
+    
+    /**
+     *  Add a file to the default file group
+     *
+     *  @return <code>METSFile</code> an object that wraps the given url into
+     *          a METS file object
+     */
+    public METSFile addFile(URL url)
+    { 
+	METSFile file = METSFile.makeMETSFile(url);
+	this.addFile(file);
+	return file;
+    }
+    
+    public void addFile(METSFile file, String toGroup)
+    { 
+	METSFileGroup destGroup = (METSFileGroup) this.fileGroups.get(toGroup);
+	
+	if (destGroup == null){
+	    destGroup = new METSFileGroup(toGroup);
+	    this.fileGroups.put(destGroup.getId(), destGroup);
+	}
+	
+	destGroup.addFile(file);
+    }
+    
+    public void addFile(URL url, String toGroup)
+    { 
+	METSFile file = METSFile.makeMETSFile(url);
+	this.addFile(file, toGroup);
+    }
+    
+    /**
+     *  Add a group at the top level of the METS File set.
+     *
+     *  @param <code>METSFileGroup</code> the file group to add.
+     */
+    public void addGroup(METSFileGroup group)
+    { 
+	this.fileGroups.put(group.getId(), group);
+    }
+    
+    /**
+     *  Get all the groups that contain a given URL/file.
+     *
+     *  @param <code>URL</code> the location of the file.
+     *  @return <code>List</code> the list of group references that contain the file.
+     */
+    public List findGroups(URL file)
+    { 
+	List resultList = new ArrayList();
+	
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    group.findGroups(file, resultList);
+	}
+	return resultList;
+    }
+    
+    /*
+     *
+     *
+     */
+    public static METSFileSet parseXML(NodeList fileSecs, METSFileSet fileSet, String parseFilePath) 
+    {
+ 	//METSFileSet set = new METSFileSet();> 
+ 	// this is in effect a group without a sense of 'self'...
+ 	for (int c = 0; c < fileSecs.getLength(); c++) {
+ 	    if (fileSecs.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+	    
+ 	    METSFileGroup groups = METSFileGroup.parse_groupXML((Element)fileSecs.item(c), fileSet, parseFilePath, null);
+ 	}
+ 	return fileSet;
+    }
+    
+
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	writer.println("<mets:fileSec>");
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    group.write(writer);
+	}
+	writer.println("</mets:fileSec>");
+    }
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	// write the set if the reference does not already exist...
+	if (this.reference == null){
+	    // Insert the file section into the database
+	    GS3SQLInsert insert = new GS3SQLInsert("filesection");
+	    insert.addValue("DocID", document.getID().toString());
+	    insert.addValue("FileSecID", "test"); // TODO: remove magic string
+	    
+	    Statement statement = null;
+	    try {
+		statement = connection.createStatement();
+		statement.execute(insert.toString());
+	    } catch (SQLException e) {
+		System.err.println("METSFileSet.writeSQL(): "+e);
+		return false;
+	    }
+	    
+	    // find the file section number
+	    GS3SQLSelect select = new GS3SQLSelect("filesection");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileSecID", "=", "test");
+	    GS3SQLWhereItem whereDoc  = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    where.add(whereDoc);
+	    select.setWhere(where);
+	    
+	    try {
+		ResultSet set = statement.executeQuery(select.toString());
+		if (set.first()) {
+		    int sectionRef = set.getInt("FileSectionRef");
+		    
+		    this.reference = Integer.toString(sectionRef);
+		}
+		statement.close();
+	    }
+	    catch (SQLException ex){
+		System.out.println("METSFileSet.writeSQL(): "+ex);
+		return false;
+	    }
+	}
+	
+	if (this.reference == null) {
+	    return false;
+	}
+	// write out the children
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    if (!group.writeSQL(document, this.reference, true, connection)){
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSFileSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	METSFileSet set = new METSFileSet();    
+	
+	// Get file sections from the filesection table (currently redundant)
+	GS3SQLSelect select = new GS3SQLSelect("filesection");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+
+	Statement statement = null;
+	
+	// Get the identifier for this file set, etc.
+	int fileSetRef = -1;
+	try {
+	    statement = connection.createStatement();
+	    ResultSet sections = statement.executeQuery(select.toString());
+	    if (sections.first()) {
+		fileSetRef = sections.getInt("FileSectionRef");
+		set.reference = Integer.toString(fileSetRef);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	    
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFileSet.readSQL() "+ex);
+	    return null;
+	}
+	// Get child file groups
+	select = new GS3SQLSelect("filegroups");
+	select.addField("*");
+	whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());    
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(fileSetRef), 
+					GS3SQLField.INTEGER_TYPE);
+	where.add(whereItem);
+	whereItem = new GS3SQLWhereItem("ParentType", "=", METSFileGroup.SECTION_PARENT);
+	where.add(whereItem);
+	select.setWhere(where);
+	// start going through the matching file groups
+	try {
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSFileGroup filegroup = METSFileGroup.readSQL(document, connection, resultSet);
+		    if (filegroup != null) {
+			set.addGroup(filegroup);
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.out.println("METSFileSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return set;
+    }
+    
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java	(revision 12189)
@@ -0,0 +1,32 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class METSHeader
+{
+    List agents;
+    METSDate createDate;
+    METSDate lastModDate;
+    
+    public METSHeader()
+    {
+	this.agents = new ArrayList();
+    }
+    
+    public METSHeader(METSAgent defaultAgent)
+    {	
+	this();
+	this.agents.add(defaultAgent);
+    }
+    
+    public void setCreateDate(METSDate createDate)
+    {	
+	this.createDate = createDate;
+    }
+    
+    public void setModifiedDate(METSDate modifiedDate)
+    {	
+	this.lastModDate = modifiedDate;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java	(revision 12189)
@@ -0,0 +1,59 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.File;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+public class METSLocation
+{
+  String type;
+  URL    location;
+
+  public METSLocation(String type, String location) throws java.net.MalformedURLException
+  { this.type = type;
+    this.location = new URL(location);
+  }
+
+  public METSLocation(URL url)
+  { this.type = "URL";
+    this.location = url;
+  }
+
+  public METSLocation(File file)
+  { this.type = "URL";
+    this.location = URLTools.getFileURL(file);
+  }
+
+  public boolean equals(URL url)
+  { return this.location.equals(url);
+  }
+
+  public URL getLocation()
+  { return this.location;
+  }
+
+  public long getModifiedDatestamp()
+  { if (this.location != null) {
+      if (this.location.getProtocol().equals("file")) {
+	File file = new File(this.location.getPath());
+	return file.lastModified();
+      }
+      else {
+	try {
+	  URLConnection connection = this.location.openConnection();
+	  return connection.getLastModified();
+	}
+	catch (java.io.IOException io)
+	{ // TODO: report error
+	}
+      }
+    }
+    return System.currentTimeMillis();
+  }
+
+  public String getType()
+  { return this.type;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java	(revision 12189)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSMetaParams
+{
+	
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java	(revision 12189)
@@ -0,0 +1,232 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.Iterator;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Implement a single namespace
+ */
+
+public abstract class METSNamespace
+{
+    protected String       type;        // the type of the metadata
+    protected String       otherType;   // if a non-standard type, then...
+    protected String       name;        // e.g. "DC" for Dublin Core
+    protected METSLocation location;    // the location of the corresponding file
+    protected String       group;       // the group of metadata namespaces to which
+    // this namespace belongs
+    protected String       id;          // the ID for this namespace
+
+    public METSNamespace(String name)
+    { 
+	this.name        = name;
+	this.location    = null;
+	this.group       = null;
+	this.id          = null;
+    }
+    
+    public METSNamespace(String name, METSLocation location)
+    { 
+	this.location    = location;
+	this.name        = name;
+	this.group       = null;
+	this.id          = null;
+    }
+    
+    /**
+     *  Get the name of the namespace...
+     *
+     *  @return <code>String</code> the name of the namespace
+     */
+    public String getName()
+    { 
+	return this.name;
+    }
+    
+    /**
+     *  Get the group of namespaces to which this namespace belongs.
+     *
+     *  @return <code>String</code> the name of the namespace group
+     */
+    public String getGroup()
+    {
+	return this.group;
+    }
+    
+    /**
+     *  Set the group of namespaces to which this namespace belongs.
+     *
+     *  @param <code>String</code> the name of the namespace group
+     */
+    public void setGroup(String group)
+    { 
+	this.group = group;
+    }
+    
+    /**
+     *  Get the ID for this namespace.
+     *
+     *  @return <code>String</code> the id of the namespace
+     */
+    public String getID()
+    {
+	return this.id;
+    }
+    
+    /**
+     *  Set the id of this namespace.
+     *
+     *  @param <code>String</code> the name of the namespace group
+     */
+    public void setID(String id)
+    {
+	this.id = id;
+    }
+
+    /**
+     *  <p>Indicate whether this namespace is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+    
+    public abstract boolean addMetadata(String label, String value);
+    public abstract boolean setMetadata(String label, String value);
+    public abstract boolean removeMetadata(String label);
+    public abstract boolean removeMetadata(String label, String value);
+    public abstract List    getMetadata(String label);
+    public abstract Iterator getMetadataNames();
+    public abstract boolean write(PrintWriter writer);
+    //  public abstract boolean writeSQL(GS3SQLConnection connection);
+
+    /**
+     *  Write out the metadata to an SQL database through a <code>GS3SQLConnection</code>.
+     *
+     *  @param <code>int</code> the sql identifier of the parent object (in "metadata") table.
+     *  @param <code>GS3SQLConnection</code> the SQL database to use.
+     */
+    public boolean writeSQL(int parentId, GS3SQLConnection connection)
+    { 
+	GS3SQLAction action;
+
+	// If the namespace is not null, then set it...
+	if (this.id != null) {
+	    // use an update action in this case...
+	    GS3SQLUpdate update = new GS3SQLUpdate("namespaces");
+	    update.addValue("NamespaceID", this.id);
+
+	    // set up the where clause
+	    GS3SQLWhere where = 
+		new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", this.id, 
+						    GS3SQLField.INTEGER_TYPE));
+	    update.setWhere(where);
+	    action = update;
+	}
+	else {
+	    GS3SQLInsert insert = new GS3SQLInsert("namespaces");
+	    action = insert;
+	}
+
+	if (this.location != null) {
+	    action.addValue("FileLoc", this.location.getLocation().toString());
+	    action.addValue("FileType", "URL");
+	}
+	else {
+	    // no location stuff
+	}
+	action.addValue("MetadataRef", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE);
+	action.addValue("NamespaceType", this.name);
+
+	// Execute the action
+	Statement statement = null;
+	try {
+	    statement = connection.createStatement();
+	    statement.execute(action.toString());
+	} catch (SQLException e) {
+	    System.err.println("METSNamespace.writeSQL(): "+e);
+	    return false;
+	}
+	// then get the namespace reference number if needsbe...
+	if (this.id == null) {
+	    GS3SQLSelect select = new GS3SQLSelect("namespaces");
+	    select.addField("NamespaceRef");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("NamespaceType", "=", this.name);
+	    where.add(where);
+	    try {
+		ResultSet result  = statement.executeQuery(select.toString());
+      		if (result.last()) {
+		    this.id = Integer.toString(result.getInt("NamespaceRef"));
+		}
+		statement.close();
+	    }
+	    catch (SQLException sqlex){
+		this.id = null;
+		System.err.println("METSNamespace.writeSQL(): "+sqlex);
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSNamespace readSQL(GS3SQLConnection connection, ResultSet resultSet)
+    {
+	METSLocation metsLocation = null;
+	
+	try {
+	    String name = resultSet.getString("NamespaceType");
+	    String id = resultSet.getString("NamespaceRef");
+	    String location = resultSet.getString("FileLoc");
+	    String type = resultSet.getString("FileType");
+	    if (location != null && type != null) {
+		metsLocation = new METSLocation(type, location);
+	    }
+	    
+	    METSNamespace namespace = NamespaceFactory.initNamespace(name, metsLocation);
+	    namespace.id = id;
+
+	    int namespaceRef = resultSet.getInt("NamespaceRef");
+      
+	    GS3SQLSelect select = new GS3SQLSelect("mdvalues");
+	    select.addField("*");
+	    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", Integer.toString(namespaceRef),
+								    GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet valuesSet = statement.executeQuery(select.toString());
+	    if (valuesSet.first()) {
+		do {
+		    String label = valuesSet.getString("Label");
+		    String value = valuesSet.getString("Value");
+
+		    namespace.addMetadata(label, value);
+		}
+		while (valuesSet.next());
+	    }
+	    statement.close();
+	    return namespace;
+	}
+	catch (java.net.MalformedURLException urlEx){
+	    System.err.println("METSNamespace.readSQL(): "+urlEx);
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSNamespace.readSQL(): "+sqlEx);
+	}    
+	return null;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java	(revision 12189)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ *  Notes: Greenstone doesn't currently use/support the SEQ file pointer type;
+ *         hence, one can use only 'area' or 
+ */
+
+public class METSPointer
+{
+	String ID;
+	METSLocation location; // Or something similar...?
+
+	public METSPointer ()
+	{
+	}	
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java	(revision 12189)
@@ -0,0 +1,323 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSStructure extends AbstractStructure
+{
+    String label;
+    String type;
+    METSDivision divGroup;
+
+    public static final String STRUCTURE_TYPE = "Structure";
+  
+    public METSStructure(String id, String label, String type)
+    { 
+	super(id);
+	
+	this.label    = label;
+	this.type     = type;
+	this.divGroup = null;
+    }
+
+    /**
+     *  Show what sort of <code>AbstractStructure</code> this structure is...
+     *
+     *  @return <code>String The identifying string for a <code>METSStructure</code> type.
+     */
+    public String getStructureType()
+    { return STRUCTURE_TYPE;
+    }
+  
+    /**
+     *  Find all the divisions that match a given list of file group
+     *  identifiers...
+     *
+     *  @param <code>List</code> the list of identifiers to find
+     *  @param <code>List</code> the list into which to place any
+     *         matching identifiers.
+     */
+    public void findDivisionsForFiles(List listOfFileIdentifiers, List resultList)
+    { Iterator groups = this.children.values().iterator();
+    while (groups.hasNext())
+	{ METSDivision group = (METSDivision) groups.next();
+
+	group.findDivisionsForFiles(listOfFileIdentifiers, resultList);
+	}    
+    }
+
+    /**
+     *  Add a division further into this structure.
+     *
+     *  @param <code>String</code> the identifier label for the division
+     *         within which the division is to be added.
+     *  @param <code>METSDivision</code> the division to add.   
+     */
+    public void addSubDivision(String divisionLabel, METSDivision division)
+    { String topDivision = METSDivision.getTopDivisionName(divisionLabel);
+
+    METSDivision child = (METSDivision) this.children.get(topDivision);
+    if (child == null)
+	{ return;
+	}
+
+    child = child.getDivision(divisionLabel);
+
+    if (child != null) {
+	child.addDivision(division);
+    }
+    }
+
+    /**
+     *  Get a subdivision of a given identifier...
+     * 
+     *  @param <code>String</code> the identifier of the division
+     *  @return <code>METSDivision</code> the division, which will be
+     *          <code>null</code> if it is not found.
+     */
+    public METSDivision getDivision(String divisionLabel)
+    { String topDivision = METSDivision.getTopDivisionName(divisionLabel);
+
+    if (!topDivision.equals("All"))
+	System.out.println("Top division is " + topDivision);
+    METSDivision child = (METSDivision) this.children.get(topDivision);
+    if (child == null)
+	{ System.out.println("No child found");
+	Iterator groups = this.children.keySet().iterator();
+	while (groups.hasNext()) {
+	    System.out.println(groups.next().toString());
+	}
+	return null;
+	}
+
+
+    child = child.getDivision(divisionLabel);
+
+    return child;
+    }
+
+
+    /*
+     *
+     * parse an XML Element as a METSStructure Section
+     *
+     * @param <code>Element</code> the XML element which represents
+     *  the mets:structMap itself 
+     */
+    
+    public static METSStructure parseXML(Element element) 
+    {
+ 	String structId = element.getAttribute("ID");
+ 	String structLabel = element.getAttribute("LABEL");
+	String structType = element.getAttribute("TYPE");
+ 
+ 	METSStructure thisStruct = new METSStructure (structId, structLabel, structType);
+ 	
+ 	NodeList divSecs = element.getChildNodes();
+ 	if (divSecs != null) {
+ 	    for (int c = 0; c < divSecs.getLength(); c++) {
+ 		if (divSecs.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		    continue;
+ 		}
+ 		Element divElement = (Element) divSecs.item(c);
+ 		String divName = divElement.getNodeName();
+ 		if (divName.equals("mets:div")) {
+ 		    METSDivision division = METSDivision.parseXML(divElement);
+ 		    thisStruct.addDivision(division);
+ 		} else {
+ 		    System.err.println("Error: mets:structMap was expecting mets:div but got "+divName);
+ 		}
+ 	    }
+ 	}
+ 	return thisStruct;
+    }
+ 
+    /**
+     *  Write this structure in XML(METS) format to a <code>PrintWriter</code>
+     */
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	String tag = XMLTools.getOpenTag("mets", "structMap");
+	tag = XMLTools.addAttribute(tag, "ID", this.ID.toString());
+	tag = XMLTools.addAttribute(tag, "TYPE", this.type);
+	tag = XMLTools.addAttribute(tag, "LABEL", this.label);
+	
+	writer.println(tag);
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    group.write(writer);
+	}
+	writer.println("</mets:structMap>");
+    }
+    
+    
+    /**
+     *  Write this METSStructure to an SQL database....
+     *
+     *  @param <code>DocumentInterface</code> the enclosing document.
+     *  @param <code>GS3SQLConnection</code> the SQL database connection.
+     */
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	int sqlRef = -1;
+	ResultSet results = null;
+	Statement statement = null;
+	
+	// Prepare query to see if this structure has already been written
+	GS3SQLSelect select = new GS3SQLSelect("structure");
+	select.addField("StructureRef");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhereItem item = new GS3SQLWhereItem("StructureID", "=", this.ID.toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	// attempt to execute the query & get the current structure's reference
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(select.toString());
+	    if (results.first()){
+		sqlRef = results.getInt("structureRef");
+	    }
+	}
+	catch (SQLException sqlEx) {
+	    System.err.print("METSStructure.writeSQL(): "+sqlEx);
+	    return false;
+	}
+	
+	// insert a new structure if it wasn't there previously
+	if (sqlRef == -1) {
+	    GS3SQLInsert insert = new GS3SQLInsert("structure");
+	    insert.addValue("DocID", document.getID().toString());
+	    insert.addValue("StructureID", this.ID.toString());
+	    insert.addValue("StructureType", this.type);
+	    insert.addValue("Label", this.label);
+	    
+	    try {
+		statement.execute(insert.toString());
+		
+		// get the new structure reference by re-running the original select object
+		results = statement.executeQuery(select.toString());
+		if (results.first()) {
+		    sqlRef = results.getInt("StructureRef");
+		} 
+	    } catch (SQLException sqlEx) {
+		System.err.print("METSStructure.writeSQL(): "+sqlEx);
+		return false;
+	    }
+	    
+	}
+	else {
+	    GS3SQLUpdate update = new GS3SQLUpdate("structure");
+	    update.setWhere(where);
+	    
+	    update.addValue("StructureType", this.type);
+	    update.addValue("Label", this.label);
+	    
+	    try {
+		statement.execute(update.toString());
+	    } catch (SQLException sqlEx) {
+		System.err.print("METSStructure.writeSQL(): "+sqlEx);
+		return false;
+	    }
+	}
+	
+	// close the statement
+	try {
+	    statement.close();
+	} catch (SQLException e) {
+	    System.err.println("METSStructure.writeSQL(): "+e);
+	}
+	// write out the child groups (Divisions) now...    
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.writeSQL(document.getID(), sqlRef, true, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+    
+    /**
+     *  Read a METSStructure from a database - on entry, the current item in the
+     *  <code>ResultSet</code> holds the row in the database with the structure's
+     *  data.
+     *
+     *  @param <code>DocumentInterface</code> the document which owns the structure
+     *  @param <code>GS3SQLConnection</code> the database connection itself, to load
+     *         child objects through, etc.
+     *  @param <code>ResultSet</code> pointing to the current row in the database.
+     */
+    public static METSStructure readSQL(DocumentInterface document, GS3SQLConnection connection,
+					ResultSet resultSet)
+    {
+	GS3SQLSelect select = null;
+	
+	try {
+	    String ID    = resultSet.getString("StructureID");
+	    String type  = resultSet.getString("StructureType");
+	    String label = resultSet.getString("Label");
+
+	    // create the metadata block object
+	    METSStructure structure = new METSStructure(ID, label, type);
+      
+	    // get its metadata reference to retrieve divisions
+	    int structureRef = resultSet.getInt("StructureRef");
+
+	    // query the database for matching division for this structure block
+	    select = new GS3SQLSelect("divisions");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(structureRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSStructure.STRUCTURE_TYPE);
+	    where.add(whereItem);
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet divisionSet = statement.executeQuery(select.toString());
+
+	    // parse through the divisions
+	    if (divisionSet.first()) {
+		do {
+		    METSDivision division = METSDivision.readSQL(connection, divisionSet);
+		    if (division != null) {
+			structure.addDivision(division);
+		    }
+		}
+		while (divisionSet.next());
+	    }
+	    statement.close();
+	    return structure;
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSStructure.readSQL(): "+ sqlEx + " " + select.toString());
+	    System.exit(1);
+	}
+	return null;
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java	(revision 12189)
@@ -0,0 +1,181 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSStructureSet
+{ 
+    Map children;
+
+    public static final String GSDL3_SECTION_STRUCTURE = "Section";
+
+    public METSStructureSet()
+    { 
+	this.children = new HashMap();
+    }
+
+    public void addStructure(METSStructure structure)
+    { 
+	this.children.put(structure.getID(), structure);
+    }
+
+    public METSStructure getStructure(String name)
+    { 
+	return (METSStructure) this.children.get(name);
+    }
+
+    /**
+     *  Find the divisions that contain a list of given files
+     *
+     *  @param <code>List</code> the list of files to find...
+     *  @return <code>List</code> the matching divisions.
+     */
+    public List findDivisionsForFiles(List listOfFileIdentifiers)
+    { 
+	List reply = new ArrayList();
+
+	// iterate across all the children...
+	Iterator structures = this.children.values().iterator();
+	
+	while (structures.hasNext()){
+	    METSStructure structure = (METSStructure) structures.next();
+	    
+	    structure.findDivisionsForFiles(listOfFileIdentifiers, reply);      
+	}    
+	return reply;
+    }
+    
+    /**
+     *  Add a division to an indicated structure.
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>METSDivision</code> the division to be added.
+     */
+    public void addDivision(String structureId, METSDivision division)
+    {
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+	
+	if (structure != null){
+	    structure.addDivision(division);
+	}
+    }
+
+    /**
+     *  Add a division to an indicated division within structure.
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>String</code> the division identifer, within which the division
+     *         is to be placed.
+     *  @param <code>METSDivision</code> the division to be added.
+     */
+    public void addSubDivision(String structureId, String divisionID, METSDivision division)
+    { 
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+
+	if (structure != null){
+	    structure.addSubDivision(divisionID, division);
+	}
+    }
+
+    /**
+     *  Get a division from within a given structure
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>String</code> the division identifer
+     *
+     *  @return <code>METSDivision</code> the division
+     */
+    public METSDivision getDivision(String structureId, String divisionId)
+    { 
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+
+	if (structure != null){
+	    return structure.getDivision(divisionId);
+	}
+	return null;
+    }
+    public static METSStructureSet parseXML(NodeList structMapSecs)
+    {
+ 	METSStructureSet set = new METSStructureSet();
+ 
+ 	for (int c = 0; c < structMapSecs.getLength(); c++) {
+ 	    METSStructure structure = METSStructure.parseXML((Element) structMapSecs.item(c));
+ 	    set.addStructure(structure);
+ 	}
+ 	return set;
+    }
+        
+
+    public void write(PrintWriter writer)
+    { 
+	Iterator structures = this.children.values().iterator();
+
+	while (structures.hasNext()){
+	    METSStructure group = (METSStructure) structures.next();
+	    
+	    group.write(writer);
+	}
+    }
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator structures = this.children.values().iterator();
+    
+	while (structures.hasNext()){
+	    METSStructure group = (METSStructure) structures.next();
+
+	    if (!group.writeSQL(document, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    public static METSStructureSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	METSStructureSet set = new METSStructureSet();
+
+	GS3SQLSelect select = new GS3SQLSelect("structure");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+	
+	// start going through the matching metadata blocks
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSStructure structure = METSStructure.readSQL(document, connection, resultSet);
+		    if (structure != null) {
+			set.addStructure(structure);
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSStructureSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return set;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java	(revision 12189)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.greenstone.gsdl3.gs3build.schema.Schema;
+import org.greenstone.gsdl3.gs3build.schema.SchemaNode;
+
+public class MetaElement
+{
+	Map    attributes;
+	List   children;
+	String name;
+
+	public MetaElement(Element element, Schema schema)
+	{	
+		// basic setup
+		this.attributes = new HashMap();
+		this.children   = new ArrayList();
+		this.name = element.getNodeName();
+		StringBuffer contentBuffer = new StringBuffer();
+		
+		// split the name into namespace/workspace
+		String workSpace = SchemaNode.getNamespace(name);
+		name = SchemaNode.getElementName(name);
+
+		// get attributes
+		NamedNodeMap attributes = element.getAttributes();
+		for (int a = 0; a < attributes.getLength(); a ++)
+		{	Node attribute = attributes.item(a);
+
+			this.attributes.put(attribute.getNodeName(), attribute.getNodeValue());
+		}
+
+		// get children
+		NodeList children = element.getChildNodes();
+		for (int c = 0; c < children.getLength(); c++)
+		{	Node child = children.item(c);
+
+			if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE)
+			{	// make a child
+				if (contentBuffer.length() > 0) {
+					this.children.add(contentBuffer.toString());
+					contentBuffer.setLength(0);
+				}
+				Object childElement = new MetaElement((Element) child, schema);
+				this.children.add(childElement);
+			}
+			else
+			{	// append to content if not an element - this may be wasteful & type of
+				// element should be checked
+ 				contentBuffer.append(child.getNodeValue());
+			}
+		}
+		if (contentBuffer.length() > 0) {
+			this.children.add(contentBuffer.toString());
+		}
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java	(revision 12189)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+
+/**
+ *  A class which provides metadata-centric tools for handling XML objects
+ */
+public class MetaTools
+{
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java	(revision 12189)
@@ -0,0 +1,33 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class MetadataLabel
+{
+  String namespace;
+  String label;
+
+  public MetadataLabel(String label)
+  { int colonAt = label.indexOf(":");
+    String namespace;
+		
+    if (colonAt > 0) {
+      this.namespace = label.substring(0, colonAt);
+      this.label     = label.substring(colonAt+1);
+    }
+    else {
+      this.namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
+      this.label = label;
+    }
+  }
+
+  public String getNamespace()
+  { return this.namespace;
+  }
+
+  public String getLabel()
+  { return this.label;
+  }
+
+  public static String getDefaultNamespace()
+  { return GSDL3Namespace.GSDL3_NAMESPACE_ID;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java	(revision 12189)
@@ -0,0 +1,71 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+public class NamespaceFactory
+{
+    public static METSNamespace initNamespace()
+    {	
+	return new GSDL3Namespace();
+    }
+    
+    public static METSNamespace initNamespace(String name)
+    {	
+	if (name.equals(GSDL3Namespace.GSDL3_NAMESPACE_ID)) {
+	    return new GSDL3Namespace();
+	}
+	return new SimpleNamespace(name);
+    }
+    
+    public static METSNamespace initNamespace(String name, METSLocation location)
+    {	
+	if (name.equals(GSDL3Namespace.GSDL3_NAMESPACE_ID)){
+	    return new GSDL3Namespace(location);
+	}
+	return new SimpleNamespace(name, location);
+    }
+    
+    /**
+     *  TODO: parse the document in based upon its namespace
+     */
+    public static METSNamespace initNamespace(String name, Element element)
+    {	
+	//return null;
+	return new SimpleNamespace(name);
+    }
+    
+    public static METSNamespace parseXML(Element element)
+    {
+	METSNamespace namespace = null;
+	
+	//String mdType = element.getAttribute("MDType");
+	//String mdType =element.getAttribute("MDTYPE");
+
+	String othermdType = element.getAttribute("OTHERMDTYPE");
+	String mimeType = element.getAttribute("mimeType");
+	
+	if (element.getNodeName().equals("mets:mdWrap")) {
+	    namespace = initNamespace(othermdType, element);
+	}
+	else if (element.getNodeName().equals("mets:mdRef")) {
+	    String mdType = element.getAttribute("MDType");
+	    // just a file wrapper on the metadata...
+	    String location = element.getAttribute("xlink:href");
+	    String locationType = element.getAttribute("LOCTYPE");
+	    
+	    try {
+		METSLocation metsLocation = new METSLocation(locationType, location);
+		namespace = initNamespace(mdType, metsLocation);
+	    }
+	    catch (java.net.MalformedURLException ex) {
+		// TODO: Error, should never happen in the case of Greenstone-exported XML
+	    }
+	}
+	return namespace;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java	(revision 12189)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A simple metadata item with a label and one value to fit...
+ */
+
+public class NamespaceItem
+{
+	private String label;
+	private String value;
+
+	public NamespaceItem(String label, String value)
+	{	this.label = label;
+		this.value = value;
+	}
+
+	public String getLabel()
+	{	return this.label;
+	}
+
+	public String getValue()
+	{	return this.value;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java	(revision 12189)
@@ -0,0 +1,259 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+/**
+ *  Implement a generic, ordered namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class OrderedNamespace extends METSNamespace
+{
+    List metadataList;
+    
+    public OrderedNamespace(String name)
+    { 
+	super(name);
+	this.metadataList = new ArrayList();
+    }
+    
+    public OrderedNamespace(String name, METSLocation location)
+    { 
+	super(name, location);
+	this.metadataList = new ArrayList();
+    }
+    
+    public boolean validate(String field, String value)
+    { 
+	return true;
+    }
+    
+    /**
+     *  Add a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean addMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataList.add(new NamespaceItem(label, value));
+	return true;
+    }
+    
+    /**
+     *  Assign a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.  Any existing metadata for that field will
+     *  be destroyed if the new value validates.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean setMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)){
+	    return false;
+	}
+	
+	// TODO:		this.metadataMap.setOnly(label, value);
+	int itemNo = this.findItem(label);
+	if (itemNo >= 0){
+	    this.metadataList.set(itemNo, new NamespaceItem(label, value));
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     *  Remove all metadata values for a given field name
+     *
+     *  @param <code>String</code> the field to delete
+     *
+     *  @return <code>boolean</code> whether the field was actually deleted; 
+     *          will return <code>true</code> if the field was already empty.
+     */
+    public boolean removeMetadata(String label)
+    { 
+	int itemNo;
+	
+	itemNo = this.findItem(label);
+	while (itemNo >= 0){
+	    this.metadataList.remove(itemNo);
+	    itemNo = this.findItem(label);
+	}
+	return true;
+    }
+    
+    /**
+     *  Remove a particular incidence of a given metadata field for a document.
+     *  If an exact match for the given value is not found, nothing changes.
+     *  N.B. if a value occurs twice, only the first incidence of it will be
+     *  deleted from the list.
+     *
+     *  @param <code>String</code> the field to have the value removed
+     *  @param <code>String</code> the value to be removed from a given field
+     *
+     *  @return <code>boolean</code> <code>true</code> if an actual metadata text
+     *          is matched against the given value and is thus deleted.
+     */
+    public boolean removeMetadata(String label, String value)
+    { 
+	int itemNo;
+	
+	itemNo = this.findItem(label);
+	if (itemNo >= 0) {
+	    this.metadataList.remove(itemNo);
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     *  Get the metadata items for a particular label
+     *
+     *  @param <code>String</code> the label to fetch values for - must be devoid
+     *                             of namespace prologue (i.e. "title" rather than
+     *                             e.g. "dc:title").
+     *  
+     *  @return <code>List</code> the list of corresponding values.  May be
+     *          <code>null</code> if no values are found for the metadata
+     */
+    public List getMetadata(String label)
+    { 
+	List resultList = new ArrayList();
+	
+	for (int i = 0; i < this.metadataList.size(); i ++) {
+	    NamespaceItem item = (NamespaceItem) this.metadataList.get(i);
+	    
+	    if (item.getLabel().equals(label)){
+		resultList.add(item.getValue());
+	    }
+	}
+	if (resultList.size() > 0){
+	    return resultList;
+	}
+	return null;
+    }
+    
+    private int findItem(String label)
+    { 
+	for (int i = 0; i < this.metadataList.size(); i ++) {
+	    if (((NamespaceItem) this.metadataList.get(i)).getLabel().equals(label)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    private int findItem(String label, String value)
+    { 
+	for (int i = 0; i < this.metadataList.size(); i ++){
+	    NamespaceItem item = (NamespaceItem) this.metadataList.get(i);
+	    if (item.getLabel().equals(label) && item.getValue().equals(value)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    public Iterator getMetadataNames() 
+    {
+ 	return this.metadataList.iterator();
+    }
+    
+    
+    /**
+     *  Write out the metadata to an XML file through a <code>PrintWriter</code>.
+     *
+     *  @param <code>PrintWriter</code> the writer to use.
+     */
+    public boolean write(PrintWriter writer)
+    { 
+	// if this is a non-file block of metadata, write it out in long hand
+	if (this.location == null){
+	    String tag = XMLTools.getOpenTag("mets", "mdWrap");
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+	    tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+	    if (this.id != null) {
+		this.id = "gsdl"+this.id;
+		tag = XMLTools.addAttribute(tag, "ID", this.id);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println(tag);
+	    
+	    Iterator items = this.metadataList.iterator();
+	    
+	    while (items.hasNext()){
+		NamespaceItem item = (NamespaceItem) items.next();
+		this.writeItem(writer, item);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println("</mets:mdWrap>");
+	}
+	// otherwise, drop the metadata out in a simplified file-reference
+	// form only
+	else
+	    { 
+		String tag = XMLTools.getOpenTag("mets", "mdRef");
+		tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+		tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+		tag = XMLTools.addAttribute(tag, "MDTYPE", this.name);
+		//tag = XMLTools.addAttribute (tag, "MDTYPE", "OTHER");
+		//tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name); 
+		if (this.id != null) {
+		    tag = XMLTools.addAttribute(tag, "ID", this.id);
+		}
+		tag = XMLTools.makeSingleton(tag);
+	    }
+	return true;
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, NamespaceItem item)
+    { 
+	// Do some default sillinesses
+	//writer.write("         "); 
+	writer.write(XMLTools.getOpenTag(this.name, item.getLabel()));
+	
+	writer.write(item.getValue());
+	
+	writer.write(XMLTools.getCloseTag(this.name, item.getLabel()));
+	return true;
+    }
+    
+    /**
+     *  <p>Indicate whether this metadata is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java	(revision 12189)
@@ -0,0 +1,347 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Implement a simple, unordered, namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class SimpleNamespace extends METSNamespace
+{
+    MultiMap metadataMap;
+    
+    public SimpleNamespace(String name)
+    { 
+	super(name);
+	this.metadataMap = new MultiMap();
+    }
+    
+    public SimpleNamespace(String name, METSLocation location)
+    { 
+	super(name, location);
+	this.metadataMap = new MultiMap();
+    }
+    
+    public SimpleNamespace(String name, Element mdWrapTag)
+    { 
+	super(name);
+	this.metadataMap = new MultiMap();
+	
+	NodeList childNodes = mdWrapTag.getChildNodes();
+	for (int c = 0; c < childNodes.getLength(); c ++) {
+	    // a metadata node
+	    if (childNodes.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+		// get the name of the metadata from the node
+		String metadataLabel = childNodes.item(c).getNodeName();
+		
+		// skip blank labels
+		// TODO: raise an error: metadata item without metadata label set
+		if (metadataLabel == null || metadataLabel.length() == 0) {
+		    continue;
+		}
+		
+		// build the value
+		String metadataValue = "";
+		
+		Element childElement = (Element) childNodes.item(c);
+		for (int i = 0; i < childElement.getChildNodes().getLength(); i ++) {
+		    metadataValue = metadataValue + childElement.getChildNodes().item(i).toString();
+		}
+		
+		// trim any leading namespace identifiers in "namespace:label" metadata
+		if (metadataLabel.startsWith(this.name + ":")) {
+		    metadataLabel = metadataLabel.substring(this.name.length() + 1);
+		}
+		
+		this.addMetadata(metadataLabel, metadataValue);
+	    }
+	}
+    }
+    
+    public boolean validate(String field, String value)
+    { 
+	return true;
+    }
+    
+    /**
+     *  Add a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean addMetadata(String label, String value)
+    {
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataMap.put(label, value);
+	return true;
+    }
+    
+    /**
+     *  Assign a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.  Any existing metadata for that field will
+     *  be destroyed if the new value validates.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean setMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataMap.setOnly(label, value);
+	return true;
+    }
+    
+    /**
+     *  Remove all metadata values for a given field name
+     *
+     *  @param <code>String</code> the field to delete
+     *
+     *  @return <code>boolean</code> whether the field was actually deleted; 
+     *          will return <code>true</code> if the field was already empty.
+     */
+    public boolean removeMetadata(String label)
+    { 
+	this.metadataMap.remove(label);
+	return true;
+    }
+    
+    /**
+     *  Remove a particular incidence of a given metadata field for a document.
+     *  If an exact match for the given value is not found, nothing changes.
+     *  N.B. if a value occurs twice, only the first incidence of it will be
+     *  deleted from the list.
+     *
+     *  @param <code>String</code> the field to have the value removed
+     *  @param <code>String</code> the value to be removed from a given field
+     *
+     *  @return <code>boolean</code> <code>true</code> if an actual metadata text
+     *          is matched against the given value and is thus deleted.
+     */
+    public boolean removeMetadata(String label, String value)
+    { 
+	return this.metadataMap.remove(label, value);
+    }
+    
+    /**
+     *  Get the metadata items for a particular label
+     *
+     *  @param <code>String</code> the label to fetch values for
+     *  
+     *  @return <code>List</code> the list of corresponding values.  May be
+     *          <code>null</code>
+     */
+    public List getMetadata(String label)
+    { 
+	return this.metadataMap.getAll(label);
+    }
+    
+    public Iterator getMetadataNames() 
+    {
+ 	return this.metadataMap.keySet().iterator();
+    }
+    
+    /**
+     *  Write out the metadata to an XML file through a <code>PrintWriter</code>.
+     *
+     *  @param <code>PrintWriter</code> the writer to use.
+     */
+    public boolean write(PrintWriter writer)
+    { 
+	// if this is a non-file block of metadata, write it out in long hand
+	if (this.location == null){
+	    String tag = XMLTools.getOpenTag("mets", "mdWrap");
+	    
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+	    tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+	    if (this.id != null) {
+		this.id = "gsdl"+this.id;
+		tag = XMLTools.addAttribute(tag, "ID", this.id);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println(tag);
+	    
+	    tag = XMLTools.getOpenTag("mets", "xmlData");
+	    writer.print("      ");  //indentation 
+	    writer.println(tag);
+	    
+	    Iterator keys = this.metadataMap.keySet().iterator();
+	    while (keys.hasNext()){
+		String thisKey = keys.next().toString();
+		writer.print("         "); //indentation
+		this.writeItem(writer, thisKey);
+	    }
+	    
+	    writer.print("      ");  //indentation 
+	    writer.println(XMLTools.getCloseTag("mets", "xmlData"));
+	    
+	    writer.print("   ");  //indentation
+	    writer.println(XMLTools.getCloseTag("mets", "mdWrap"));
+	}
+	// otherwise, drop the metadata out in a simplified file-reference
+	// form only
+	else
+	    { 
+		String tag = XMLTools.getOpenTag("mets","mdRef");
+		tag = XMLTools.addAttribute(tag, "LOCTYPE", "URL");
+		tag = XMLTools.addAttribute(tag, "xlink:href", location.getLocation().toString());
+		//tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+		//tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+		tag = XMLTools.addAttribute(tag, "MDTYPE", this.name);	
+		if (this.id != null) {
+		    tag = XMLTools.addAttribute(tag, "ID", this.id);
+		}
+		writer.println(tag);
+		writer.print("   ");  //indentation	
+		writer.println("</mets:mdRef>");
+	    }
+	return true;
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, String label)
+    { 
+	if (this.location == null) {
+	    // just place the metadata in a simple wrapper
+	    Iterator values = this.metadataMap.getAll(label).iterator();
+	    
+	    while (values.hasNext()) {
+		String value = values.next().toString();
+		String metaTagName = "Metadata name=" +'"'+ label +'"';
+		//writer.write(XMLTools.getOpenTag(this.name, label));
+		writer.write(XMLTools.getOpenTag(this.name, metaTagName));
+		writer.write(value);
+		
+		//writer.println(XMLTools.getCloseTag(this.name, label));
+		writer.println(XMLTools.getCloseTag(this.name, "Metadata"));
+	    }
+	}
+	else {
+	    String tag = XMLTools.getOpenTag("mets", "mdRef");
+	    tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	    tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", this.getName());
+	    tag = XMLTools.makeSingleton(tag);
+	    writer.print("      "); //indentation 
+	    writer.println(tag);
+	}
+	return true;
+    }
+    
+    /**
+     *  Write out the metadata to an SQL database through a <code>GS3SQLConnection</code>.
+     *
+     *  @param <code>GS3SQLConnection</code> the SQL database to use.
+     */
+    public boolean writeSQL(int parentId, GS3SQLConnection connection)
+    { 
+	// write the general stuff
+	String sqlId;
+	
+	if (!super.writeSQL(parentId, connection)) {
+	    return false;
+	}
+	
+	Statement statement = null;
+	ResultSet results = null;
+	try {
+	    statement = connection.createStatement();
+	    if (this.id == null) {
+		GS3SQLSelect select = new GS3SQLSelect("namespaces");
+		select.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE)));
+		select.addField("NamespaceRef");
+		results = statement.executeQuery(select.toString());
+		if (results.first()) {
+		    sqlId = Integer.toString(results.getInt("NamespaceRef"));
+		} else {
+		    statement.close();
+		    return false;
+		}
+	    }
+	    else {
+		sqlId = this.id;
+	    }
+	    
+	    // clear all the existing metadata items for this namespace
+	    GS3SQLDelete delete = new GS3SQLDelete("mdvalues");
+	    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", sqlId, GS3SQLField.INTEGER_TYPE));
+	    delete.setWhere(where);
+	    statement.execute(delete.toString());
+	    
+	    // write out the metadata for this namespace
+	    Iterator keys = this.metadataMap.keySet().iterator();
+	    while (keys.hasNext()){
+		String thisKey = keys.next().toString();
+		
+		Iterator values = this.metadataMap.getAll(thisKey).iterator();
+		
+		while (values.hasNext()){
+		    String value = values.next().toString();
+		    
+		    GS3SQLInsert insert = new GS3SQLInsert("mdvalues");
+		    insert.addValue("NamespaceRef", sqlId, GS3SQLField.INTEGER_TYPE);
+		    insert.addValue("Label", thisKey);
+		    insert.addValue("Value", value);
+		    statement.execute(insert.toString());
+		}
+	    }
+	    statement.close();
+	}
+	catch (SQLException sql) {
+	    System.out.println("SimpleNamespace.writeSQL(): "+sql);
+	}
+	return true;
+    }
+    
+    
+    /**
+     *  <p>Indicate whether this metadata is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java	(revision 12189)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  TODO: complete a class to create file identifiers
+ */
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class StructureIdentifierFactory extends AbstractIdentifierFactory
+{
+    public static final String STRUCTURE_IDENTIFIER_PRELUDE = "DS";
+    
+    public StructureIdentifierFactory()
+    {
+	super(STRUCTURE_IDENTIFIER_PRELUDE);
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java	(revision 12189)
@@ -0,0 +1,166 @@
+package org.greenstone.gsdl3.gs3build.notifier;
+
+import java.io.*;
+import java.net.*;
+import java.sql.*;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public class NotifierManager {
+
+    public NotifierManager() {
+    }
+    
+    public void detectEvents(CollectionManager collManager) {
+	String notifyHost = collManager.getNotifyHost();
+	if (notifyHost == null)
+	    return;
+	
+	System.out.println("detecting events");
+	
+	String collectionName = collManager.getCollectionName();
+	String host = "http://localhost:8080/soap/servlet/rpcrouter";
+	
+	String lastBuild = dateToSqlString(collManager.getBuildDate());
+	System.out.println("last build was " + lastBuild);
+        try {
+	    Statement statement = collManager.getDatabase().createStatement();
+            // detect all new documents. A document is new if and only if
+            // AccessionDate >= CollectionLastRebuiltDate
+	    ResultSet results = statement.executeQuery("SELECT DocID FROM document WHERE DocType != 'GSMETADATA' AND AccessionDate >= " + lastBuild);
+	    notify(collectionName, host, notifyHost, results, collManager.getDatabase(), collManager.getBuildDate(), "new_document");
+	    
+	    // detect modified documents. A document is modified if and only if
+	    // AccessionDate < CollectionLastRebuiltDate (ie, it is not new) and 
+	    // IndexedDate >= CollectionLastRebuiltDate 
+	    results = statement.executeQuery("SELECT DocID FROM document WHERE DocType != 'GSMETADATA' AND AccessionDate < " + lastBuild + " AND IndexedDate >= " + lastBuild);
+	    notify(collectionName, host, notifyHost, results, collManager.getDatabase(), collManager.getBuildDate(), "document_modified");
+	    
+	    statement.close();
+	    // TODO deleted docs?
+        } catch (SQLException e) {
+	    // TODO Auto-generated catch block
+	    e.printStackTrace();
+        }
+    }
+    
+    /**
+     * @param collName
+     * @param hostURL
+     * @param results
+     * @param date
+     * @param eventType TODO
+     * @throws MalformedURLException
+     * @throws IOException
+     * @throws ProtocolException
+     * @throws SQLException
+     */
+    private void notify(String collName, String hostURL, String notifyHost, ResultSet results, GS3SQLConnection database, Date date, String eventType) {
+	try {
+	    System.out.println("notifyHost is " + notifyHost);
+	    URL url = new URL("http://" + notifyHost + "/alerting/service");
+	    System.out.println("trying to send to " + url);
+	    
+	    while(results.next()) {
+		System.out.println(eventType + ": " + results.getString("DocID"));
+	            
+                String documentID = results.getString("DocID");
+                
+		try {
+                    HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // TODO what about proxies?
+                    conn.setUseCaches(false);
+                    conn.setDoInput(true);
+                    conn.setDoOutput(true);
+                    conn.setRequestMethod("POST");
+                    conn.connect();
+                    // Construct data
+                    OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream());
+                    
+                    writer.write("action=receiveEvent&");
+                    writer.write("documentID=");
+                    writer.write(URLEncoder.encode(documentID, "UTF-8"));
+                    writer.write("&type=");
+                    writer.write(URLEncoder.encode(eventType, "UTF-8"));
+                    writer.write("&collectionID=");
+                    writer.write(URLEncoder.encode(collName, "UTF-8"));
+                    writer.write("&hostID=");
+                    writer.write(URLEncoder.encode(hostURL, "UTF-8"));
+                    writer.write("&document_title=");
+                    writer.write(URLEncoder.encode(getDocumentTitle(documentID, database), "UTF-8"));
+                    writer.write("&document_url=");
+                    writer.write(URLEncoder.encode("http://localhost:8080/gsdl3/library?a=d&c=" + collName + "&d=" + documentID, "UTF-8"));
+                    writer.write("&timestamp=");
+                    writer.write(URLEncoder.encode(new SimpleDateFormat("yyyy-MM-dd HH:mm z").format(date), "UTF-8"));
+                    writer.flush();
+                    System.out.println(conn.getResponseCode());
+                    InputStream fromReceiver = conn.getInputStream();
+                    int i;
+                    while ((i = fromReceiver.read()) != -1) {
+                        System.out.write(i);
+                    }
+                    conn.disconnect();
+                } catch (ProtocolException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                } catch (UnsupportedEncodingException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                } catch (IOException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                }
+	    }
+	} catch (SQLException e) {
+	    // TODO Auto-generated catch block
+	    e.printStackTrace();
+	} catch (MalformedURLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * @param documentID
+     * @return
+     * @throws SQLException
+     */
+    private String getDocumentTitle(String documentID, GS3SQLConnection database) {
+        System.out.println("getting title for document " + documentID + " from database");
+
+        try {
+	    Statement statement = database.createStatement();
+            String sqlString = "SELECT mdvalues.value " +
+		"FROM structure s JOIN divisions d ON (s.structureref = d.parentref) " +
+		"                 JOIN divisionmetarefs USING (divisionref) " + 
+		"                 JOIN metadata m USING (metaid) " +
+		"                 JOIN namespaces USING (metadataref) " + 
+		"                 JOIN mdvalues USING (namespaceref) " +
+		"  WHERE mdvalues.label = 'Title' AND s.docid = '" + documentID + "'" + 
+		"                                 AND s.structureType = 'Whole Document' " +
+		"                                 AND d.parenttype = 'Structure' " + 
+		"                                 AND m.docID = s.docid;";
+            ResultSet results = statement.executeQuery(sqlString);
+            if (results.first()) {
+                String title = results.getString("value");
+                System.out.println("title is " + title);
+                return title;
+            }
+	    statement.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        return "";
+    }
+
+    /**
+     * @param date
+     * @return
+     */
+    private String dateToSqlString(Date date) {
+        return new SimpleDateFormat("yyyyMMddHHmmss").format(date);
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java	(revision 12189)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class ChildWrapper
+{
+    int minOccurs;
+    int maxOccurs;
+
+
+    /**
+     *  The default number of occurrences for an element is 1 (one) for
+     *  both minimum and maximum counts.
+     */
+    public ChildWrapper()
+    {
+	this.minOccurs = 1;
+	this.maxOccurs = 1;
+    }
+
+    public void setMinOccurs(int minOccurs)
+    {
+	this.minOccurs = minOccurs;
+    }
+
+    public void setMaxOccurs(int maxOccurs)
+    {
+	this.maxOccurs = maxOccurs;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java	(revision 12189)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.*;
+
+public class EnumConstraint
+{
+    List options;
+
+    public EnumConstraint()
+    { this.options = new ArrayList();
+    }
+
+  public void addOption(String option)
+  {
+    this.options.add(option);
+  }
+
+  public boolean validate(String value)
+  {
+	  return this.options.contains(value);
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java	(revision 12189)
@@ -0,0 +1,34 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class RangeConstraint
+{
+    public static final int RANGE_MIN_EXCLUSIVE = 0x00;
+    public static final int RANGE_MAX_EXCLUSIVE = 0x01;
+    public static final int RANGE_MIN_INCLUSIVE = 0x10;
+    public static final int RANGE_MAX_INCLUSIVE = 0x11;
+
+    int type;
+    double value;
+
+    public RangeConstraint(int type, double value)
+    {
+	    this.type = type;
+	    this.value = value;
+    }
+
+  public boolean validate(double value)
+  {
+  	switch (this.type) {
+	  case RANGE_MIN_EXCLUSIVE:
+	    return (value > this.value);
+  	case RANGE_MAX_EXCLUSIVE:
+	    return (value < this.value);
+   case RANGE_MIN_INCLUSIVE:
+	    return (value >= this.value);
+   case RANGE_MAX_INCLUSIVE:
+	    return (value <= this.value);
+	}
+	return false;
+}
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/Schema.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/Schema.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/Schema.java	(revision 12189)
@@ -0,0 +1,78 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class Schema
+{
+	List children;
+
+	public Schema(Node node)
+	{
+		this.children = new ArrayList();
+
+		NodeList elements = node.getChildNodes();
+
+		System.out.println("Schema has " + elements.getLength() + " children");
+		for (int e = 0; e < elements.getLength(); e++) {
+			System.out.println("Subcomponent " + e);
+			// TODO: actually test what is being parsed here!
+			if (elements.item(e).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+				continue;
+			}
+
+			System.out.println(elements.item(e));
+
+			Element element = (Element) elements.item(e);
+
+			String elementName = element.getLocalName();
+			String fullName = element.getNodeName();
+			String valueName = element.getNodeValue();
+
+			System.out.println(elementName + " " + fullName + " " + valueName);
+
+			if (elementName == null) {
+				elementName = fullName;
+			}
+
+			System.out.println(elementName);
+			elementName = SchemaNode.getElementName(elementName);
+
+			if (elementName.equals("element")) {
+
+				SchemaElement thisElement = new SchemaElement(element);
+/*			
+				NodeList attributes = element.getElementsByTagName("attribute");
+
+				for (int i = 0; i < attributes.getLength(); i++) {
+			    Element attribute = (Element) attributes.item(i);
+					// handle that attribute, create its node and check what you get out
+					try {
+						SchemaAttribute thisAttribute = new SchemaAttribute(attribute);
+						thisElement.addAttribute(thisAttribute);
+					}
+					catch (SchemaException ex) {
+						// do nothing at the moment - though a malformed attribute should be
+						// cause for throwing everything in the air
+					}
+				}*/
+				this.children.add(thisElement);
+			}
+			else {
+				try {
+					Object child = SchemaNodeFactory.makeNode(element);
+					if (child != null) {
+						this.children.add(child);
+					}
+				}
+				catch (SchemaException metaEx)
+				{
+				}
+			}
+		}
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java	(revision 12189)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaAll extends SchemaNode
+{
+	List compulsory;
+
+	public SchemaAll(Element element) throws SchemaException
+	{	super(element);
+	}
+
+	protected List _allocate()
+	{	this.compulsory = new ArrayList();
+		return this.compulsory;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java	(revision 12189)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+public class SchemaAttribute
+{ 
+	public final static String ATTRIBUTE_TYPE_STRING = "String";
+
+  String name;
+  String type;
+  boolean required;
+
+	public SchemaAttribute(Element element) throws SchemaException
+	{
+		// get the name - things are totally unworkable if it doesn't exist!
+		this.name = element.getAttribute("name");
+		if (this.name == null) {
+			throw new SchemaException("All attributes must have names");
+		}
+
+		// ids are quite important in some metadata schema - copy that too
+		String id = element.getAttribute("id");
+
+		// get the required field - we don't actually cope with "prohibited" at the moment!
+		String use = element.getAttribute("use");
+		this.required = (use != null && use.equals("required"));
+
+		// and the type - we don't reflect any further upon this, actually
+		this.type = element.getAttribute("type");
+		if (this.type == null) {
+			this.type = SchemaAttribute.ATTRIBUTE_TYPE_STRING;
+		}
+
+		// Constraints should now be added, but at present that is not done, as we
+		// are non-validating!
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java	(revision 12189)
@@ -0,0 +1,23 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaChoice extends SchemaNode
+{
+	List alternatives;
+
+	public SchemaChoice(Element element) throws SchemaException
+	{
+		super(element);
+	}
+
+	protected List _allocate()
+	{	this.alternatives = new ArrayList();
+		return this.alternatives;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java	(revision 12189)
@@ -0,0 +1,65 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaComplexType implements SchemaType
+{
+	public String name;
+	public List children;
+
+	public SchemaComplexType(Element element) throws SchemaException
+	{
+		this.name = element.getAttribute("name");
+		if (name == null)	
+		{ throw new SchemaException("Type defined without name");
+		}
+
+		this.children = new ArrayList();
+
+		NodeList childNodes = element.getChildNodes();
+		for (int i = 0; i < childNodes.getLength(); i ++) {
+			Node node = childNodes.item(i);
+
+			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+				String nodeName = node.getNodeName();
+
+				if (nodeName == null) {
+					continue;
+				}
+
+				nodeName = SchemaNode.getElementName(nodeName);
+				// a simple-based type
+				if (node.getNodeName().equals("simpleContent")) {
+					// we're really just a simple type with attributes - nothing more to do
+				}
+				// a complex-based type
+				else if (node.getNodeName().equals("complexContent")) {
+					// we just contain elements, no text
+				}
+				// the following pretty much speak for themselves
+				else {
+					try {
+						Object child = SchemaNodeFactory.makeNode((Element) node);
+						if (child != null) {
+							this.children.add(child);
+						}
+					}
+					catch (SchemaException ex)
+					{	
+					}
+				}
+			}
+		}
+
+		SchemaTypeMap.theMap.addType(this);
+	}
+	
+	public String getName()
+	{	return this.name;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java	(revision 12189)
@@ -0,0 +1,104 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.*;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SchemaElement
+{ 
+	// the following may contain duplicates when compared to SchemaAttribute, but this is
+	// deliberate...
+	public static final String ELEMENT_TYPE_STRING = "String";
+
+  String name;
+	String type;
+  List   attributes;
+  List   childElements;
+	int    minOccurs;
+	int    maxOccurs;
+
+	public SchemaElement(Element element)
+	{
+		// get the name of this element
+		this.name = element.getAttribute("name");
+		if (this.name == null) {
+			// TODO: this is a reference object
+			String ref = element.getAttribute("ref");
+
+			if (ref != null) {
+				// TODO: fetch the actual object
+			}
+			else {
+				// TODO: abort
+			}
+		}
+
+		this.type = element.getAttribute("type");
+		if (this.type == null || this.type.length() == 0)
+		{ this.type = SchemaElement.ELEMENT_TYPE_STRING;
+		}
+
+		String temp;
+		temp = element.getAttribute("minOccurs");
+		if (temp != null && temp.length() > 0) {
+			this.minOccurs = Integer.parseInt(temp);
+			this.minOccurs = 1;
+		}
+		temp = element.getAttribute("maxOccurs");
+		if (temp != null && temp.length() > 0) {
+			this.maxOccurs = Integer.parseInt(temp);
+		}
+		else {
+			this.maxOccurs = 1;
+		}
+
+		this.expand(element);
+	}
+
+	private void expand(Element element)
+	{ // TODO
+		// get the first child element
+		NodeList children = element.getChildNodes();
+		if (children != null) {
+			for (int i = 0; i < children.getLength(); i ++)
+			{ if (children.item(i).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+					// TODO: raise an error
+					continue;
+				}
+
+				String name = children.item(i).getNodeName();
+				if (name == null) {
+					continue;
+				}
+				
+				try {
+					Object child = SchemaNodeFactory.makeNode((Element) children.item(i));
+					if (child != null) {
+						this.childElements.add(child);
+					}
+				}
+				catch (SchemaException ex)
+				{
+				}
+			}
+		}
+	}
+
+  public SchemaElement(String name)
+  { this.name = name;
+    this.attributes = new ArrayList();
+    this.childElements = new ArrayList();
+  }
+
+  public void addAttribute(SchemaAttribute attribute)
+  {
+	  this.attributes.add(attribute);
+  }
+
+    public void addElement(SchemaElement element)
+    {
+			this.childElements.add(element);
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaException.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaException.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaException.java	(revision 12189)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class SchemaException extends Exception
+{
+	public SchemaException(String name)
+	{
+		super(name);
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java	(revision 12189)
@@ -0,0 +1,21 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaGroup extends SchemaNode
+{
+	List members;
+
+  public SchemaGroup(Element element) throws SchemaException
+  {
+		super(element);
+  }
+
+	public List _allocate()
+	{	this.members = new ArrayList();
+		return this.members;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java	(revision 12189)
@@ -0,0 +1,68 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+
+public abstract class SchemaNode
+{
+	public SchemaNode(Element element) throws SchemaException
+	{	List childList = this._allocate();
+		this.initialise(element, childList);
+		System.out.println(element.getNodeName());
+	}
+	
+	abstract protected List _allocate();
+
+	protected void initialise(Element element, List childList) throws SchemaException
+	{
+		// TODO: check minimum, maximum occurrences etc.
+
+		NodeList children = element.getChildNodes();
+		if (children != null) {
+			System.out.println("  " + children.getLength() + " children");
+			for (int i = 0; i < children.getLength(); i ++)
+			{ if (children.item(i).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+					Node child = children.item(i);
+
+					String type = child.getNodeName();
+					System.out.println("Child is " + type);
+
+					if (type == null) {
+						throw new SchemaException("Badly defined child - no name");
+					}
+
+					type = SchemaNode.getElementName(type);
+
+					if (type.equals("element")) {
+						SchemaElement elementChild = new SchemaElement((Element) children.item(i));
+						childList.add(elementChild);
+					}
+					else {
+						Object elementChild = SchemaNodeFactory.makeNode((Element) child);
+						if (elementChild != null) {
+							childList.add(elementChild);
+						}
+					}
+					// TODO: do an abstraction between here and SchemaElement
+				}
+			}
+		}
+	}
+
+	public static String getElementName(String fullname)
+	{	if (fullname.indexOf(':') >= 0) {
+			return fullname.substring(fullname.indexOf(':') + 1);
+		}
+		return fullname;
+	}
+
+	public static String getNamespace(String fullname)
+	{	if (fullname.indexOf(':') >= 0) {
+			return fullname.substring(0, fullname.indexOf(':'));
+		}
+		return null;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java	(revision 12189)
@@ -0,0 +1,33 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+public class SchemaNodeFactory
+{
+	public static Object makeNode(Element element) throws SchemaException
+	{
+		String elementName = element.getNodeName();
+
+		System.out.println("Element name " + elementName);
+
+		elementName = SchemaNode.getElementName(elementName);
+
+		if (elementName.equals("choice")) {			
+			return new SchemaChoice(element);
+		}
+		else if (elementName.equals("group")) {
+			return new SchemaGroup(element);
+		}
+		else if (elementName.equals("all")) {
+			return new SchemaAll(element);
+		}
+		else if (elementName.equals("sequence")) {
+			return new SchemaSequence(element);
+		}
+		else if (elementName.equals("element")) {
+			return new SchemaElement(element);
+		}
+		return null;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java	(revision 12189)
@@ -0,0 +1,73 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.io.*;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+public class SchemaParser
+{
+	public SchemaParser(File file, String namespace)
+	{
+	  try {
+			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+			DocumentBuilder builder = factory.newDocumentBuilder();
+			Document document = builder.parse(file);
+
+			// TODO: get all the types in the tree
+
+			// TODO: do a traverse, and thus cope with elements-within-elements if needsbe, but
+			// this shouldn't happen except in a directly defined situation - actually doing the
+			// parsing in part inside each node would work well provided one checked for a node
+			// having already been done...
+			System.out.println("Read");
+
+			NodeList schemas = document.getElementsByTagName("xs:schema");
+			if (schemas.getLength() == 0) {
+				schemas = document.getElementsByTagName("xsd:schema");
+			}
+			if (schemas.getLength() == 0) {
+				schemas = document.getElementsByTagName("schema");
+			}
+
+			System.out.println(" " + schemas.getLength() + " schemas");
+
+			for (int s = 0; s < schemas.getLength(); s ++) {
+				Schema schema = new Schema(schemas.item(s));
+			}
+		}
+		catch (FactoryConfigurationError e) {
+			System.out.println(e);
+		}
+		catch (ParserConfigurationException ex) {
+			System.out.println(ex);
+		}
+		catch (SAXException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println(ex);
+		}
+	}
+
+	public static void main(String args[])
+	{	try
+		{	File file = new File(args[0]);
+
+			SchemaParser parser = new SchemaParser(file, "");			
+		}
+		catch (Exception ex)
+		{ System.out.println(ex);
+		}
+		return;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java	(revision 12189)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.HashMap;
+
+public class SchemaRefMap
+{
+	HashMap map;
+
+	public SchemaRefMap()
+	{
+		this.map = new HashMap();
+	}
+
+	public void addItem(String reference, Object item)
+	{
+		this.map.put(reference, item);
+	}
+
+	public Object getItem(String reference)
+	{	return this.map.get(reference);
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java	(revision 12189)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaSequence extends SchemaNode
+{
+	List sequence;
+
+	public SchemaSequence(Element element) throws SchemaException
+	{	super(element);
+	}
+
+	public List _allocate()
+	{	this.sequence = new ArrayList();
+		return this.sequence;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java	(revision 12189)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SchemaSimpleType implements SchemaType
+{
+	String name;
+
+	public void SchemaSimpleType(Element element) throws SchemaException
+	{
+		this.name = element.getAttribute("name");
+		if (name == null)	
+		{ throw new SchemaException("Type defined without name");
+		}
+
+		NodeList childNodes = element.getChildNodes();
+		for (int i = 0; i < childNodes.getLength(); i ++) {
+			Node node = childNodes.item(i);
+
+			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+			}
+		}
+	}
+
+	public String getName()
+	{	return this.name;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaType.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaType.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaType.java	(revision 12189)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public interface SchemaType
+{
+	public String getName();
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java	(revision 12189)
@@ -0,0 +1,26 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SchemaTypeMap
+{
+	Map map;
+	public static final SchemaTypeMap theMap;
+
+	static {
+		theMap = new SchemaTypeMap();
+	}
+
+	public SchemaTypeMap()
+	{	this.map = new HashMap();
+	}
+
+	public void addType(SchemaType type)
+	{	this.map.put(type.getName(), type);
+	}
+
+	public SchemaType getType(String name)
+	{	return (SchemaType) this.map.get(name);
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/DOMUtils.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/DOMUtils.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/DOMUtils.java	(revision 12189)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+public class DOMUtils
+{
+  public static String getNodeAttribute(Node node, String attribute)
+  {
+    NamedNodeMap atts = node.getAttributes();
+    if (atts == null) {
+      return null;
+    }
+    Node attributeNode = atts.getNamedItem(attribute);
+    if (attributeNode == null) {
+      return null;
+    }
+    return attributeNode.getNodeValue();
+  }
+    
+  public static String getNodeChildText(Node node)
+  {
+    StringBuffer fieldName = new StringBuffer();
+
+    NodeList fieldChildren = node.getChildNodes();
+    for (int f = 0; f < fieldChildren.getLength(); f ++) {
+      if (fieldChildren.item(f).getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	fieldName.append(fieldChildren.item(f).getNodeValue());
+      }
+    }
+    return fieldName.toString();
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/FileTrie.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/FileTrie.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/FileTrie.java	(revision 12189)
@@ -0,0 +1,145 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class FileTrie
+{	String   stem;
+	List     childTries;
+	List     childDocs;
+	FileTrie parent;
+
+	public FileTrie(String stem)
+	{	this.childTries = new ArrayList();
+		this.childDocs  = new ArrayList();
+		this.stem = stem;
+		this.parent = null;
+	}
+
+	public void setParent(FileTrie trie)
+	{	this.parent = trie;
+	}
+
+	public FileTrie getParent()
+	{	return this.parent;
+	}
+
+	public void addChild(FileTrie trie)
+	{	trie.setParent(this);
+		this.childTries.add(trie);
+	}
+
+	public void replaceChild(FileTrie oldChild, FileTrie newChild)
+	{	for (int i = 0; i < this.childTries.size(); i ++)
+		{	FileTrie thisChild = (FileTrie) this.childTries.get(i);
+			if (thisChild == oldChild)
+			{	this.childTries.set(i, newChild);
+				return;
+			}
+		}
+	}
+
+	public void addDocument(DocumentID documentID)
+	{	this.childDocs.add(documentID);
+	}
+
+	public boolean postDocument(String filename, DocumentID documentID)
+	{	int common = 0;
+		int i;
+		
+		while (common < filename.length() &&
+		       common < this.stem.length() &&
+					  filename.charAt(common) == stem.charAt(common))
+		{	common ++;
+		}
+
+		if (common == 0)
+		{ return false;
+		}
+
+		if (common == this.stem.length())
+		{	// post it in this node
+			filename = filename.substring(this.stem.length());
+
+			// select which sub-trie to attempt...
+			for (i = 0; i < this.childTries.size(); i ++)
+			{	FileTrie child = (FileTrie) this.childTries.get(i);
+				
+				if (child.postDocument(filename, documentID))
+				{	break;
+				}
+			}
+
+			if (i == this.childTries.size())
+			{ FileTrie newChild = new FileTrie(filename);
+				this.addChild(newChild);
+
+				newChild.addDocument(documentID);
+			}
+		}
+		else
+		{	// divide this node at the 'common' point...
+			FileTrie parentTrie = new FileTrie(this.stem.substring(0, common));
+			this.parent.replaceChild(this, parentTrie);
+
+			// truncate our root
+			this.stem = this.stem.substring(common);
+
+			// add ourselves to our new parent
+			parentTrie.addChild(this);
+
+			// trim back the file name
+			filename = filename.substring(common);
+
+			// recurse into posting the document
+			this.postDocument(filename, documentID);
+		}
+		return true;
+	}
+
+	public boolean postFile(String filename)
+	{	return this.postDocument(filename, null);
+	}
+
+	public List getDocuments(String filename)
+	{	FileTrie leaf = this.getNode(filename);
+		return leaf.childDocs;
+	}
+
+	public FileTrie getNode(String filename)
+	{	int common = 0;
+		int i;
+		
+		while (common < filename.length() &&
+		       common < this.stem.length() &&
+					 filename.charAt(common) == stem.charAt(common))
+		{	common ++;
+		}
+
+		if (common == 0)
+		{ return null;
+		}
+
+		if (common == this.stem.length())
+		{	if (common == filename.length())
+			{	return this;
+			}
+
+			filename = filename.substring(this.stem.length());
+
+			// select which sub-trie to attempt...
+			for (i = 0; i < this.childTries.size(); i ++)
+			{	FileTrie child = (FileTrie) this.childTries.get(i);
+				
+				child = child.getNode(filename);
+
+				if (child != null)
+				{	return child;
+				}
+			}
+		}
+		return null;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java	(revision 12189)
@@ -0,0 +1,111 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class GS2TextFileHandler
+{
+  String  content;
+  String  line;
+
+  public GS2TextFileHandler(String content)
+  { this.content = content;
+  }
+    
+  public boolean hasMore()
+  { return this.line != null;
+  }
+
+  public boolean hasMoreLines()
+  { return this.content != null;
+  }
+
+  public String getEntry()
+  { return this.getEntry(false);
+  }
+
+  public String getEntry(boolean breakSpace)
+  { String reply;
+    int start, tab = 0;
+    boolean quoted = false;
+
+    if (!this.hasMore()) {
+      return null;
+    }
+
+    start = 0;
+    while (start < this.line.length() &&
+	   this.line.charAt(start) == ' ') {
+      start ++;
+    }
+
+    if (start == this.line.length()) {
+      this.line = null;
+      return null;
+    }
+    
+    if (this.line.charAt(start) == '"') {
+      quoted = true;
+      breakSpace = false;
+      start ++;
+    }
+    tab = start;
+
+    while (tab != this.line.length() &&
+	   this.line.charAt(tab) != '\t' &&
+	   !(quoted && this.line.charAt(tab) == '"') &&
+	   !(this.line.charAt(tab) == ' ' && breakSpace))
+    { tab ++;
+    }
+
+    if (start > 0) {
+      this.line = this.line.substring(start);
+      tab -= start;
+    }
+    
+    if (tab == this.line.length()) {
+      reply = this.line;
+      this.line = null;
+    }
+    else {
+      reply = this.line.substring(0, tab);
+      this.line = this.line.substring(tab+1);
+    }
+
+    reply.trim();
+    
+    return reply;
+  }
+
+  public String getLine()
+  { if (this.content == null) {
+      this.line = null;
+      return null;
+    }
+	
+    do {
+      int eol = this.content.indexOf('\n');
+      if (eol < 0) {
+	this.line = this.content;
+	this.content = null;
+      }
+      else {
+	this.line = this.content.substring(0, eol);
+	this.content = this.content.substring(eol+1);
+	while (this.content.length() > 0 &&
+	       this.content.charAt(0) < ' ')
+	{ this.content = this.content.substring(1);
+	}
+      }
+
+      if (this.line != null) {
+	this.line.trim();
+      }
+      if (this.line.length() > 0 &&
+	  this.line.charAt(this.line.length() - 1) == (char) 13) {
+	this.line = this.line.substring(0, this.line.length()-1);
+	this.line.trim();
+      }
+    } while (this.content != null && this.line != null && this.line.length() == 0);
+    
+    return this.line;
+  }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java	(revision 12189)
@@ -0,0 +1,383 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLBlock
+{
+    Vector content;
+    int    tagpos;
+    int    endpos;
+
+    public HTMLBlock(int docpos)
+    { 
+	this.content = new Vector(1);
+	this.tagpos  = docpos;
+    }
+    
+    public HTMLBlock(int docpos, int endpos)
+    { 
+	this.content = new Vector(1);
+	this.tagpos  = docpos;
+	this.endpos  = endpos;
+    }
+
+    public void addTag(HTMLTag tag)
+    { 
+	if (tag.endPos() > this.endpos){
+	    this.endpos = tag.endPos();
+	}
+	content.addElement(tag);
+    }
+
+    /**
+     *  @deprecccate
+     public void addText(String text)
+     { content.addElement(text);
+     }
+    */
+
+    public void addText(HTMLCText text)
+    { 
+	if (text.endPos() > this.endpos){
+	    this.endpos	= text.endPos();
+	}
+	content.addElement(text);
+    }
+    
+    /**
+     * @return HTML encoded <code>String</code> of the document
+     */
+    public String contentString()
+    { 
+	int member;
+	StringBuffer reply;
+	String type;
+	
+	reply = new StringBuffer();
+	for (member = 1; member < content.size() - 1; member ++){
+	    /*if (content.elementAt(member) instanceof String){
+	      reply.append((String) content.elementAt(member));
+	      }
+	      else 
+	  */
+	    if (content.elementAt(member) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(member)).toString());
+	    }
+	}
+	return reply.toString();
+    }
+    
+    /**
+     * Returns document position of start of the block
+     */
+    public int startPos()
+    { 
+	return this.tagpos;
+    }
+
+    /**
+     * Returns document position of end of the block
+     */
+    public int endPos()
+    { 
+	return this.tagpos;
+    }
+
+    /**
+     * @return head (first) tag of the block
+     */
+    public HTMLTag headTag()
+    { 
+	return ((HTMLTag) content.elementAt(0));
+    }
+
+    /**
+     * @return name (as a <code>String</code>) of the tag at the head of the block
+     */
+    public String headTagName()
+    { 
+	return ((HTMLTag) content.elementAt(0)).tagName();
+    }
+
+    /**
+     * @return name of the tail (last) tag of the block as a <code>String</code>
+     */
+    public String tailTagName()
+    { 
+	return (((HTMLTag) content.elementAt(content.size() - 1)).tagName());
+    }
+	
+	
+    /**
+     *  @return: starting element character position of the item-th item
+     */
+    private int elementStartPos(int item)
+    { 
+	Object element;
+	int	pos = -1;
+		
+	element = this.content.elementAt(item);
+	if (element instanceof HTMLTag){
+	    pos	= ((HTMLTag) element).startPos();
+	}
+	else if (element instanceof HTMLCText){
+	    pos	= ((HTMLCText) element).startPos();
+	}
+	return pos;		
+    }
+    
+    /**
+     *  @return: ending element character position of the item-th item
+     */
+    private int elementEndPos(int item)
+    { 
+	Object	element;
+	int		pos = -1;
+	
+	element = this.content.elementAt(item);
+	if (element instanceof HTMLTag){
+	     pos = ((HTMLTag) element).endPos();
+	}
+	else if (element instanceof HTMLCText){
+	     pos = ((HTMLCText) element).endPos();
+	}
+	return pos;		
+    }
+
+    /**
+     *  @return the HTMLBlock of the indicated subitems
+     */
+    public HTMLBlock subBlock(int startitem, int enditem)
+    { 
+	int       i;
+	Object    element;
+	HTMLBlock reply;
+	reply = new HTMLBlock(this.elementStartPos(startitem),
+			      this.elementEndPos(enditem-1));
+	// copy all the bits
+	for (i = startitem; i < enditem; i++){
+	    element = this.content.elementAt(i);
+	    if (element instanceof HTMLTag){
+		reply.addTag((HTMLTag) element);
+	    }
+	    else if (element instanceof HTMLCText){
+		reply.addText((HTMLCText) element);
+	    }
+	/*
+	  else if (element instanceof String)
+	  {	reply.addText((String) element);
+	  }
+	*/
+	}
+	return reply;
+    }
+
+    /**
+     * Return the position of the given <code>HTMLBlock</code> within this block
+     * @return the tag index of the child block; or < 0 if not a chile
+     */
+    public int find(HTMLBlock subblock)
+    { 
+	int start, end, at, attag;
+	HTMLTag tag;
+	
+	start = 0;
+	end = this.content.size();
+	while (start != end){
+	     /* compare positions */
+	    at = (start + end) / 2;
+	    attag = at;
+	    while (attag < end &&
+		   (content.elementAt(attag) instanceof HTMLTag) == false){
+		attag ++;
+	    }
+	    
+	    if (attag == end){
+		end = at;
+		continue;
+	    }
+	    tag =	(HTMLTag) content.elementAt(attag);
+	    if (tag.startPos() > subblock.startPos()){
+		end = at;				
+	    }
+	    else if (tag.startPos() < subblock.startPos()){
+		start = attag + 1;
+	    }
+	    else { 
+		return attag;
+	    }
+	}
+	return -1;
+    }
+    
+    /**
+     * @return The raw HTML of the block
+     */
+    public String HTMLString()
+    { 
+	int loop;
+	StringBuffer reply;
+
+	reply	= new StringBuffer();
+	for (loop = 0; loop < this.content.size(); loop ++){
+	    /*
+	      if (content.elementAt(loop) instanceof String)
+	      {	reply.append((String) content.elementAt(loop));
+	      }
+	      else 
+	    */
+	    if (content.elementAt(loop) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(loop)).toString());
+	    }
+	    else if (content.elementAt(loop) instanceof HTMLTag){
+		reply.append(((HTMLTag) content.elementAt(loop)).toString());
+	    }
+	}
+	return reply.toString();
+    }
+    
+    /**
+     * @return The text of the block - no HTML tags.  If no text is present, any
+     *         <code>alt</code> information for <code>img</code> tags will be given 
+     *         instead
+     */
+    public String toString()
+    {
+	int member;
+	StringBuffer reply;
+	StringBuffer ireply;
+	
+	reply = new StringBuffer("");
+	ireply = new StringBuffer("");
+	for (member = 0; member < content.size(); member ++){
+	     /*if (content.elementAt(member) instanceof String)
+	       { reply.append((String) content.elementAt(member));
+	       }
+	     */
+	    if (content.elementAt(member) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(member)).toString());
+	    }
+	    else if (content.elementAt(member) instanceof HTMLTag){
+		if (((HTMLTag) content.elementAt(member)).tagName().equals("img")){
+		    ireply.append(((HTMLTag) content.elementAt(member)).idValue("alt"));
+		}
+	    }
+	}
+	reply = HTMLCText.cleanString(reply);
+	ireply = HTMLCText.cleanString(ireply);
+	//		reply = reply.trim();
+	if (reply == null || reply.length() == 0){
+	    return ireply.toString();
+	}
+	return reply.toString();
+    }
+    
+    public HTMLObject elementAt(int at)
+    { 
+	return (HTMLObject) this.content.elementAt(at);
+    }
+
+    /**
+     * @return the number of elements in the block
+     */
+    public int size()
+    { 
+	return this.content.size();
+    }
+
+    /**
+     * @return an enumeration of all elements in the block
+     */
+    public Enumeration elements()
+    { 
+	Enumeration enumer;
+  
+	enumer = new HTMLBlockEnumerator(this);
+	return enumer;
+    }
+
+    /**
+     * @return an enumeration of all tags in the block
+     */
+    public Enumeration tagElements()
+    { 
+	Enumeration enumer;
+	
+	enumer = new HTMLBlockTagEnumerator(this);
+	return enumer;
+    }
+
+    /**
+     * Finalization method
+     */
+    protected void finalize() throws Throwable
+    { 
+	this.content	= null;
+	super.finalize();
+    }
+}
+
+final class HTMLBlockEnumerator implements Enumeration
+{ 
+    private HTMLBlock block;
+    int			member;
+
+    public HTMLBlockEnumerator(HTMLBlock block)
+    { 
+	this.block	= block;
+	this.member	= 0;
+    }
+
+    public Object nextElement()
+    { 
+	Object element;
+
+	element = this.block.content.elementAt(member);
+	member ++;
+	return element;
+    }
+
+    public boolean hasMoreElements()
+    {
+	if (block == null || this.member == this.block.content.size()){
+	    return false;
+	}
+	return true;
+    }
+}
+
+final class HTMLBlockTagEnumerator implements Enumeration
+{
+    private HTMLBlock block;
+    int	member;
+    
+    public HTMLBlockTagEnumerator(HTMLBlock block)
+    { 
+	this.block = block;
+	this.member = 0;
+	while ( this.member < this.block.content.size() &&
+		this.block.content.elementAt(this.member) instanceof HTMLTag == false){
+	    this.member ++;
+	}
+    }
+
+    public Object nextElement()
+    { 
+	Object element;
+	
+	element = this.block.content.elementAt(member);
+	do {
+	    member ++;
+	} while (this.block.content.elementAt(member) instanceof HTMLTag == false);
+	return element;
+    }
+    
+    public boolean hasMoreElements()
+    { 
+	if (block == null || this.member == this.block.content.size()){
+	    return false;
+	}
+	return true;
+    }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java	(revision 12189)
@@ -0,0 +1,409 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLBlockList
+{ Vector blocks;
+	String title;
+	protected static HTMLTagVector anchorStartTags;
+	protected static HTMLTagVector anchorEndTags;
+	protected static HTMLTagVector anchorOptEndTags;
+	protected static HTMLTagVector headingStartTags;
+	protected static HTMLTagVector headingEndTags;
+
+	static
+	{	anchorStartTags 	= new HTMLTagVector("a");
+		anchorEndTags			= new HTMLTagVector("a,h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		anchorOptEndTags 	= new HTMLTagVector("div,center,form");
+		headingStartTags 	= new HTMLTagVector("h1,h2,h3,h4,h5,h6");
+		headingEndTags	 	= new HTMLTagVector("h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+	}
+
+	// opening   opentag  incontent	result
+	//    T					T					T				*       {if opening, incontent must be false}
+	//    T					T					F				T
+	//    T					F					T				*       {if opening, incontent must be false}
+	//    T					F					F				F
+	//    F					T					T				T
+	//    F					T					F				F				{if getting an opening tag
+	//		F					F					T				T
+	//		F					F					F				T
+	private boolean inTagList(HTMLTag tag, Vector tags, boolean opening)
+	{ int tagref;
+
+		for (tagref = 0; tagref < tags.size(); tagref ++)
+		{ if ((tag.xtagName().equals((String) tags.elementAt(tagref))) &&
+		      (tag.tagIsOpening() || !opening))
+			{ return true;
+			}
+		}
+		return false;
+	}
+
+	//
+	// Common initialisation; parse the document for text surrounded by
+	// pairs of <starttag>....<endtag/starttag> combos
+	//
+	protected void initialise(HTMLDoc doc, HTMLTagVector starttags, HTMLTagVector endtags, HTMLTagVector optendtags)
+	{ boolean intag;
+		boolean incontent;
+		HTMLCText		text;
+		HTMLTag			tag;
+		HTMLTag 		starttag;
+		HTMLBlock 	block;
+		HTMLBlock		docBlock;
+		Enumeration	elements;
+		Object			element;
+
+		this.blocks = new Vector(1);
+		block				= null;
+		intag				= false;
+		incontent		= false;
+		starttag		= null;
+
+		docBlock		= doc.getCodedContent();
+		elements		= docBlock.elements();
+
+		while (elements.hasMoreElements())
+		{ element	= elements.nextElement();
+
+			// if it's a tag
+			if (element instanceof HTMLTag)
+			{ tag = (HTMLTag) element;
+			
+				// end the fragment if getting a closing tag
+				if (intag)
+				{ if (endtags.matches(tag, false) ||
+							(incontent && optendtags.matches(tag, false)))
+					{ block.addTag(starttag.endTag());
+						intag = false;
+						incontent = false;
+					}
+					else
+					{ block.addTag(tag);
+					}
+				}
+
+				// !intag is checked for explicitly, as it may have been
+				// switched off by a non-partner opening tag above: eg:
+				//   <H2>A Heading<H3>A Smaller Heading</H3>
+				//                ^^^^
+				//                here
+
+				// start fragment if the tag is a starter
+				if (!intag)
+				{ if (starttags.matches(tag, true))
+					{ block = new HTMLBlock(tag.startPos(), tag.endPos());
+						this.blocks.addElement((Object) block);
+						block.addTag(tag);
+						starttag = tag;
+						intag = true;
+						incontent = false;
+					}
+				}
+			}	
+			// if the parse returned text
+			else
+			{ text	= (HTMLCText) element;
+
+				if (intag)
+				{ block.addText(text);
+					if (!text.nullString())
+					{ incontent = true;
+					}
+				}
+			}
+		}
+
+		// if a tag wasn't "finished");
+		if (intag)
+		{ block.addTag(starttag.endTag());
+		}
+
+		this.title = doc.urlString();
+	}
+
+	protected HTMLBlockList()
+	{
+	}
+	
+	//
+	// constructor for given vectors of tags: startlist elements always start a new
+	// block [ending an open one if necessary]; endlist elements always end an open 
+	// block; optendlist elements close a block iff a block has been started, and
+	// some non-blank content (ie. some "real" text, not just returns/spaces) has
+	// been read since the beginning of the block
+	//
+	public HTMLBlockList(HTMLDoc doc, Vector starttags, Vector endtags, Vector optendtags)
+	{ HTMLTagVector _starttags;
+		HTMLTagVector _endtags;
+		HTMLTagVector _optendtags;
+
+		_starttags	= new HTMLTagVector(starttags);
+		_endtags		= new HTMLTagVector(starttags);
+		_optendtags	= new HTMLTagVector(starttags);
+		this.initialise(doc, _starttags, _endtags, _optendtags);
+	}
+	
+	//
+	// constructor for explicit lists; a tagVector is created for each list;
+	// see above for list roles
+	//
+	public HTMLBlockList(HTMLDoc doc, String startlist, String endlist, String optendlist)
+	{ HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		starttags		= new HTMLTagVector(startlist);
+		endtags			= new HTMLTagVector(endlist);
+		optendtags	= new HTMLTagVector(optendlist);
+		this.initialise(doc, starttags, endtags, optendtags);
+	}
+
+	//
+	// constructor; get all HTMLBlocks for h{level} through to h1;
+	// eg hlevel==3 test h3,h2,h1
+	//
+	public HTMLBlockList(HTMLDoc doc, int hlevel)
+	{ int loop;
+		StringBuffer levelstring;
+		HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		levelstring = new StringBuffer("h1");
+		for (loop = 2; loop <= hlevel; loop ++)
+		{ levelstring.append(",h");
+		  levelstring.append(Integer.toString(loop));
+		}
+
+		starttags		= new HTMLTagVector(levelstring.toString());
+		levelstring.append(",table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		endtags			= new HTMLTagVector(levelstring.toString());
+		optendtags	= new HTMLTagVector("div,center,form");
+
+		this.initialise(doc, starttags, endtags, optendtags);
+	}
+
+	/**
+	 * Static method returning list of anchors of a document
+	 */
+	public static synchronized HTMLBlockList anchorlist(HTMLDoc doc)
+	{ HTMLBlockList anchors;
+		HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		anchors = new HTMLBlockList();
+
+		anchors.initialise(doc, anchorStartTags, anchorEndTags, anchorOptEndTags);
+
+		return anchors;
+	}
+
+	//
+	// return how many tags are in the block list
+	//
+	public int size()
+	{ if (this.blocks == null)
+		{ return 0;
+		}
+
+		return this.blocks.size();
+	}
+
+	/**
+	 * @return number of blocks in the block list
+	 * @deprecated
+	 */
+	public int tagCount()
+	{	return this.size();
+	}
+
+	//
+	// return the position of the tag in the document
+	//
+	public int tagPos(int tag)
+	{ HTMLBlock block;
+
+		if (this.blocks == null)
+		{ return 0;
+		}
+		block = (HTMLBlock)	this.blocks.elementAt(tag);
+		return block.startPos();
+	}
+
+	//
+	// return the nth block in the list
+	//
+	public HTMLBlock tagBlock(int tag)
+	{ HTMLBlock block;
+
+		block = (HTMLBlock) this.blocks.elementAt(tag);
+		return block;
+	}
+
+	//
+	// return the index of the next blocklist of the same level
+	//
+	public int tagNext(int attag)
+	{ HTMLBlock block;
+		String		headTag;
+		int				nextTag;
+
+		block = (HTMLBlock) this.blocks.elementAt(attag);
+		headTag = block.headTagName();
+		nextTag	= attag+1;
+
+		while (nextTag < this.blocks.size())
+		{	block	=	(HTMLBlock) this.blocks.elementAt(nextTag);
+			if (block.headTagName().equals(headTag))
+			{	break;
+			}
+			nextTag ++;
+		}
+		return nextTag;
+	}
+
+	//
+	// return the number of children at the next heading level
+	// 
+	public int tagChildren(int fromtag, int level)
+	{ int sibling;
+		int descendent;
+		int children;
+		int	atlevel;
+		HTMLTag tag;
+
+		// get current tag level; level 6 headings cannot have
+		// children
+		if (level == 6)
+		{ return 0;
+		}
+
+		// now, if the next tag at this level is the next tag in
+		// the document, then we can't have any children
+		if (fromtag > 0)
+		{ sibling	= this.tagNext(fromtag-1);
+		}
+		else
+		{	sibling	= this.size();
+		}
+		if (sibling == fromtag)
+		{ children = 0;
+		}
+		// if the next tag at this level is *not* next, then we have
+		// at least one child (ie the next tag in sequence), even if
+		// it is not at the next heading level down.
+		//
+		// if no further children occur at the next level down, we
+		// have only one child.  If any further children occur at the
+		// next level down, then they are counted, except where it is
+		// the next tag in sequence, which is automatically accounted
+		// for
+		else
+		{	children	= 1;
+			for (descendent = fromtag+1; descendent < sibling; descendent ++)
+			{ tag	= this.tagBlock(descendent).headTag();
+				
+				if (tag.tagLevel() ==	level + 1)
+				{	children ++;
+				}
+			}
+		}
+		return children;
+	}
+
+
+	public int tagChildren(int attag)
+	{ int sibling;
+		int descendent;
+		int children;
+		int	atlevel;
+		HTMLTag tag;
+
+		// get current tag level; level 6 headings cannot have
+		// children
+		atlevel		=	this.tagBlock(attag).headTag().tagLevel();
+		if (atlevel == 6)
+		{ return 0;
+		}
+
+		// now, if the next tag at this level is the next tag in
+		// the document, then we can't have any children
+		sibling		= this.tagNext(attag);
+		if (sibling == attag + 1)
+		{ children = 0;
+		}
+		// if the next tag at this level is *not* next, then we have
+		// at least one child (ie the next tag in sequence), even if
+		// it is not at the next heading level down.
+		//
+		// if no further children occur at the next level down, we
+		// have only one child.  If any further children occur at the
+		// next level down, then they are counted, except where it is
+		// the next tag in sequence, which is automatically accounted
+		// for
+		else
+		{	children	= 1;
+			for (descendent = attag + 2; descendent < sibling; descendent ++)
+			{ tag	= this.tagBlock(descendent).headTag();
+				
+				if (tag.tagLevel() ==	atlevel + 1)
+				{	children ++;
+				}
+			}
+		}
+		return children;
+	}
+
+	//
+	// write out a structured output onto System.out
+	//
+  public void sysoutList()
+	{ int item;
+		HTMLBlock block;
+
+		if (this.blocks == null)
+		{ return;
+		}
+
+		for (item = 0; item < this.blocks.size(); item ++)
+		{ block = (HTMLBlock) this.blocks.elementAt(item);
+			System.out.println("  @"+ block.startPos()+":"+block.headTagName() +":"+ block);
+		}
+	}
+
+	//
+	// write out HTMLBlocks as an Enumeration
+	//
+	public Enumeration elements()
+	{ return new HTMLBlockListEnumerator(this);
+	}
+}
+
+final class HTMLBlockListEnumerator implements Enumeration
+{ private HTMLBlockList blocklist;
+	private int						member;
+
+	public HTMLBlockListEnumerator(HTMLBlockList blocklist)
+	{ this.blocklist	= blocklist;
+		this.member			= 0;
+	}
+
+	public Object nextElement()
+	{ Object element;
+
+		element = this.blocklist.blocks.elementAt(member);
+		member ++;
+		return element;
+	}
+
+	public boolean hasMoreElements()
+	{ if (this.blocklist == null || this.member == this.blocklist.blocks.size())
+		{ return false;
+		}
+		return true;
+	}
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLCText.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLCText.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLCText.java	(revision 12189)
@@ -0,0 +1,105 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLCText implements HTMLObject
+{ String text;
+  int    start;
+  int    end;
+
+  public static final String HTML_TEXT_TYPE = "Text";
+
+  public HTMLCText(String text, int start, int end)
+  { this.text  = text;
+    this.start = start;
+    this.end   = end;
+  }
+
+  public String getHTMLType()
+  { return HTML_TEXT_TYPE;
+  }
+
+  public int startPos()
+  { return this.start;
+  }
+
+  public int endPos()
+  { return this.end;
+  }
+
+  public boolean isEmpty()
+  { return emptyString(this.text);
+  }
+
+  public static boolean emptyString(String s)
+  { int c;
+
+    for (c = 0; c < s.length(); c ++)
+    { if (s.charAt(c) > 32)
+      { return false;
+      }
+    }
+    return true;
+  }
+
+  public static StringBuffer cleanString(StringBuffer buffer)
+  { int c;
+    int w;
+
+    w = 0;	
+    for (c = 0; c < buffer.length(); c ++)
+    { if (buffer.charAt(c) >= 32)
+      { if (buffer.charAt(c) == 32)
+	{ if (w > 0 && buffer.charAt(w-1) > 32)
+	  { buffer.setCharAt(w, buffer.charAt(c));
+	    w ++;
+	  }
+	}
+        else
+	{ if (c != w)
+	  { buffer.setCharAt(w, buffer.charAt(c));
+	  }
+	  w ++;
+	}
+      }
+      else
+      { if (w > 0 && buffer.charAt(w-1) > 32)
+	{ buffer.setCharAt(w,' ');
+	  w ++;
+	}
+      }
+    }
+    buffer.setLength(w);
+    return buffer;
+  }
+
+  /**
+   * nullString; used by initialisation for checking for
+   * non-blank content (ie. text other than just spaces/returns)
+   * see constructor (ie HTMLBlockList) methods below for role of
+   * this in the larger plan of things
+   */
+  public boolean nullString()
+  { int ch;
+
+    if (text != null)
+    { for (ch = 0; ch < text.length(); ch ++)
+      { if (text.charAt(ch) > ' ')
+	{ return false;
+	}
+      }
+    }
+    return true;
+  }
+
+  public String toString()
+  { return this.text;
+  }
+
+  public String toString(boolean cleaned)
+  { if (cleaned == false)
+    { return toString();
+    }
+    else
+    { return HTMLEntity.decodeText(this.text);
+    }
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLContext.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLContext.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLContext.java	(revision 12189)
@@ -0,0 +1,201 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLContext
+{ boolean valid;
+	StringBuffer	text;
+	
+	public static	HTMLTagVector breakTags = new HTMLTagVector("a,h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+
+	private int offset (String string, int direction)
+	{	int offset;
+
+		if (direction > 0)
+		{	offset = string.indexOf('.');
+		}
+		else
+		{	offset = string.lastIndexOf('.');
+		}
+		return offset;
+	}
+
+	public HTMLContext(HTMLBlock parent, HTMLBlock child, int scope)
+	{ int pos;
+		int start, startoffset;
+		int	end, endoffset;
+		int	length;
+		String	string;
+		HTMLTag	tag;
+
+		this.text	= null;
+
+		// if we can't match it, give up!
+		pos = parent.find(child);
+		if (pos < 0)
+		{ this.valid	= false;
+			return;
+		}
+
+		// get start!
+		if (pos > 0)
+		{ start = pos - 1;
+		}
+		else
+		{	start	= 0;
+		}	
+		startoffset	= -1;
+		length	= 0;
+		while (	start > 0)
+		{	
+		  /*
+		  if (parent.elementAt(start) instanceof String)
+			{	string = (String) parent.elementAt(start);
+
+				startoffset	= offset(string, -1);
+				if (startoffset > 0)
+				{	break;
+				}
+
+				length += string.length();
+				
+				if (length >= scope)
+				{	break;
+				}
+				}
+				else 
+		  */
+			if (parent.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.elementAt(start)).toString();
+
+				startoffset	= offset(string, -1);
+				if (startoffset > 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.elementAt(start) instanceof HTMLTag)
+			{	// do some checks
+				tag = (HTMLTag) parent.elementAt(start);
+
+				if (breakTags.matches(tag, false))
+				{	startoffset	= -1;
+					break;
+				}
+			}
+			start --;
+		}
+
+		// get end!
+		pos = pos + child.size();
+		if (pos < parent.size())
+		{ end	= pos + 1;
+		}
+		else
+		{	end	= parent.size();
+		}	
+		endoffset	= -1;
+		length	= 0;
+		string	= null;
+		while (end < parent.size())
+		{	if (parent.content.elementAt(start) instanceof String)
+			{	string = (String) parent.content.elementAt(start);
+
+				endoffset	= offset(string, +1);
+				if (endoffset >= 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(start)).toString();
+
+				endoffset	= offset(string, +1);
+				if (endoffset >= 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLTag)
+			{	tag = (HTMLTag) parent.elementAt(start);
+
+				if (breakTags.matches(tag, false))
+				{	endoffset	= -1;
+					end	= end - 1;
+					break;
+				}
+			}
+			System.out.println("String:"+string);
+			end ++;
+		}
+
+		this.text	= new StringBuffer();
+		if (startoffset >= 0)
+		{	string	= null;
+
+			if (parent.content.elementAt(start) instanceof String)
+			{	string = (String) parent.content.elementAt(start);
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(start)).toString();
+			}
+			
+			if (string != null)
+			{ this.text.append(string.substring(startoffset+1));
+			}
+			start ++;
+		}
+
+		for (pos = start; pos <= end; pos ++)
+		{ if (pos == end && endoffset >= 0)
+			{	break;
+			}
+
+			if (parent.content.elementAt(pos) instanceof String)
+			{	this.text.append((String) parent.content.elementAt(pos));
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLCText)
+			{	this.text.append(((HTMLCText) parent.content.elementAt(pos)).toString());
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLTag)
+			{ // don't add it!
+			}
+		}
+
+		if (pos == end && endoffset > 0)	// this deliberately ignore endoffset == 0
+																		 	// 'cos nothing should happen anyhow!
+		{	string	= null;
+
+			if (parent.content.elementAt(pos) instanceof String)
+			{	string = (String) parent.content.elementAt(pos);
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(pos)).toString();
+			}
+		
+			if (string != null)
+			{	this.text.append(string.substring(0, endoffset));
+			}
+		}
+	}
+
+	public String toString()
+	{	if (this.text == null)
+		{ return "";
+		}
+		return this.text.toString();
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java	(revision 12189)
@@ -0,0 +1,658 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.*;
+import java.io.*;
+import java.applet.*;
+import java.util.*;
+
+/** 
+ *  HTMLDOC: handles an individual HTML document
+ */
+
+public class HTMLDoc
+{ 
+    String        content;
+    String        title;
+    URL           url;
+    int           pos;
+    int           status;
+    HTMLBlockList blocklist;
+    List          urls_out;
+    List          urls_in;
+    Object        note;
+    HTMLLoader    loader;
+    HTMLBlock     coded;
+
+    /**
+     * Constructor which loads a given document; the ref indicates from where!
+     */
+    public HTMLDoc(URL url, int ref)
+    { 
+	URLConnection connect;
+	int length, i, c;
+	byte inbuffer[];
+	InputStream in;
+
+	this.url = url;
+	this.pos = 0;
+	this.blocklist = null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+	this.note = null;
+	this.status = 0;
+
+	if (ref >= 0){
+	    int attempts;
+	    
+	    attempts = 0;
+	    loader = new HTMLLoader(url);
+	    loader.load();
+	    if (ref == 0){
+		while(this.ready() == false){
+		    try {
+			Thread.sleep(100);
+		    }
+		    catch (InterruptedException ex){
+			
+		    }
+		}
+	    }
+	}
+	else {
+	    this.content = "";
+	}
+    }
+    
+    public HTMLDoc(URL url)
+    {	
+	this(url, 0);
+    }
+    
+    /**
+     * @return <code>true</code> when a document has finished loading.
+     */
+    public synchronized boolean ready()
+    {
+	if (this.loader == null){
+	    return true;
+	}
+	
+	boolean loaded 	= loader.loaded();
+	
+	if (loaded == true){
+	    // reload if necessary
+	    if (this.loader.error == true){
+		if (this.status < 3 && this.loader.notFound == false){
+		    this.status ++;
+		    this.loader = new HTMLLoader(this.url);
+		    loader.load();
+		    return false;
+		}
+		else {
+		    this.status 	= 4;
+		    this.content	= null;
+		    this.loader		= null;
+		    return true;
+		}
+	    }
+	    this.content 	= loader.HTML();
+	    this.loader		= null;
+	}
+	return loaded;
+    }
+    
+    /**
+     * Create a document from a file
+     */
+    public HTMLDoc(URL url, String file)
+    { 
+	FileInputStream _in;
+	byte[] data = new byte[128];
+	int databytes;
+	StringBuffer content;
+	
+	try {
+	    _in = new FileInputStream(file);
+	    if (_in != null){
+		content = new StringBuffer();
+		do{  
+		    databytes = _in.read(data);
+		    if (databytes > 0){
+			content.append(new String(data, 0, databytes));
+		    }		
+		} while (databytes >= 0);
+		this.content		= content.toString();
+	    }
+	}
+	catch (IOException ex){
+	    this.content = null;
+	    System.out.println(ex);
+	}
+	this.url = url;
+	this.pos = 0;
+	this.status = 0;
+	this.note = null;
+	this.blocklist	= null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+    }
+
+    /**
+     * Create a url-only document: the document body is not stored
+     */
+    public HTMLDoc(String url)
+    {
+	if (url != null){
+	    try{
+		this.url = new URL(url);
+	    }
+	    catch (MalformedURLException ex){
+		this.url = null;
+	    }
+	}
+	else {
+	    this.url = null;
+	}
+	this.pos = 0;
+	this.status = 0;
+	this.note = null;
+	this.content = null;
+	this.blocklist = null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+    }
+    
+    static public HTMLDoc fromString(String url, String string)
+    {	
+	HTMLDoc reply;
+	
+	reply = new HTMLDoc(url);
+	reply.content = string;
+	return reply;
+    }
+    
+    static public boolean validExtension(String url)
+    {	
+	String extension;
+	
+	// get file extension
+	extension = URLString.extension(url);
+	if (extension != null){
+	    extension = extension.toLowerCase();
+	}
+	
+	if (extension == null ||
+	    extension.indexOf("htm") >= 0 || 
+	    extension.indexOf("asp") >= 0){
+	    return true;
+	}
+	return false;
+    }
+    
+    static boolean validExtension(URL url)
+    {	
+	return validExtension(url.toString());
+    }
+
+    /**
+     * Uncache a single document
+     */
+    public static HTMLDoc uncacheDoc(String filebase, String url)
+    {	
+	String 	pathname;
+	String	filename;
+	HTMLDoc	doc;
+	
+	pathname = URLString.pathName(url);
+	if (pathname == null){
+	    return null;
+	}
+	
+	filename = filebase + pathname.replace('/', File.pathSeparatorChar);
+	doc = new HTMLDoc(URLString.toURL(url), filename);
+	
+	return doc;
+    }
+
+    /**
+     * @return URL (ie. Java URL object) of the document
+     */
+    public URL getUrl()
+    { 
+	return this.url;
+    }
+
+    // Report url to AppletContext
+    public void show(AppletContext ac)
+    { 
+	URL url;
+
+	url = this.getUrl();
+	if (url != null){
+	    ac.showDocument(url);
+	}
+    }
+    
+    // Get a HTMLBlock list of the wanted text in an HTMLdoc
+    public void getSelectContent(int level)
+    { 
+	HTMLBlockList list;
+
+	list = new HTMLBlockList(this, level);
+	this.blocklist = list;
+    }
+    
+    /**
+     * return the selected content
+     */
+    public HTMLBlockList selectContent()
+    { 
+	return this.blocklist;
+    }
+
+    /**
+     * get header blocks as selected content
+     */
+    public HTMLBlockList headerBlocks()
+    {	
+	if (this.blocklist == null){
+	    this.getSelectContent(6);
+	}
+	return this.blocklist;
+    }
+    
+    /**
+     * return a tagged-up select with an <A NAME=> at the beginning of each Hx
+     */
+    public String taggedSelectContent()
+    {
+	int heading, offset, pos;
+	String reply;
+	
+	if (this.blocklist == null || 
+	    this.blocklist.size() == 0){
+	    return this.content;
+	}
+	
+	reply = "";
+	offset	= 0;
+	for (heading = 0; heading < this.blocklist.size(); heading ++){
+	    reply = reply + 
+		this.content.substring(offset, this.blocklist.tagPos(heading)) +
+		"<A NAME=\"_L"+heading+"\"></A>";
+	    offset = this.blocklist.tagPos(heading);
+	}
+	reply = reply + this.content.substring(offset, this.content.length());
+	return reply;
+    }
+    
+    /**
+     * Return enumeration of headings in document
+     */
+    public Enumeration headings()
+    {
+	if (this.blocklist == null){
+	    this.getSelectContent(6);
+	}
+	return this.blocklist.elements();
+    }
+    
+    /**
+     *	@return HTMLBlock of the child text underneath a heading
+     */
+    public HTMLBlock headingChildText(int headref)
+    { 
+	HTMLBlock pageblock;
+	HTMLBlock headblock;
+	HTMLBlock childblock;
+	int blockstart, blockend;
+	
+	pageblock = this.getCodedContent(); // get all the coded bits of the page
+	headblock = this.headerBlocks().tagBlock(headref);
+	
+	blockstart = pageblock.find(headblock);
+	blockstart += headblock.size();
+	if (headref == this.headerBlocks().size() - 1) {
+	    blockend = pageblock.size();
+	}
+	else {
+	    blockend = pageblock.find(this.headerBlocks().tagBlock(headref+1));
+	}
+	childblock = pageblock.subBlock(blockstart, blockend);
+	
+	return childblock;
+    }
+    
+    /**
+     * Add incoming url to list of incoming references
+     */
+    public void addUrlIn(String url)
+    {
+	if (this.urls_in == null){
+	    this.urls_in = new ArrayList();
+	}
+	this.urls_in.add(url);
+    }
+    
+    /**
+     * @return a String of all incoming urls
+     */
+    public String urlsInString()
+    { 
+	if (this.urls_in != null){
+	    return this.urls_in.toString();
+	}
+	return "";
+    }
+    
+    /**
+     * @return an enumeration of all incoming urls as Strings
+     */
+    public Iterator urlsIn()
+    {
+	if (this.urls_in == null){
+	    return null;
+	}
+	return this.urls_in.iterator();
+    }
+    
+    /**
+     * @return the <code>StringVector</code> of all incoming urls
+     */
+    public List urlsInList()
+    {	
+	return this.urls_in;
+    }
+
+    /**
+     * @return <code>true</code> if the given url is linked to by this
+     *  document
+     */
+    public boolean linkedTo(URL url)
+    {	
+	if (this.urls_out != null &&
+	    this.urls_out.indexOf(url.toString()) >= 0){
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     * See above: returns <code>true</code> if the document is linked to
+     * by this document.
+     */
+    public boolean linkedTo(HTMLDoc doc)
+    {
+	return linkedTo(doc.getUrl());
+    }
+
+    /**
+     * Add outgoing url to list of outgoing references
+     */
+    public void addUrlOut(String url)
+    { 
+	if (this.urls_out == null){
+	    this.urls_out = new ArrayList();
+	}
+	this.urls_out.add(url);
+    }
+    
+    /**
+     * @return a String of all outgoing urls
+     */
+    public String urlsOutString()
+    { 
+	if (this.urls_out != null){
+	    return this.urls_out.toString();
+	}
+	return "";
+    }
+    
+    /**
+     * @return the <code>StringVector</code> of all outgoing urls
+     */
+    public List urlsOutList()
+    {	
+	return this.urls_out;
+    }
+
+    /**
+     * Return an enumeration of outbound urls
+     */
+    public Iterator urlsOut()
+    { 
+	if (this.urls_out == null){
+	    return null;
+	}
+	return this.urls_out.iterator();
+    }
+
+    private static int urlListPos(List urlList, String url)
+    {	
+	for (int i = 0; i < urlList.size(); i ++){
+	    if (((String) urlList.get(i)).equals(url)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    // Rename internally the content of a list
+    private void __renameUrl(List urlList, String oldUrl, String newUrl)
+    {	
+	if (urlList != null){
+	    int oldPos = urlListPos(urlList, oldUrl);
+	    
+	    if (oldPos >= 0){
+		if (urlListPos(urlList, newUrl) >= 0){
+		    // delete oldUrl: now duplicate
+		    urlList.remove(oldPos);
+		}
+		else {
+		    // rename oldUrl
+		    urlList.set(oldPos, newUrl);
+		}
+	    }
+	}
+    }
+    
+    /**
+     * Rename inbound/outbound url references
+     */
+    public void renameUrl(String oldUrl, String newUrl)
+    {	
+	this.__renameUrl(this.urls_in, oldUrl, newUrl);
+	this.__renameUrl(this.urls_out, oldUrl, newUrl);
+    }
+    
+    /**
+     * Save document
+     */
+    public boolean save(String filename, boolean select)
+    {	
+	FileOutputStream fout;
+	PrintWriter pout;
+	
+	String flattened = URLString.flattenedUrl(filename);
+	
+	try{	
+	    HTMLDocAnchorList anchorList = new HTMLDocAnchorList(this);
+
+	    anchorList.flattenLinks();
+			
+	    fout = new FileOutputStream(flattened);
+	    pout = new PrintWriter(fout);
+	    if (select == true) {
+		pout.print(this.taggedSelectContent());
+	    }
+	    else {
+		pout.print(this.content);
+	    }
+	    pout.close();
+	    fout.close();
+	}
+	catch (IOException ex) {
+	    System.out.println(ex);
+	    return false;
+	}
+	return true;
+    }
+    
+    /**
+     * Save document content
+     */
+    public boolean save(String base)
+    {
+	return this.save(base, false);
+    }
+    
+    /**
+     * Save selected content
+     */
+    public boolean saveSelect(String base)
+    {	
+	return this.save(base, true);
+    }
+
+    /**
+     * Get title of page
+     */
+    public void getTitle()
+    { 
+	HTMLDocTitle doctitle;
+
+	if (this.content != null){
+	    doctitle = new HTMLDocTitle(this);
+	    this.title	= doctitle.Title();
+	}
+    }
+    
+    /**
+     * @return: title of page
+     */
+    public String titleString()
+    {
+	if (this.title == null){
+	    this.getTitle();
+	}
+	return this.title;
+    }
+    
+    /**
+     * @return a <code>String</code> of the url of the page
+     */
+    public String urlString()
+    {
+	return this.url.toString();
+    }
+
+    /**
+     * 	Replace the HTML text of the page with the provided HTML string
+     *	content.  This is an extremely dangerous call to use, as dependent
+     *	objects such as tag objects will be immediately rendered inaccurate
+     *	in terms of pointers held to the original text.
+     */
+    protected void setContent(String newContent)
+    { 
+	this.content = newContent;
+    }
+
+    /**
+     * @return a <code>String</code> of the HTML text of the page
+     */
+    public String getContent()
+    { 
+	return this.content;
+    }
+
+    /**
+     * @return a <code>HTMLBlock</code> of the page content
+     */
+    public HTMLBlock getCodedContent()
+    {	
+	// if we haven't already got it, code the bugger up right now!
+	if (this.coded == null){
+	    HTMLParser parser;
+	    String next;
+	    HTMLBlock coded;
+	    HTMLTag tag;
+	    HTMLCText text;
+
+	    if (this.content == null){
+		return null;
+	    }
+	    
+	    parser = new HTMLParser(this);
+	    coded = new HTMLBlock(0, this.content.length());
+
+	    parser.startParse();
+	    next = parser.fullParse();
+	    while (next != null){
+		if (HTMLTag.isTag(next)){
+		    tag = new HTMLTag(next, parser.lastParse(), parser.atParse());
+		    coded.addTag(tag);
+		}
+		else {
+		     text = new HTMLCText(next, parser.lastParse(), parser.atParse());
+		     coded.addText(text);
+		}
+		next = parser.fullParse();
+	    }
+	    this.coded	= coded;
+	}
+	return this.coded;
+    }
+    
+    /**
+     * @return <code>true</code> if this document is not errored from loading
+     * etc.
+     */
+    public boolean valid()
+    {	
+	return (this.status < 4 && this.content != null && this.content.equals("null") == false);
+    }
+    
+    /**
+     * HTMLString: returns content
+     */
+    public String HTMLString()
+    { 
+	return this.content;
+    }
+    
+    /**
+     * Return text of document as ascii text;
+     */
+    /*
+      public String toString()
+      {	StringBuffer	reply = new StringBuffer();
+      int						parastart = 0;
+
+      return reply.toString();
+      }
+    */
+
+    /**
+     * Set current note on this page
+     * @return old note object
+     */
+    public Object setNote(Object note)
+    {
+	Object oldnote;
+	
+	oldnote	= this.note;
+	this.note = note;
+	return oldnote;
+    }
+    
+    /**
+     * @return current note <code>Object</code> on this page
+     */
+    public Object getNote()
+    {
+	return this.note;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java	(revision 12189)
@@ -0,0 +1,160 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+public class HTMLDocAnchorList extends HTMLBlockList
+
+{ HTMLDoc doc;
+
+	public HTMLDocAnchorList(HTMLDoc doc)
+
+	{ super();
+
+		this.doc = doc;	// remember the document reference
+
+
+
+		super.initialise(doc, anchorStartTags, anchorEndTags, anchorOptEndTags);
+
+	}
+
+
+
+	/**
+	 *	Return the HTMLBlock containing the named local anchor and it's
+	 *	child (contained) tags.
+	 *
+	 *	@param <code>String</code> name of the local anchor to find.
+	 *	@return <code>HTMLBlock</code> the corresponding block of HTML
+	 *					contained by the anchor.  If the local anchor does not
+	 *					exist, this value will be <code>null</code>.
+	 */
+	public HTMLBlock localAnchor(String name)
+
+	{	int block;
+
+		String	localname;
+
+
+
+		for (block = 0; block < this.size(); block ++)
+
+		{ localname = this.tagBlock(block).headTag().idValue("name");
+
+			if (localname == null)
+
+			{ continue;
+
+			}
+
+			if (name.equalsIgnoreCase(localname))
+
+			{	return this.tagBlock(block);
+
+			}
+
+		}
+
+		return null;
+
+	}
+
+	private int indexOf(String base, String subString, int from)
+	{	int i, j;
+
+		for (i = from; i < base.length() - subString.length(); i ++)
+		{	if (Character.toLowerCase(base.charAt(i)) == subString.charAt(0))
+			{	j = 0;
+				for (j = 1; j < subString.length(); j ++)
+				{	if (Character.toLowerCase(base.charAt(i+j)) != subString.charAt(j))
+					{	break;
+					}
+				}
+
+				if (j == subString.length())
+				{	return i;
+				}
+			}
+		}
+		return -1;
+	}
+
+	public void flattenLinks()
+	{	int 					block;
+		String				url;
+		char					endsWith;
+		StringBuffer	newContent = new StringBuffer(doc.getContent());
+		boolean				changed = false;
+
+//		System.out.println("Flattening");
+
+		for (block = 0; block < this.size(); block ++)
+		{ HTMLTag tag;
+			int			tagPos;
+			int			valueStart, valueEnd;
+
+			tag = this.tagBlock(block).headTag();	// get the <A> tag itself
+
+			url = tag.idValue("href"); 						// if there is no HREF member, abandon this one
+			if (url == null)
+			{ System.out.println("No href");
+				continue;
+			}
+	
+			tagPos	= tag.startPos();
+			tagPos	=	indexOf(doc.getContent(), "href", tagPos);	// get the HREF itself
+			if (tagPos > tag.endPos() || tagPos < 0)					// if none in the tag, skip it
+			{	System.out.println("Didn't get href");
+				continue;
+			}
+			tagPos += "href".length();												// skip past it
+
+			while (	doc.getContent().charAt(tagPos) <= ' ' ||		// skip pass equals, etc.
+							doc.getContent().charAt(tagPos) == '=')
+			{	tagPos ++;
+			}
+
+			if (doc.getContent().charAt(tagPos) == '"' ||				// get the demarcating whitespace/quotes
+					doc.getContent().charAt(tagPos) == '\'')
+			{	endsWith	= doc.getContent().charAt(tagPos);
+				tagPos ++;
+			}
+			else
+			{	endsWith  = ' ';
+			}
+			valueStart = tagPos;
+
+			//			System.out.println("Flattening " + doc.getContent().length() + " " + tagPos);
+
+			while (tagPos < doc.getContent().length() &&
+			       doc.getContent().charAt(tagPos) != endsWith)	// read it in
+			{	if (doc.content.charAt(tagPos) == '?' ||				// replace special characters			
+						doc.content.charAt(tagPos) == '&')
+				{	newContent.setCharAt(tagPos, '_');						//   and note that we've done so
+					changed	= true;
+				}
+				tagPos ++;
+			}
+			valueEnd = tagPos;
+
+			tagPos = indexOf(doc.getContent(), ".html", valueStart);
+			if (tagPos >= 0 && tagPos < valueEnd)
+			{	// shuffle around the html bit
+				while (tagPos < valueEnd - 5)
+				{	newContent.setCharAt(tagPos, newContent.charAt(tagPos + 5));
+					tagPos ++;
+				}
+				newContent.setCharAt(tagPos ++, '.');
+				newContent.setCharAt(tagPos ++, 'h');
+				newContent.setCharAt(tagPos ++, 't');
+				newContent.setCharAt(tagPos ++, 'm');
+				newContent.setCharAt(tagPos ++, 'l');
+			}
+		}
+
+		if (changed == true)																// replace the document as needsbe
+		{	doc.setContent(newContent.toString());
+		}
+	}
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java	(revision 12189)
@@ -0,0 +1,25 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+public class HTMLDocHeadingList extends HTMLBlockList
+
+{ public HTMLDocHeadingList(HTMLDoc doc)
+
+	{ super();
+
+
+
+		super.initialise(doc, headingStartTags, headingEndTags, anchorOptEndTags);
+
+	}
+
+
+
+	public int headingLevel(int element)
+
+	{	return this.tagBlock(element).headTag().tagLevel();
+
+	}
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java	(revision 12189)
@@ -0,0 +1,187 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+import java.util.*;
+
+
+
+public class HTMLDocLocalHeading
+
+{	String		heading;
+
+	HTMLBlock	block;
+
+
+
+	public HTMLDocLocalHeading(HTMLDoc doc, String local)
+
+	{	HTMLBlock 				nextheading, lastheading, anchor;
+
+		Enumeration				headings;
+
+		HTMLDocAnchorList	anchors;
+
+		HTMLBlock					docBlock;
+
+		int								_start, _end, _pos;
+
+
+
+		// initialise (to undefined)
+
+		this.heading	= null;
+
+		this.block		= null;
+
+
+
+		// get anchors
+
+		anchors	= new HTMLDocAnchorList(doc);
+
+		anchor	= anchors.localAnchor(local);
+
+		if (anchor == null)
+
+		{	return;
+
+		}
+
+
+
+		// get headings position; lastheading will be previous, nextheading will be next
+
+		headings		= doc.headings();
+
+		lastheading	= null;
+
+		nextheading	= null;
+
+
+
+		while (headings.hasMoreElements())
+
+		{	nextheading	= (HTMLBlock) headings.nextElement();
+
+			if (nextheading.startPos() > anchor.startPos())
+
+			{	break;
+
+			}
+
+			
+
+
+
+			lastheading	= nextheading;
+
+		}
+
+
+
+		// didn't get any actual headings; abort.
+
+		if (nextheading == null)
+
+		{	return;
+
+		}
+
+
+
+		// got to the end
+
+		if (nextheading == lastheading)
+
+		{	this.block = nextheading;
+
+		}
+
+		else
+
+		{	// now see where anchor is; try to find null textual content between anchor
+
+			// and heading
+
+			docBlock	= doc.getCodedContent();
+
+			_start		= docBlock.find(anchor);
+
+			_end			= docBlock.find(nextheading);
+
+			for (_pos = _start; _pos < _end; _pos ++)
+
+			{	// HTML tag
+
+				if (docBlock.elementAt(_pos) instanceof HTMLTag)
+
+				{	
+
+				}
+
+				// HTML text - if non-blank, abort
+
+				else
+
+				{	if (((HTMLCText) docBlock.elementAt(_pos)).nullString() == false)
+
+					{	break;
+
+					}
+
+				}
+
+			}
+
+
+
+			// assign
+
+			if (_pos != _end)
+
+			{	this.block	=	lastheading;
+
+			}
+
+			else
+
+			{	this.block	= nextheading;
+
+			}
+
+		}
+
+		this.heading	= this.block.toString();
+
+	}
+
+
+
+	public String toString()
+
+	{	return this.heading;
+
+	}
+
+
+
+	public HTMLBlock block()
+
+	{	return this.block;
+
+	}
+
+
+
+	protected void finalize() throws Throwable
+
+	{	this.heading	= null;
+
+		this.block		= null;
+
+		super.finalize();
+
+	}
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java	(revision 12189)
@@ -0,0 +1,89 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLDocTitle
+{ StringBuffer title;
+	private static HTMLTagVector titleStartTags;
+	private static HTMLTagVector titleEndTags;
+	private static HTMLTagVector titleOptEndTags;
+
+	static
+	{	titleStartTags 	= new HTMLTagVector("title,h1,h2,h3,h4,h5,h6");
+		titleEndTags 		= new HTMLTagVector("title,h1,h2,h3,h4,h5,h6,body,head,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		titleOptEndTags	= new HTMLTagVector("div,center,form");
+	}
+
+	/**
+	 * Constructor: generates a title for the supplied <code>HTMLDoc</code>.
+	 */
+	public HTMLDocTitle(HTMLDoc doc)
+	{ boolean				intitle;
+		boolean				incontent;
+		HTMLCText			text;
+		HTMLTag				tag;
+		HTMLBlock			parser;
+		Enumeration		elements;
+		Object				element;
+		
+		this.title 	= null;
+		intitle			= false;
+		incontent		= false;
+		
+		parser		=	doc.getCodedContent();
+		if (parser == null)
+		{	return;
+		}
+		elements	= parser.elements();
+
+		while (elements.hasMoreElements())
+		{ //System.out.println(read + ":" +  read.charAt(0));
+			element	= elements.nextElement();
+			
+			if (element instanceof HTMLTag)
+			{ tag = (HTMLTag) element;
+
+				if (intitle)
+				{	if (titleEndTags.matches(tag, false) ||
+							(incontent && titleOptEndTags.matches(tag, false))) // if closing
+					{ intitle = false;
+						break;
+					}
+					else // else ignore it
+					{
+					}
+				}
+				else
+				{	if (titleStartTags.matches(tag, true) == true)
+					{ intitle			= true;
+						incontent		= false;
+						this.title	= new StringBuffer();
+					}
+				}
+			}
+			else
+			{ text	= (HTMLCText) element;
+
+				if (intitle == true)
+				{	this.title.append(text.toString());
+					if (!text.nullString())
+					{ incontent	= true;
+					}
+				}
+			}
+		}
+		if (this.title != null)
+		{ this.title = HTMLCText.cleanString(this.title);
+		}
+	}
+
+	/**
+	 * @return the Title as a <code>String</code>.
+	 */
+	public String Title()
+	{	if (this.title == null)
+		{ return null;
+		}
+		return this.title.toString();
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java	(revision 12189)
@@ -0,0 +1,346 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.*;
+import java.util.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+
+public class HTMLEntity
+{ String  name;
+  int     code;
+  static Vector entities = null;
+  static Vector	nmapping = null;
+
+  public HTMLEntity(String name, int code)
+  { int start, end, at;
+    HTMLEntity compareEntity;
+
+    if (HTMLEntity.entities == null)
+    { HTMLEntity.entities = new Vector();
+    }
+    if (HTMLEntity.nmapping == null)
+    { HTMLEntity.nmapping = new Vector();
+    }
+
+    this.name	= name;
+    this.code	= code;
+
+    entities.addElement(this);
+    start = 0;
+    end   = nmapping.size();
+    while (start != end)
+    { at = (start + end) / 2;
+	
+      compareEntity = 
+	(HTMLEntity) entities.elementAt(((Integer) nmapping.elementAt(at)).intValue());
+      if (compareEntity.code > code)
+      { end = at;
+      }
+      else
+      { start = at + 1;
+      }
+    }
+    if (end == nmapping.size())
+    { nmapping.addElement(new Integer(nmapping.size()));
+    }
+    else
+    { nmapping.insertElementAt(new Integer(nmapping.size()), start);
+    }
+  }
+
+  public String toString()
+  { Character c;
+
+    c = new Character((char) this.code);
+    return c.toString();
+  }
+
+  public static String codeString(char c)
+  { StringBuffer 	reply;
+    String        codestring;
+
+    if (c < 128 || c == '&' || c == '<' || c == '>')
+    { return null;
+    }
+    codestring = Integer.toString((int) c);
+    reply      = new StringBuffer("&#");
+    reply.append(codestring);
+    reply.append(";");
+
+    return reply.toString();
+  }
+
+  public static String nameCodeString(char c)
+  { int start, end, at, compare;
+    HTMLEntity compareEntity;
+
+    start = 0;
+    end		= nmapping.size();
+    while (start != end)
+    { at	= (start + end) / 2;
+      
+      compareEntity = 
+	(HTMLEntity) entities.elementAt(((Integer) nmapping.elementAt(at)).intValue());
+      if (compareEntity.code == (int) c)
+      { StringBuffer reply;
+
+        reply = new StringBuffer("&");
+	reply.append(compareEntity.name);
+	reply.append(";");
+	return reply.toString();
+      }
+      else if (compareEntity.code < c)
+      {	end	= at;
+      }
+      else
+      {	start	= at + 1;
+      }
+    }
+    return codeString(c);
+  }
+
+  public static void getEntities(String source)
+  { HTMLEntity 	newentity;
+    int         pos;
+    int					start;
+    String			name;
+    int					code;
+    
+    if (source == null)
+    { return;
+    }
+
+    start = 0;
+    while (start < source.length())
+    { pos = start;
+      while (source.charAt(pos) != ':')
+      { pos ++;
+      }
+      name = source.substring(start, pos);
+
+      // skip colon and reset
+      pos ++;
+      start = pos;
+
+      while(pos < source.length() && source.charAt(pos) > ' ')
+      { pos ++;
+      }
+      code = Integer.parseInt(source.substring(start, pos));
+
+      newentity = new HTMLEntity(name, code);
+      
+      while ( pos < source.length() && source.charAt(pos) < ' ')
+      { pos ++;
+      }
+      start = pos;
+    }
+  }
+
+  public static void getEntities(InputStream in)
+  { getEntities(DocumentLoader.getAsString(in));
+  }
+
+  public static void getEntities(File file)
+  { FileInputStream in;
+
+    try
+    { in = new FileInputStream(file);
+      getEntities(in);
+      in.close();
+    }
+    catch (IOException io)
+    { System.out.println("No entities");
+    }
+  }
+
+  public static void getEntities()
+  { ClassLoader loader;
+    InputStream	in;
+
+    in = ClassLoader.getSystemResourceAsStream("org\\greenstone\\gsdl3\\gs3build\\util\\Entities.map");
+    if (in == null)
+    { System.out.println("Unable to load Entities.map from org.greenstone.gsdl3.gs3build.util");
+    }
+    getEntities(in);
+  }
+
+  public static String encodeText(String source, boolean named)
+  { StringBuffer reply;
+    int          start,at;
+
+    if (source == null)
+    { return null;
+    }
+
+    if (HTMLEntity.entities == null)
+    {	getEntities();
+    }
+
+    reply	= new StringBuffer();
+    at 		= 0;
+    start	= 0;
+    while (at < source.length())
+    { if (source.charAt(at) > 128)
+      {	if (at != start)
+	{ reply.append(source.substring(start, at));
+	}
+
+        if (named)
+	{ reply.append(nameCodeString(source.charAt(at)));
+	}
+	else
+	{ reply.append(codeString(source.charAt(at)));
+	}
+	start	= at + 1;
+      }
+      at ++;
+    }
+		
+    return reply.toString();
+  }
+
+  public static String decodeText(String source)
+  { int start, end, loff, roff;
+    int sstart, send, test;
+    int startoff, endoff, baseoff;
+    boolean matched;
+    String  tail;
+    StringBuffer reply;
+
+    if (source == null)
+    { return null;
+    }
+
+    if (HTMLEntity.entities == null)
+    {	getEntities();
+    }
+
+    reply   = new StringBuffer();
+    baseoff = 0;
+
+    tail = source;
+    do
+    { // get next ampersand
+	    startoff = tail.indexOf('&');
+      if (startoff < 0 || startoff == tail.length() - 1)
+      { break;
+      }
+
+      // if it's followed by a hash, evaluate as a number
+      if (tail.charAt(startoff + 1) == '#')
+      { if (startoff < tail.length() - 2 &&
+      			tail.charAt(startoff+2) >= '0' &&
+            tail.charAt(startoff+2) <= '9')
+      	{ int code;
+
+        	loff = startoff + 2;
+        	while (tail.charAt(loff) >= '0' &&
+          				tail.charAt(loff) <= '9')
+        	{ loff ++;
+          }
+
+          code = Integer.parseInt(tail.substring(startoff+2, loff));
+        	reply.append(tail.substring(0, startoff));
+					reply.append(new Character((char) code));
+
+          if (tail.charAt(loff) == ';')
+          { loff ++;
+          }
+          loff = loff - startoff - 1;
+        }
+        else
+        { loff = 0;
+        	reply.append(tail.substring(0, startoff+1));
+        }
+      }
+      else
+      {	start = 0;
+        end   = HTMLEntity.entities.size();
+        loff  = 0;
+
+	while (startoff+1+loff < tail.length() && start != end)
+	{ sstart = start;
+	  send   = end;
+          test	 = (sstart + send) >> 1;
+
+	  matched	= false;
+		      
+	  roff	= startoff + 1 + loff;
+
+	  while (sstart != send)
+	  { test = (sstart + send) >> 1;
+
+	    if (tail.charAt(roff) < ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.charAt(loff))
+	    { send = test;
+	    }
+	    else if (tail.charAt(roff) > ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.charAt(loff))
+	    { sstart	= test + 1;
+	    }
+	    else
+	    { break;
+	    }
+	  }
+
+	  //					System.out.println(sstart+">"+send+":"+loff);
+
+	  if (sstart != send) // found a match
+	  { sstart = test;
+	    while (sstart >= start &&
+		   tail.charAt(roff) ==
+		   ((HTMLEntity) HTMLEntity.entities.elementAt(sstart)).name.charAt(loff))
+	    { sstart --;
+	    }
+	    sstart ++;
+
+	    send = test + 1;
+	    while (send < end &&
+		   tail.charAt(roff) ==
+		   ((HTMLEntity) HTMLEntity.entities.elementAt(send)).name.charAt(loff))
+	    { send ++;
+	    }
+
+//						System.out.println(sstart+"!"+send+":"+loff);
+
+	    start = sstart;
+	    end	= send;
+	  }
+	  else
+	  { loff	= 0;
+	    reply.append(tail.substring(0, startoff+1));
+	    break;
+	  }
+
+//					System.out.println(start+">"+end+":"+loff);
+	  
+	  loff ++;
+	  if (end == start + 1 &&
+	      ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.length() == loff)
+	  { reply.append(tail.substring(0, startoff));
+	    reply.append(((HTMLEntity) HTMLEntity.entities.elementAt(test)).toString());
+	    if (roff < tail.length() - 1 && tail.charAt(roff+1) == ';')
+	    { loff ++;
+	    }
+	    break;
+	  }
+	}
+      }
+
+      tail = tail.substring(startoff + 1 + loff);
+    } while (tail != null && tail.length() > 0);
+    
+    if (tail != null)
+    { reply.append(tail);
+    }
+    return reply.toString();
+  }
+
+  public static void main(String args[])
+  { String reply;
+
+    getEntities();
+    
+    reply = decodeText(args[0]);
+    System.out.println(reply);
+  }
+}
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java	(revision 12189)
@@ -0,0 +1,208 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.*;
+import java.util.*;
+
+class HTMLLoadManager implements Runnable
+{	Vector		 	toLoad;
+	Vector			loading;
+	Vector			loaded;
+	boolean			stopFlag;
+	boolean			started;
+	Thread			thread;
+	static int	running;
+
+	static
+	{	running	= 0;
+	}
+
+	/**
+	 *	Vanilla constructor of an object to load HTML documents in the background
+	 */
+	public HTMLLoadManager()
+	{	this.toLoad		= new Vector();
+		this.loading 	= new Vector();
+		this.loaded		= new Vector();
+		this.thread		= null;
+		this.stopFlag	= false;
+		this.started	= false;
+	}
+
+	/**
+	 *	Check if a given document is loading
+	 *
+	 *	@return <code>boolean</code> whether the document is loading or not
+	 */
+	public boolean isLoading(String docURL)
+	{	int i;
+
+		for (i = 0; i < this.toLoad.size(); i ++)
+		{	if (this.toLoad.elementAt(i).toString().equals(docURL))
+			{	return true;
+			}
+		}
+
+		for (i = 0; i < this.loading.size(); i ++)
+		{	if (((HTMLDoc) this.loading.elementAt(i)).urlString().equals(docURL))
+			{	return true;
+			}
+		}
+
+		for (i = 0; i < this.loaded.size(); i ++)
+		{	if (((HTMLDoc) this.loaded.elementAt(i)).urlString().equals(docURL))
+			{	return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 *	Start this object as a thread
+	 */
+	public synchronized void start()
+	{	if (this.thread == null)
+		{	this.thread	= new Thread(this);
+			this.thread.start();		// start the thread
+			this.stopFlag	= false;	// reset the stop flag as it could have been reset
+			this.started 	= true;		// note that we're running
+			running ++;
+		}
+	} 
+
+	/**
+	 *	The main thread function; it calls a number of private service functions to
+	 *  achieve it's affect.  Once the stop flag is set, it runs until all loading
+	 *  documents are loaded, then stops.
+	 */
+	public void run()
+	{ int i;
+
+		while (this.stopFlag == false || this.loading.size() > 0)
+		{	// slumber while there's nothing to do
+			while (this.toLoad.size() == 0 && this.loading.size() == 0)
+			{	try
+				{
+					thread.sleep(100);
+				}
+				catch (InterruptedException ex)
+				{
+				}
+			}
+
+			// first check for finished documents
+			i = 0;
+			while (i < this.loading.size())
+			{	HTMLDoc doc = (HTMLDoc) this.loading.elementAt(i);
+
+				if (doc.ready())
+				{	this.deschedule(i);
+				}
+				else
+				{	i ++;
+				}
+			}
+
+			// then check for a document to load
+			while (this.stopFlag == false &&
+						 this.toLoad.size() > 0 &&
+						 this.loading.size() < 10)
+			{	this.scheduleNext();
+			}
+		}
+		this.started = false;
+		running --;
+	}
+
+	/**
+	 *	Returns the total number of loader managers running at this time
+	 *
+	 *	@return <code>int</code> the number of managers running.
+	 */
+	public synchronized int running()
+	{	return running;
+	}
+
+	private synchronized void scheduleNext()
+	{	// pop url from list to load
+		URL docURL	= (URL) this.toLoad.elementAt(0);
+		this.toLoad.removeElementAt(0);
+
+		// generate new document and add it to the "loading" vector;
+		// the document content is loaded in another thread
+		HTMLDoc	doc = new HTMLDoc(docURL, 1);
+		this.loading.addElement(doc);
+	}
+
+	private synchronized void deschedule(int i)
+	{	// get the document and place it in the loaded list
+		HTMLDoc	doc = (HTMLDoc) this.loading.elementAt(i);
+		this.loaded.addElement(doc);
+
+		// remove it from the loading list
+		this.loading.removeElementAt(i);
+	}
+
+	/**
+	 *	Get a loaded document, not blocking if nothing is ready yet.
+	 *
+	 *  @return <code>HTMLDoc</code> the document; NB. this may be <code>null</code> as
+	 *                               it is not guaranteed that there is currently a loaded
+	 *                               document.
+	 */
+	public synchronized HTMLDoc nextLoaded()
+	{	HTMLDoc reply;
+
+		// if nothing is loaded, return null
+		if (this.loaded.size() == 0)
+		{	return null;
+		}
+
+		// get the first element from the loaded vector and return it,
+		// removing it from the vector in the meantime
+		reply = (HTMLDoc) this.loaded.elementAt(0);
+		this.loaded.removeElementAt(0);
+		return reply;
+	}
+
+	/**
+	 *	Request the loading of a document; this does not block but schedules
+	 *  the document for loading.
+	 *
+	 *  @param <code>URL</code> the URL of the document to load
+	 */
+	public synchronized void load(URL docURL)
+	{	// check if it's being loaded already; if so do nothing
+		if (this.isLoading(docURL.toString()) == false)
+		{	this.toLoad.addElement(docURL);
+
+			// self-start if the loading thread isn't running
+			if (this.started == false)
+			{	this.start();
+			}
+		}
+	}
+
+	/**
+	 *	Get the total number of documents loading/loaded through this loader
+	 *
+	 *	@param <code>int</code> the number of documents
+	 */
+	public synchronized int loadingDocs()
+	{	return this.toLoad.size() + this.loading.size() + this.loaded.size();
+	}
+
+	/**
+	 *	Instruct the manager to stop loading documents
+	 */
+	public synchronized void stopLoading()
+	{	this.stopFlag = true;
+	}
+
+	protected void finalize() throws Throwable
+	{	this.thread 	= null;
+		this.loading 	= null;
+		this.toLoad		= null;
+		this.loaded		= null;
+		super.finalize();
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java	(revision 12189)
@@ -0,0 +1,353 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+import java.net.*;
+
+import java.io.*;
+
+
+
+public class HTMLLoader implements Runnable
+
+{ URL 					from;
+
+	StringBuffer	HTML;
+
+	boolean				loaded;
+
+	boolean				error;
+
+	boolean				notFound;
+
+	Thread				thread;
+
+	static int		running;
+
+
+
+	/**
+
+	 * Constructor to load given URL
+
+	 */
+
+	public HTMLLoader(URL from)
+
+	{ this.from 	= from;
+
+		this.HTML 	= new StringBuffer();
+
+		this.loaded	= false;
+
+		this.error	= false;
+
+		this.thread = null;
+	}
+
+
+
+	/**
+
+	 * @return <code>0</code> if finished running; any other value if
+
+	 *         still executing
+
+	 */
+
+	public static synchronized int running()
+
+	{ return running;
+
+	}
+
+
+
+	/**
+
+	 * Starts the thread; the method <code>load</code> is a synonym for this
+
+	 */
+
+	public synchronized void start()
+
+	{ if (thread == null)
+
+		{ thread = new Thread(this);
+
+			thread.start();
+
+			running	++;
+
+		}
+
+//		System.out.println("Running: "+running);
+
+	}
+
+
+
+	/**
+
+	 * Do the actual loading; this will multi-thread with calling task
+
+	 */
+
+	public void run()
+
+	{	URLConnection connect;
+
+		int						length, i, c;
+
+		byte					inbuffer[];
+
+		InputStream		in;
+
+
+
+		inbuffer = new byte[128];
+
+
+
+		this.notFound	= false;
+//		System.out.println("Running loader");
+
+
+
+		// get connection and test length
+
+		try
+
+		{	connect = this.from.openConnection();
+
+		}
+
+		catch (IOException ioex)
+
+		{ this.loaded	= true;
+
+			this.error	=	true;
+
+			running --;
+			System.out.println(ioex);
+
+			return;
+
+		}
+
+
+		length = connect.getContentLength();
+
+		if (length == 0)
+
+		{ this.loaded = true;
+
+			running --;
+			System.out.println("Empty");
+
+			return;
+
+		}
+
+
+
+		// get an inputstream to the document body
+
+		try
+
+		{ in = connect.getInputStream();
+
+		}
+
+		catch (IOException ioex)
+
+		{ this.error	= true;
+
+			this.loaded	= true;
+
+			running --;
+			System.out.println("Open failed" + ioex);
+
+			return;
+
+		}
+
+
+
+		if (length < 0)
+
+		{	if (connect.getHeaderField(0) != null &&
+
+					connect.getHeaderField(0).indexOf("404") >= 0)
+
+			{	this.notFound	= true;
+
+				this.error		= true;
+
+				this.loaded		= true;
+
+				running --;
+				System.out.println("Not found");
+
+				return;
+
+			}
+
+		}
+
+//		System.out.println("Ready to read " + length);
+
+
+
+		// read in the document body
+
+		i = 0;
+
+		c = 0;
+
+		while (true || i < length)
+
+		{ // read in 128 characters (abort on IO Exception)
+
+			try
+
+			{ c = in.read(inbuffer);
+
+			}
+
+			catch (IOException ioex)
+
+			{ break;
+
+			}
+
+			// System.out.println("read " + c);
+
+
+
+			// if no characters were read, end, otherwise add it to the body
+
+			if (c != -1)
+
+			{ if (c > 0)
+
+				{ this.HTML.append(new String(inbuffer, 0, c));
+
+					i += c;
+
+				}
+
+			}
+
+			else
+
+			{ break;
+
+			}
+
+			// if an incomplete buffer was read (ie no characters remain), end
+
+/*			if (c < inbuffer.length && i >= length)
+
+			{ break;
+
+			}*/
+
+		}
+
+
+
+		// Close file
+
+		try
+
+		{	in.close();
+
+		}
+
+		catch (IOException ioex)
+
+		{	//System.out.println("Closed except "+ioex);
+
+		}
+
+	
+		this.loaded = true;
+
+  	running --;
+
+	}
+
+
+
+	/**
+
+	 * Do the actual loading (calls a Start on the Thread)
+
+	 */
+
+	public void load()
+
+	{	this.start();
+
+//		this.run();
+
+	}
+
+
+
+	/**
+
+	 * @return HTML text as a <code>String</code>.
+
+	 */
+
+	public synchronized String HTML()
+
+	{ if (this.error)
+
+		{ return "Error";
+
+		}
+
+		return this.HTML.toString();
+
+	}
+
+
+
+	/**
+
+	 * @return  <code>true</code> if the file has successfully loaded
+
+	 */
+
+	public boolean loaded()
+
+	{ return this.loaded;
+
+	}
+
+
+
+	protected void finalize() throws Throwable
+
+	{	this.thread = null;
+
+		this.from		= null;
+
+		this.HTML		= null;
+
+		super.finalize();
+
+	}
+
+
+
+	static
+
+	{ running = 0;
+
+	}
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java	(revision 12189)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLLocalAnchor
+{	
+	public static int anchorRef(HTMLBlockList anchors, String name)
+	{ int loop;
+		String	idvalue;
+
+		for (loop = 0; loop < anchors.size(); loop ++)
+		{	idvalue = anchors.tagBlock(loop).headTag().idValue("name");
+			if (idvalue != null && idvalue.equalsIgnoreCase(name))  //TODO: check if this should be case senstitive
+			{ return loop;
+			}
+		}
+		return -1;
+	}
+
+	public static int header(HTMLDoc doc, String local)
+	{	HTMLBlockList anchors;
+		int						heading;
+		HTMLBlock			headingblock;
+		int						anchor;
+		HTMLBlock			anchorblock;
+
+		anchors	= HTMLBlockList.anchorlist(doc);
+		anchor	= anchorRef(anchors, local);
+		if (anchor < 0)
+		{	return -1;
+		}
+
+		anchorblock	= anchors.tagBlock(anchor);
+		for (heading = 0; heading < doc.headerBlocks().size(); heading ++)
+		{	headingblock	= doc.headerBlocks().tagBlock(heading);
+
+			if (headingblock.startPos() > anchorblock.endPos())
+			{	break;
+			}
+		}
+		return heading - 1;
+	}
+	
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLObject.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLObject.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLObject.java	(revision 12189)
@@ -0,0 +1,5 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public interface HTMLObject
+{ public String getHTMLType();
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLParser.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLParser.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLParser.java	(revision 12189)
@@ -0,0 +1,243 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.applet.*;
+
+public class HTMLParser
+{ 
+    HTMLDoc document;
+    int pos;
+    int	lastpos;
+
+    public HTMLParser(HTMLDoc document)
+    { 
+	this.document = document;
+	this.pos = 0;
+    }
+    
+    public void startParse()
+    { 
+	this.pos = 0;
+	this.lastpos = -1;
+	//System.out.println("Starting "+document.urlString());
+    }
+
+    public int atParse()
+    { 
+	return this.pos;
+    }
+
+    public int lastParse()
+    { 
+	return this.lastpos;
+    }
+
+    // do a full text/tag parse
+    public String fullParse()
+    { 
+	String reply = null;
+	int end;
+	int start;
+
+	this.lastpos = this.pos;
+	
+	if (this.pos >= this.document.getContent().length()){
+	    return reply;
+	}
+	
+	if (this.document.getContent().charAt(this.pos) == '<'){
+	    start = this.pos;
+	    
+	    // if we're not at the end of the document,
+	    // read the rest of the tag
+	    if (this.pos == this.document.getContent().length() - 1){
+		this.pos = this.document.getContent().length();
+		return reply;
+	    }
+	    
+	    // if the tag is a comment
+	    if (this.pos < this.document.getContent().length() - 3 &&
+		this.document.getContent().substring(this.pos, this.pos+4).equals("<!--")){
+		end = this.document.getContent().substring(this.pos).indexOf("-->") + 3 + this.pos;
+		reply = this.document.getContent().substring(this.pos, end);
+		this.pos = end;
+	    }
+	    else 
+		{ // read up to the end of the tag
+		    end = this.pos + 1;
+		    while (end < this.document.getContent().length() &&
+			   this.document.getContent().charAt(end) != '>'){
+			end ++;
+		    }
+
+		    // get the whole of the tag into 'reply', and
+		    // set the current pos to immediately after the tag
+		    if (end < this.document.getContent().length()){
+			reply = this.document.getContent().substring(this.pos, end + 1);
+			this.pos = end + 1;
+		    }
+		    // patch the trailing > onto the tag string
+		    else {
+			reply = this.document.getContent().substring(this.pos, end) + ">";
+			this.pos = end;
+		    }
+		}
+	}
+	else
+	    { // hunt for the beginning of the next tag
+		start = this.pos;
+		while ((this.pos < this.document.getContent().length()) &&
+		       (this.document.getContent().charAt(this.pos) != '<')){
+		    this.pos ++;
+		}
+		
+		// return everything up to that tag
+	    reply = this.document.getContent().substring(start, this.pos);
+	    }
+	return reply;
+    }
+
+    // Get the next tag to parse
+    public String nextParse()
+    { 
+	String reply = null;
+	int end;
+
+	this.lastpos = this.pos;
+
+	if (this.document.getContent() == null){
+	    return null;
+	}
+	
+	if (this.pos >= this.document.getContent().length()){
+	    return reply;
+	}
+	
+	// hunt for the beginning of the next tag
+	while ((this.pos < this.document.getContent().length()) &&
+	       (this.document.getContent().charAt(this.pos) != '<')){
+	    this.pos ++;
+	}
+	
+	// if we're not at the end of the document,
+	// read the rest of the tag
+	if (this.pos < this.document.getContent().length()){
+	    if (this.pos == this.document.getContent().length() - 1){
+		this.pos = this.document.getContent().length();
+		return reply;
+	    }
+	    
+	    end = this.pos + 1;
+	    while (end < this.document.getContent().length() &&
+			this.document.getContent().charAt(end) != '>'){
+		end ++;
+	    }
+	    
+	    // get the whole of the tag into 'reply', and
+	    // set the current pos to immediately after the tag
+	    if (end < this.document.getContent().length()){
+		reply = this.document.getContent().substring(this.pos, end + 1);
+		this.pos = end + 1;
+	    }
+	    else{ 
+		this.pos = end;
+	    }
+	}
+	return reply;
+    }
+    
+    
+    /* --
+       -- return next HREF value we come across in the document
+       --
+       -- NB: the checking of the quotes etc is rather lazy - and should be tidied
+       --
+    */
+    public String nextHREF()
+    { 
+	String reply;
+	int start, end;
+	boolean quoted;
+	HTMLTag tag;
+	
+	reply = this.nextParse();
+	while (this.pos < this.document.getContent().length()){
+	    tag = new HTMLTag(reply);
+	    
+	    if (tag.tagName().equals("a")) {
+		start = reply.indexOf("href");
+		if (start == -1) {
+		    start = reply.indexOf("HREF");
+		}
+		
+		if (start >= 0) {
+		    start += 4;
+		    quoted = false;
+		    while (reply.charAt(start) == ' ' ||
+			   reply.charAt(start) == '=' ||
+			   reply.charAt(start) == '"') {
+			if (reply.charAt(start) == '"') {
+			    quoted = true;
+			}
+			start ++;
+		    }
+		    
+		    end = -1;
+		    if (quoted) {
+			end = reply.indexOf('"', start+1);
+		    }
+		    if (end == -1) {
+			end = reply.indexOf(' ', start+1);
+			if (end == -1) {
+			    end = reply.length() - 1;
+			}
+		    }
+		    
+		    reply = reply.substring(start, end);
+		    return reply;
+		}
+	    }
+	    reply = this.nextParse();
+	}
+	return null;
+    }
+
+    // Return the next link in the page
+    public String nextLink(AppletContext ac)
+    { 
+	String reply, reply2;
+	int start, end;
+	boolean quoted;
+	HTMLTag tag;
+	
+	if (this.document.getContent() == null){
+	    return null;
+	}
+	
+	reply = this.nextParse();
+	while (this.pos < this.document.getContent().length()){
+	    tag = new HTMLTag(reply);
+	    if (tag.tagName().equals("a")){
+		reply = tag.idValue("href");
+	    }
+	    else if (tag.tagName().equals("frame")){ // image tag
+		reply = tag.idValue("src");
+	    }
+	    else if (tag.tagName().equals("area")) { // image map area
+		reply = tag.idValue("href");
+	    }
+	    else if (tag.tagName().equals("frame")){
+		reply = tag.idValue("src");
+	    }
+	    else {
+		reply	= null;
+	    }
+	    
+	    if (reply != null){
+		return reply;
+	    }
+ 
+	    reply = this.nextParse();
+	}
+	return null;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTag.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTag.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTag.java	(revision 12189)
@@ -0,0 +1,265 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLTag implements HTMLObject
+{ String tagtext;
+  String name;
+  int	 docpos;
+  int    endpos;
+
+  public static final String HTML_ELEMENT_TYPE = "Element";
+
+  public HTMLTag(String tagtext, int docpos, int endpos)
+  { this.tagtext = tagtext;
+    this.name    = null;
+    this.docpos  = docpos;
+    if (this.docpos < 0)
+    { this.endpos = -1;
+    }
+    else
+    { this.endpos = endpos;
+    }
+  }
+
+  public HTMLTag(String tagtext)
+  { this.tagtext 	= tagtext;
+    this.docpos		= -1;
+    this.name			= null;
+  }
+
+  public String getHTMLType()
+  { return HTML_ELEMENT_TYPE;
+  }
+
+	public HTMLTag endTag()
+	{ HTMLTag end;
+		String	endtext;
+
+		endtext = "</"+this.xtagName()+">";
+
+		end = new HTMLTag(endtext);
+		return end;
+	}
+
+	public boolean tagIsOpening()
+	{ return (this.tagtext.charAt(1) != '/');
+	}
+
+	/**
+	 *  Tells you if a tag is a singleton tag - TODO: complete list
+	 */
+	public boolean tagIsSingleton()
+	{	String tagName = this.tagName();
+		
+		return (tagName.equals("img"));
+	}
+
+	/**
+	 *  @return: the name of a tag including any leading '/'
+	 */
+	public String tagName()
+	{ int offset, start, end;
+		String reply;
+
+		if (this.name != null)
+		{	return this.name;
+		}
+	
+		// NB: a starting '<' and an ending '>' are guaranteed
+		offset = 1; // skip <
+		while (this.tagtext.charAt(offset) <= 32 ||
+					 this.tagtext.charAt(offset) == 127)
+		{ offset ++;
+		}
+		start = offset;
+		while (this.tagtext.charAt(offset) > 32 &&
+					 this.tagtext.charAt(offset) != '>' &&
+					 this.tagtext.charAt(offset) != 127)
+		{ offset ++;
+		}
+		end = offset;
+
+		reply = this.tagtext.substring(start, end);
+		reply = reply.toLowerCase();
+
+		this.name	= reply;
+
+		return reply;
+	}
+
+	// Returns the name of the tag *minus* any leading / for
+	// closing tags
+	public String xtagName()
+	{ String reply;
+
+		reply = this.tagName();
+		if (reply == null || reply.length() == 0)
+		{	return null;
+		}
+
+		if (reply.charAt(0) == '/')
+		{ if (reply.length() > 1)
+			{	reply = reply.substring(1, reply.length());
+			}
+			else
+			{ reply = "";
+			}
+		}
+		return reply;
+	}
+
+	public String endTagName()
+	{	String reply;
+
+		reply = "/" + xtagName();
+		return reply;
+	}
+
+	public boolean isClosing()
+	{ String name;
+
+		name	= this.tagName();
+		if (name.charAt(0) == '/')
+		{	return true;
+		}
+		return false;
+	}
+
+	public int tagLevel()
+	{ String	name;
+		int			level;
+
+		level = 7;
+
+		name = this.xtagName();
+		if (name.charAt(0) == 'h')
+		{ level = Integer.parseInt(name.substring(1, 2));
+		}
+		
+		return level;
+	}
+
+	public int startPos()
+	{	return this.docpos;
+	}
+
+	public int endPos()
+	{	return this.endpos;
+	}
+
+	static public boolean isTag(String text)
+	{ if (text.charAt(0) == '<')
+		{ return true;
+		}
+		return false;
+	}
+
+	public String toString()
+	{	return this.tagtext;
+	}
+
+	//
+	// Returns the value of the identifier given as a string
+	//
+	public String idValue(String idName)
+	{	int offset, start, end;
+		int idstart, idend, valuestart, valueend;
+		boolean isvalue, isquoted, isstopped;
+		char		quotechar;
+		String reply;
+
+		// NB: a starting '<' and an ending '>' are guaranteed
+		offset = 1; // skip <
+		while (this.tagtext.charAt(offset) <= 32 ||
+					 this.tagtext.charAt(offset) == 127)
+		{ offset ++;
+		}
+		start = offset;
+		while (this.tagtext.charAt(offset) > 32 &&
+					 this.tagtext.charAt(offset) != '>' &&
+					 this.tagtext.charAt(offset) != 127)
+		{ offset ++;
+		}
+		end = offset;
+		// End of tag name
+
+		if (this.tagtext.charAt(offset) == '>')
+		{ return null;
+		}
+
+		isstopped = false;
+		while (!isstopped)
+		{	isvalue		= false;
+			isquoted	= false;
+			valuestart = -1;
+			valueend = -1;
+
+			while (this.tagtext.charAt(offset) <= 32 ||
+						this.tagtext.charAt(offset) == 127)
+			{ offset ++;
+			}
+			idstart = offset;
+
+			if (this.tagtext.charAt(offset) == '>')
+			{ break;
+			}
+
+			while (this.tagtext.charAt(offset) > 32 &&
+						 this.tagtext.charAt(offset) != '=' &&
+						 this.tagtext.charAt(offset) != '>' &&
+						 this.tagtext.charAt(offset) != 127)
+			{ offset ++;
+			}
+			idend = offset;
+
+			while (this.tagtext.charAt(offset) <= 32 ||
+						 this.tagtext.charAt(offset) == 127)
+			{ offset ++;
+			}
+
+			if (this.tagtext.charAt(offset) == '=')
+			{ isvalue = true;
+				quotechar = ' ';
+				offset ++;				
+
+				while (this.tagtext.charAt(offset) <= 32 ||
+							 this.tagtext.charAt(offset) == 127)
+				{ offset ++;
+				}
+				if (this.tagtext.charAt(offset) == '"' ||
+						this.tagtext.charAt(offset) == '\'')
+				{ quotechar = this.tagtext.charAt(offset);
+					isquoted = true;
+					offset ++;
+				}
+				valuestart = offset;
+
+				while (offset < this.tagtext.length() &&
+							 (isquoted && this.tagtext.charAt(offset) != quotechar) ||
+							 (!isquoted && this.tagtext.charAt(offset) > 32 &&
+							   this.tagtext.charAt(offset) != '>'))
+				{ offset ++;
+				}
+				valueend = offset;
+			}
+
+			if (offset >= this.tagtext.length() ||
+					this.tagtext.charAt(offset) == '>')
+			{ isstopped = true;
+			}
+			offset ++;
+
+			reply = this.tagtext.substring(idstart, idend);
+			reply = reply.toLowerCase();
+
+			if (reply.equalsIgnoreCase(idName))
+			{ if (isvalue)
+				{ return this.tagtext.substring(valuestart, valueend);
+				}
+				else
+				{ return reply;
+				}
+			}
+		}
+		return null;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java	(revision 12189)
@@ -0,0 +1,60 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLTagVector
+{ Vector vector;
+
+	//
+	// private: creates a Vector of tags from a comma separated list
+	//
+	public HTMLTagVector(String list)
+	{ int pos;
+		String tagname;
+
+		this.vector = new Vector(1);
+
+		// get first index
+		pos = list.indexOf(',');
+
+		// add leading tag to index, if there is > 1 tag currently
+		// in the list, and discard the tag
+		while (pos >= 0)
+		{ tagname = list.substring(0, pos);
+			list		= list.substring(pos+1, list.length());
+			this.vector.addElement((Object) tagname);
+
+			pos			= list.indexOf(',');
+		}
+
+		// add the last tag to the list
+		tagname = list;
+		this.vector.addElement((Object) tagname);
+	}
+
+	public HTMLTagVector(Vector vector)
+	{ this.vector = vector;
+	}
+
+	/**
+	 * Check whether a given tag is in the tagVector.  If <CODE>opening</CODE> is false,
+	 * then the method will return true for both opening and closing versions of the tag.
+	 * If the
+	 */
+	public boolean matches(HTMLTag tag, boolean opening)
+	{ int tagref;
+
+		if (tag.xtagName() == null)
+		{	return false;
+		}
+
+		for (tagref = 0; tagref < this.vector.size(); tagref ++)
+		{ if ((tag.xtagName().equals((String) this.vector.elementAt(tagref))) &&
+		      (tag.tagIsOpening() || !opening))
+			{ return true;
+			}
+		}
+		return false;
+	}
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java	(revision 12189)
@@ -0,0 +1,232 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.File;
+import java.io.FileReader;
+
+import java.net.URL;
+
+import org.w3c.dom.*;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.ext.LexicalHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.doctypes.GMLDocument;
+
+public class HTMLTidy extends DefaultHandler
+{
+    String label;
+    File   file;
+    StringBuffer value;
+    boolean inElement;
+    boolean accumulate;
+    Document document;
+    Element  currentElement;
+    Element root;
+
+    class HTMLCommentHandler implements LexicalHandler
+    {
+	HTMLTidy parent;
+
+	HTMLCommentHandler(HTMLTidy tidier)
+	{ 
+	    this.parent = tidier;
+	}
+
+	public void comment(char ch[], int start, int length)
+	{ 
+	    String comment = new String(ch, start, length);
+	    parent.comment(comment);
+	}
+	
+	public void endCDATA()
+	{
+	}
+
+	public void endDTD()
+	{
+	}
+
+	public void endEntity(String name)
+	{
+	}
+
+	public void startCDATA()
+	{
+	}
+
+	public void startDTD(String name, String publicId, String systemId)
+	{
+	}
+
+	public void startEntity(String name)
+	{
+	}
+    }
+
+    public HTMLTidy(URL url)
+    { 
+	// TODO: complete
+    }
+
+    public HTMLTidy(File file)
+    { 
+	this.file = null;
+	this.label = null;
+	this.value = null;
+	
+	this.document = new org.apache.html.dom.HTMLDocumentImpl();
+	
+	this.currentElement = this.document.getDocumentElement();
+	this.root = this.currentElement;
+	
+	try {
+	    XMLReader handler  = new org.ccil.cowan.tagsoup.Parser();
+	    HTMLCommentHandler commentHandler = new HTMLCommentHandler(this);
+	    handler.setContentHandler(this);
+	    handler.setProperty("http://xml.org/sax/properties/lexical-handler", commentHandler);
+	    handler.parse(file.toURI().toString());
+	}
+	catch (java.io.IOException io)
+	    {
+		System.out.println(io);
+	    }
+	catch (org.xml.sax.SAXException saxEx)
+	    {
+		System.out.println(saxEx);
+	    }
+    }
+    
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { 
+	//System.out.println("<"+localName+">");
+	
+	if (localName.equals("html") == false) {
+	    Element childElement = this.document.createElement(localName);
+	    
+	    this.currentElement.appendChild(childElement);
+	    this.currentElement = childElement;
+	    
+	    for (int a = 0; a < attributes.getLength(); a ++){
+		String name = attributes.getLocalName(a);
+		String value = attributes.getValue(a);
+		
+		childElement.setAttribute(name, value);
+	    }
+	}
+    }
+    
+    public void comment(String text)
+    {
+	Comment comment = document.createComment(text);
+	this.currentElement.appendChild(comment);
+	//System.out.println("<!-- "+text+" -->");
+    }
+    
+    public void endElement(String URI, String localName, String qName)
+    { 
+	// System.out.println("Ending " + localName);
+	// System.out.println("</"+localName+">");
+	
+	if (localName.equals("html")) {
+	    return;
+	}
+	
+	// TODO: check is the same name as current node
+	if (!this.currentElement.getTagName().equals(localName.toUpperCase())) {
+	    // TODO: error handling
+	    System.out.println("Bad end at " + localName + " " + this.currentElement.getTagName());
+	    return;
+	}
+	
+	// Optimisation: is this an empty tag?
+	boolean isEmpty = false;
+	
+	if (this.currentElement.getChildNodes().getLength() == 1){
+	    Node child = this.currentElement.getFirstChild();
+	    
+	    if (child.getNodeType() == Node.TEXT_NODE) {
+		String text = child.getNodeValue();
+		
+		int i = 0;
+		while (i < text.length()){
+		    if (text.charAt(i) > 32 && text.charAt(i) != 160) {
+			break;
+		    }
+		    i++;
+		}
+		if (text.length() == i) {
+		    isEmpty = true;
+		}
+	    }
+	}
+	else if (this.currentElement.getChildNodes().getLength() == 0){
+	    isEmpty = true;
+	}
+	
+	boolean isSingleton = false;
+	String tagName = this.currentElement.getTagName().toLowerCase();
+	
+	if (tagName.equals("img") ||
+	    tagName.equals("br") ||
+	    tagName.equals("meta") ||
+	    tagName.equals("input") ||
+	    tagName.equals("area") ||
+	    tagName.equals("link") ||
+	    tagName.equals("base") ||
+	    //tagName.equals("img") ||
+	    tagName.equals("hr")) {
+	    isSingleton = true;
+	}
+	
+	Element parent =  (Element) this.currentElement.getParentNode();
+
+	if (isEmpty == true && !isSingleton) {
+	    // trim the current child
+	    parent.removeChild(this.currentElement);
+	    //      System.out.println("***Removing child***");
+	}
+	
+	this.currentElement = parent;
+    }
+    
+    public void characters(char c[], int start, int length)
+    { 
+	String string = new String(c, start, length);
+	Node text_node = this.document.createTextNode(string);
+	this.currentElement.appendChild(text_node);
+	//    System.out.println(string);
+    }
+    
+    public void recursePrint(Element element, int indent)
+    {
+	for (int i = 0; i < indent; i++) {
+	    System.out.print(" ");
+	}
+	System.out.println(element.toString());
+	for (int c = 0; c < element.getChildNodes().getLength(); c ++){
+	    Node n = element.getChildNodes().item(c);
+	    
+	    if (n.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+		Element e = (Element) n;
+		
+		this.recursePrint(e, indent+2);
+	    }
+	}
+    }
+    
+    public Document getDocument()
+    { 
+	//    this.recursePrint(this.root, 0);
+	//    System.out.println(this.document.toString());
+	//    System.out.println(this.root);
+	return this.document;
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTTPTools.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTTPTools.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/HTTPTools.java	(revision 12189)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.IOException;
+import java.net.*;
+
+public class HTTPTools
+{
+	public static String getMIMEType(URL url)
+	{ try
+		{	// initiate the connection
+			URLConnection connection = url.openConnection();
+
+			connection.connect(); // actually make the connection
+
+			return connection.getContentType();
+		}
+		catch (IOException exception)
+		{ // TODO: Log error reading url here...
+		}
+		return null;
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/MultiMap.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/MultiMap.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/MultiMap.java	(revision 12189)
@@ -0,0 +1,74 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.ArrayList;
+
+public class MultiMap
+{	Map map;
+	
+	public MultiMap()
+	{	this.map = new HashMap();
+	}
+
+	public boolean containsKey(Object key)
+	{	return this.map.containsKey(key);
+	}
+
+	public void setOnly(Object key, Object value)
+	{	List child = new ArrayList();
+		child.add(value);
+		this.map.put(key, child);
+	}
+
+	public void put(Object key, Object value)
+	{	List children;
+
+		children = (List) this.map.get(key);
+		if (children == null)
+		{	children = new ArrayList();
+			this.map.put(key, children);
+		}
+		children.add(value);
+	}
+
+	public Object get(Object key)
+	{	List list = this.getAll(key);
+		if (list == null)
+		{	return null;
+		}
+		return list.get(0);
+	}
+
+	public List getAll(Object key)
+	{	return (List) this.map.get(key);
+	}
+
+	public void remove(Object key)
+	{	this.map.remove(key);
+	}
+
+	public boolean remove(Object key, Object value)
+	{	if (!this.map.containsKey(key))
+		{ return false;
+		}
+
+		List children = (List) this.map.get(key);
+
+		if (children != null)
+		{	for (int i = 0; i < children.size(); i ++)
+			{	if (children.get(i).equals(value))
+				{	children.remove(i);
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+	public Set keySet()
+	{	return this.map.keySet();
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/SQLUtils.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/SQLUtils.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/SQLUtils.java	(revision 12189)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class SQLUtils
+{
+	public void initCollection(String collectionName)
+	{	
+		// Check if the collection already exists
+		// if so, abort
+		// else
+		//   create document table
+		//   create section table
+		//   create metadata table
+		//   create namespace table
+		//   create metadata values table
+		//   create file section table
+		//   create file groups table
+		//   create file table
+		//   create admin table...etc.
+	}
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLString.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLString.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLString.java	(revision 12189)
@@ -0,0 +1,619 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.applet.*;
+import java.net.*;
+
+public class URLString
+{ private String url;
+
+  private String urlbase(String url)
+  { int index;
+    String tailString, bodyString;
+    int body;
+
+    index = url.indexOf("//");
+    tailString = url.substring(index+2, url.length());
+    
+    body = tailString.indexOf('/');
+    bodyString = tailString.substring(0, body+1);
+    
+    tailString = url.substring(0, index+2) + bodyString;
+    return tailString;
+  }
+  
+  public URLString(String s)
+  { this.url = s;
+  }
+
+  public URLString(String s, String target)
+  { // reconcile
+    int sref;
+    int cref;
+    int	c;
+
+    // trim fragment identifiers
+    target = noRef(target);
+    if (target == null)
+    { this.url = s;
+      return;
+    }
+
+    // test for absolute
+    cref = target.indexOf(':');
+    if (cref >= 0)
+    { for (c = 0; c < cref; c ++)
+      { if (target.charAt(c) != '+' &&
+	    target.charAt(c) != '-' &&
+	    target.charAt(c) != '.' &&
+	    Character.isLetterOrDigit(target.charAt(c)) == false)
+	{ break;
+	}
+      }
+    }
+    else
+    { c = 0;
+    }
+
+    if (c == cref)
+    { // absolute reference
+      String tmp;
+
+      tmp = target.substring(0, c);
+      tmp = tmp.toLowerCase();
+      if (target.indexOf("//") == cref + 1)
+      { if (target.lastIndexOf('/') == cref + 2)
+	{ target = target + "/";
+	}
+      }
+      /*			if (tmp.equals("http") == false)
+				{ this.url = null;
+				return;
+				}*/
+      this.url = target;
+    }
+    else if (target.indexOf("//") == 0)
+    { // absolute - implicit scheme (ie http).; also check for no trailing '/'
+      if (target.lastIndexOf('/') == 1)
+      { target = target + "/";
+      }
+      target = "http:" + target;
+      this.url = target;
+    }
+    else
+    { // relative - trim silly '/' at end
+      if (target.charAt(0) == '/')	
+      { // if we get a leading '/' then add it to the base of the url
+	s = urlbase(s);
+	target = target.substring(1);
+	this.url = s + target;
+//				System.out.println(this.url+"<"+s+"<"+target);
+	return;
+      }
+
+      while(target.length() >= 2 && target.charAt(0) == '.' && target.charAt(1) == '.')
+      { // move up a level on the left
+	
+	sref = s.lastIndexOf('/');
+	s = s.substring(0, sref);
+
+	target = target.substring(3);
+      }
+      // strip trailing leaf from s
+      sref = s.lastIndexOf('/');
+      s = s.substring(0, sref+1);
+      // append s+target
+      this.url = s + target;
+    }
+    
+    // abandon the url if it is a call to a script
+    if (this.url.indexOf('?') >= 0)
+    { this.url = null;
+    }
+  }
+
+  public static boolean dirParent(String parent, String child)
+  { while (child.length() >= parent.length())
+    { child = child.substring(0, child.lastIndexOf('/'));
+
+      if (parent.equals(child))
+      {	return true;
+      }
+    }
+    return false;
+  }
+
+  public static int dirPos(String from, String to)
+  { String fromdir;
+    String todir;
+
+    fromdir = from.substring(0, from.lastIndexOf('/'));
+    todir   = to.substring(0, to.lastIndexOf('/'));
+
+    if (fromdir.equals(todir))
+    { return 0;
+    }
+    if (fromdir.length() > todir.length())
+    { if (dirParent(todir, fromdir) == true)
+      {	return -1;
+      }
+    }
+    else 
+    { if (dirParent(fromdir, todir) == true)
+      { return 1;
+      }
+    }
+    return -2;
+  }
+
+  public static int commonLevels(String a, String b)
+  { String 	path_a, path_b;
+    int			offset_a, offset_b;
+    String	dir_a, dir_b;
+    int			common;
+
+    path_a	= path(a);
+    path_b	= path(b);
+    common	= 0;
+
+    if (path_a == null || path_b == null)
+    { return 0;
+    }
+
+    do
+    { offset_a = path_a.indexOf('/');
+      offset_b = path_b.indexOf('/');
+
+      if (offset_a > 0)
+      { dir_a		=	path_a.substring(0, offset_a);
+        if (path_a.length() > offset_a + 1)
+	{	path_a	= path_a.substring(offset_a+1);
+	}
+	else
+	{	path_a	= null;
+	}
+      }
+      else
+      {	dir_a		=	path_a;
+        path_a	= null;
+      }
+
+      if (offset_b > 0)
+      {	dir_b		=	path_b.substring(0, offset_b);
+        if (path_b.length() > offset_b + 1)
+	{	path_b	= path_b.substring(offset_b+1);
+	}
+	else
+	{	path_b	= null;
+	}
+      }
+      else
+      {	dir_b		=	path_b;
+        path_b	= null;
+      }
+
+      if (dir_a.length() != dir_b.length() ||
+	  dir_a.equals(dir_b) == false)
+      {	return common;
+      }
+      common ++;
+    } while (path_a != null && path_b != null);
+    
+    return common;
+  }
+
+	public static String dirLevelName(String url, int level)
+	{	String 	lpath;
+		String	dir;
+		int			thislevel = 0;
+		int			offset;
+
+		lpath = path(url);
+		if (lpath == null)
+		{ return null;
+		}
+		do
+		{ offset = lpath.indexOf('/');
+
+			if (offset >= 0)
+			{	dir = lpath.substring(0, offset);
+				lpath = lpath.substring(offset+1);
+			}
+			else
+			{	dir = lpath;
+			}
+
+			thislevel ++;
+
+			if (level == thislevel)
+			{ return dir;
+			}
+		} while (offset >= 0);
+
+		return null;
+	}
+
+	public static int dirLevel(String url)
+	{	String 	lpath;
+		int			level;
+		int			offset;
+
+		lpath = path(url);
+		if (lpath == null)
+		{	return 0;
+		}
+		level = 0;
+		do
+		{ offset = lpath.indexOf('/');
+			if (offset >= 0)
+			{	lpath = lpath.substring(offset+1);
+			}
+			level ++;
+		} while (offset >= 0);
+
+		return level;
+	}
+
+	public static String tidyURL(String urlString, boolean permitScript)
+	{	int cref;
+		int c;
+		int sref;
+		String tmp, s;
+
+		s = urlString;
+		cref = s.indexOf(':');
+		if (cref >= 0)
+		{ for (c = 0; c < cref; c ++)
+			{ if (s.charAt(c) != '+' &&
+						s.charAt(c) != '-' &&
+						s.charAt(c) != '.' &&
+						Character.isLetterOrDigit(s.charAt(c)) == false)
+				{ break;
+				}
+			}
+		}
+		else
+		{ c = 0;
+		}
+
+		if (cref == c)
+		{ tmp = s.substring(0, c).toLowerCase();
+			s = tmp + s.substring(c, s.length());
+			if (s.lastIndexOf('/') == cref + 2)
+			{ s = s + "/";
+			}
+		}
+		else
+		{ sref = s.indexOf("//");
+			if (sref == 0)
+			{ s = "http:" + s;
+			}
+			else
+			{ s = "http://" + s;
+			}
+			if (s.lastIndexOf('/') == 6)
+			{ s = s + "/";
+			}
+		}
+		return s;
+	}
+
+	public static String tidyURL(String urlString)
+	{	return tidyURL(urlString, false);
+	}
+
+	public static URL toURL(String urlString)
+	{ URL reply;
+
+	  try
+		{ reply = new URL(URLString.tidyURL(urlString));
+		}
+		catch (MalformedURLException ex)
+		{ return null;
+		}
+		return reply;
+	}
+
+  public void tidy()
+  { this.url = tidyURL(this.url);
+  }
+
+  public boolean isNull()
+  { if (this.url == null)
+    { return true;
+    } 
+    return false;
+  }
+
+  public static boolean isForeign(String home, URL url)
+  { if (url.toString().length() < home.length())
+    { return true;
+    }
+
+    if (home.equals(url.toString().substring(0, home.length())))
+    { return false;
+    }
+    return true;
+  }
+  
+  /**
+   *	Remove any trailing "#localanchor" local reference from a URL string
+   *
+   *  @param  <code>String</code> the original url
+   *  @return <code>String</code> the cleaned url, sans local references
+   */
+  public static String noRef(String target)
+  { if (target == null)
+    { return null;
+    }
+    if (target.indexOf('#') >= 0)
+    { if (target.indexOf('#') == 0)
+      { return null;
+      }
+      target = target.substring(0, target.indexOf('#'));
+    }
+    return target;
+  }
+
+  /**
+   *	Obtain the trailing local anchor reference from a URL, 
+   *  if it exists.
+   *    e.g. for http://www.test.mdx.ac.uk/example/index.html#credits the
+   *         response would be "credits"
+   *
+   *  @param  <code>String</code> the original url.
+   *  @return <code>String</code> the local reference.  <code>null</code> is
+   *                              returned if no local reference exists.
+   */
+  public static String subRef(String target)
+  { if (target.indexOf('#') >= 0)
+    {	return target.substring(target.indexOf('#')+1);
+    }
+    return null;
+  }
+
+  /**
+   *	Clean a URL of the form GET parameters.
+   *
+   *  @param  <code>String</code> the original url, with GET message parameters
+   *  @return <code>String</code> the cleaned url.
+   */
+  public static String noParameters(String url)
+  { int						c;
+
+    c = url.indexOf("?");
+    if (c < 0)
+    { return url;
+    }
+    return url.substring(0, c);
+  }
+
+	/**
+	 *	Obtain the form parameters from a url
+	 *
+	 *  @param  <code>String</code> the original url, with GET message parameters
+	 *  @return <code>String</code> the form "GET" parameters.  The leading '?' is
+	 *                              preserved in the reply.  If no parameters were
+	 *                              given, the result is <code>null</code>.
+	 */
+	public static String formParameters(String url)
+	{	int						c;
+
+		c = url.indexOf("?");
+		if (c < 0)
+		{	return null;
+		}
+		return url.substring(c);
+	}
+
+	/**
+	 *	Flatten a URL, making the '?' and '&' items into underscores
+	 *
+	 *  @param  <code>String</code> the original url, with GET message parameters
+	 *  @return <code>String</code> the "flattened" url
+	 */
+	public static String flattenedUrl(String oldUrl)
+	{	StringBuffer	reply;
+		int						c;
+		boolean				modified = false;
+
+		// replace ? with _
+		// replate & with _
+		reply = new StringBuffer(oldUrl);
+		for (c = 0; c < reply.length(); c ++)
+		{	if (reply.charAt(c) == '?' ||
+					reply.charAt(c) == '&')
+			{	reply.setCharAt(c, '_');
+				modified = true;
+			}
+		}
+
+		if (modified == true)
+		{	c	= oldUrl.indexOf(".html");
+			if (c >= 0)
+			{	while (c < oldUrl.length() - 5)
+				{	reply.setCharAt(c, reply.charAt(c+5));
+					c ++;
+				}
+				reply.setCharAt(c ++, '.');
+				reply.setCharAt(c ++, 'h');
+				reply.setCharAt(c ++, 't');
+				reply.setCharAt(c ++, 'm');
+				reply.setCharAt(c ++, 'l');
+			}
+		}
+
+		return reply.toString();
+	}
+
+	/**
+	 *	Provide the complete pathname in the given url
+	 *    eg. "example/index.html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> leaf file name
+	 */
+	public static String pathName(String url)
+	{	int offset;
+		String remainder;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset = url.indexOf("//");
+		if (offset < 0)
+		{ return null;
+		}
+
+		remainder = url.substring(offset + 2);
+		offset = remainder.indexOf('/');
+		if (offset < 0)
+		{	return null;
+		}
+		return remainder.substring(offset + 1);
+	}
+
+	/**
+	 *	Provide the leaf filename in the given url
+	 *    eg. "index.html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> leaf file name
+	 */
+	public static String leafName(String url)
+	{	int offset;
+		int dot;
+
+		if (url == null)
+		{	return null;
+		}
+		offset = url.lastIndexOf('/');
+		if (offset < 0)
+		{ return null;
+		}
+		if (offset == url.length() - 1)
+		{ return null;
+		}
+		dot = url.lastIndexOf('/');
+		if (dot < offset)
+		{	return null;
+		}
+		return url.substring(offset+1);
+	}
+
+	/**
+	 *	Provide the hostname in the given url
+	 *    eg. "www.test.mdx.ac.uk" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> host name
+	 */
+	public static String host(String url)
+	{	String scheme, tail;
+		int		 spos;
+
+		spos = url.indexOf("//");
+		if (spos >= 0)
+		{ scheme = url.substring(0, spos + 2);
+			tail = url.substring(spos + 2, url.length());
+			// do sub-areas
+			spos = tail.indexOf('/');
+			if (spos >= 0)
+			{ tail = tail.substring(0, spos+1);
+			}
+			scheme = scheme + tail;
+			return scheme;
+		}
+		return null;
+	}
+
+	/**
+	 *	Return the logical path of the url on the remote server
+	 *    eg. "example" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> logical path
+	 */
+	public static String path(String url)
+	{ String tail;
+		int offset;
+		String leaf;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset 	= url.indexOf("//");
+		if (offset < 0)
+		{	return null;
+		}
+
+		tail		= url.substring(offset+2);
+		offset	= tail.indexOf('/');
+		if (offset < 0)
+		{	return null;
+		}
+
+		tail		= tail.substring(offset+1);
+		if (tail == null || tail.length() == 0)
+		{	return null;
+		}
+
+		offset	= tail.lastIndexOf('/');
+		if (offset < 0)
+		{ return null;
+		}
+		else
+		{	tail = tail.substring(0, offset);
+		}
+		return tail;
+	}
+
+	/**
+	 *	Give the 'dot' extension of a url; 
+	 *    eg. "html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *	@param <code>String</code> url
+	 *  @return <code>String</code> extension.
+	 */
+	public static String extension(String url)
+	{ int offset;
+		String leaf;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset = url.lastIndexOf('/');
+		if (offset >= 0 && offset < url.length() - 1)
+		{ leaf = url.substring(offset+1, url.length());
+			offset = leaf.lastIndexOf('.');
+
+			// if we've got an extension then use it
+			if (offset >= 0)
+			{ int poffset;
+
+				// get parameter offset
+				poffset = leaf.indexOf('?');
+				if (poffset < 0)
+				{	poffset = leaf.length();
+				}
+
+				// give leaf value
+				return leaf.substring(offset, poffset);
+			}
+			else
+			{ return null;
+			}
+		}
+		return null;
+	}
+
+  public static String extension(URL url)
+  { return extension(url.toString());
+  }
+
+  public String toString()
+  { return this.url;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLTools.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLTools.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/URLTools.java	(revision 12189)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.URL;
+import java.io.File;
+
+public class URLTools
+{
+  public static URL getFileURL(File file)
+  { URL url;
+    String normalisedFileName = file.toString().replace(File.separatorChar, '/');
+
+    try {
+      url = new URL("file", "", normalisedFileName);
+    }
+    catch (java.net.MalformedURLException ex) {
+      return null;
+    }
+    return url;
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/XMLTools.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/XMLTools.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/util/XMLTools.java	(revision 12189)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class XMLTools
+{
+    public static String getTag(String namespace, String label, boolean open)
+    {	
+	StringBuffer reply = new StringBuffer();
+
+	reply.append("<");
+	if (!open)
+	    {	
+		reply.append("/");
+	    }
+	if (namespace != null && namespace.length() > 0) {
+	    reply.append(namespace);
+	    reply.append(":");
+	}
+	reply.append(label);
+	reply.append(">");
+	return reply.toString();
+    }
+
+    public static String addAttribute(String tag, String attribute, String value)
+    {	
+	StringBuffer reply = new StringBuffer(tag);
+	reply.insert(reply.length() - 1 , ' ');
+	reply.insert(reply.length() - 1 , attribute);
+	reply.insert(reply.length() - 1 , "=\"");
+	reply.insert(reply.length() - 1 , value);
+	reply.insert(reply.length() - 1 , '\"');
+	return reply.toString();
+    }
+
+    public static String makeSingleton(String tag)
+    {	
+	StringBuffer reply = new StringBuffer(tag);
+	reply.insert(tag.length() - 1, " /");
+	return reply.toString();
+    }
+    
+    public static String getOpenTag(String namespace, String label)
+    {	
+	return getTag(namespace, label, true);
+    }
+    
+    public static String getCloseTag(String namespace, String label)
+    {	
+	return getTag(namespace, label, false);
+    }
+
+    /**
+     *  Clean a string to remove whitespace...
+     *
+     *  @param <code>String</code> the original text
+     *  @return <code>String</code> the 'cleaned' text
+     */
+    public static String cleanString(String original)
+    {	
+	StringBuffer buffer = new StringBuffer(original);
+
+	while (buffer.charAt(0) <= 32) {
+	    buffer.deleteCharAt(0);
+	}
+	
+	while (buffer.charAt(buffer.length() - 1) <= 32){
+	    buffer.deleteCharAt(buffer.length() - 1);
+	}
+	
+	int c = 1;
+	while (c < buffer.length()){
+	    if (buffer.charAt(c) < 32){
+		buffer.setCharAt(c, ' ');
+	    }
+	    if (buffer.charAt(c) == ' ' && buffer.charAt(c-1) == ' '){
+		buffer.deleteCharAt(c);
+		continue;
+	    }
+	    c ++;
+	}
+	return buffer.toString();
+    }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java	(revision 12189)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.validator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public interface ValidatorInterface
+{
+	/**
+	 *  Record the build manager to use for this expander - this is used
+	 *  to pass back any new directories into which expanded files are placed.
+	 */
+	public void configure();
+
+	/**
+	 *  Attempt to expand a file.  The returned result is <code>true</code> if the
+	 *  file was actually expanded.
+	 *
+	 *  @param <code>DocumentID</code> the identity code of the document
+	 *  @param <code>DocumentInterface</code> the document itself
+	 *
+	 *  @return <code>boolean</code> whether the file was validated.
+	 */
+	public boolean validateDocument(DocumentID docID, DocumentInterface document);
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java	(revision 12189)
@@ -0,0 +1,59 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+public class IdentifierXPointer extends XPointer
+{ NodeList nodes;
+
+  public IdentifierXPointer(Document document, String xpointer)
+  { super(document, xpointer);
+
+    this.xpointer = XPointer.expandXPath(this.xpointer);
+
+    this.nodes = this.getNodes();
+  }
+
+  private NodeList getNodes()
+  {
+    try {
+      NodeList nodes = XPathAPI.selectNodeList(this.document, this.xpointer, 
+					       this.document.getDocumentElement());
+      return nodes;
+    }
+    catch (Exception ex)
+    { System.out.println(ex.toString());
+    }
+    return null;
+  }
+
+  public Node getStartNode()
+  { return this.nodes.item(0);
+  }
+
+  private void recurseStringify(NodeList nodes, StringBuffer buffer)
+  { 
+    for (int n = 0; n < nodes.getLength(); n ++) {
+      Node node = nodes.item(n);
+
+      // append the actual node itself
+      this.printNode(node, buffer, false);
+
+      // then process its children...
+      if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	NodeList children = node.getChildNodes();
+	this.recurseStringify(children, buffer);
+      }
+
+      // do the close node
+      this.printNode(node, buffer, true);
+    }
+  }
+
+  public String toString()
+  {
+    StringBuffer reply = new StringBuffer();
+    this.recurseStringify(this.nodes, reply);
+    return reply.toString();
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java	(revision 12189)
@@ -0,0 +1,69 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+public class RangeToXPointer extends XPointer
+{
+  public static final String RANGE_TO = "range-to(";
+  NodeList startNodes;
+  NodeList endNodes;
+
+  public RangeToXPointer(Document document, String xpointer)
+  { super(document, xpointer);
+
+    String fromPointer, toPointer;
+
+    int rangeAt = xpointer.indexOf(RANGE_TO);
+
+    fromPointer = xpointer.substring(0, rangeAt-1);
+    toPointer = xpointer.substring(rangeAt+RANGE_TO.length(), xpointer.length()-1);
+
+    fromPointer = XPointer.expandXPath(fromPointer);
+    toPointer   = XPointer.expandXPath(toPointer);
+    //    System.out.println(fromPointer + " " + toPointer);
+    
+    this.startNodes = this.getNodes(fromPointer);
+    this.endNodes = this.getNodes(toPointer);
+    
+    //    System.out.println(this.startNodes.item(0).toString() + " " + this.endNodes.item(0).toString());
+  }
+
+  public Node getStartNode()
+  { return this.startNodes.item(0);
+  }
+
+  private NodeList getNodes(String pointer)
+  {
+    try {
+      NodeList nodes = XPathAPI.selectNodeList(document, pointer, document.getDocumentElement());
+      return nodes;
+    }
+    catch (Exception ex)
+    { System.out.println(ex.toString());
+    }
+    return null;
+  }
+
+  private void recurseStringify(Node at, Node end, StringBuffer reply)
+  {
+    while (at != end && at != null) {
+      this.printNode(at, reply, false);
+      at = this.getNextNode(at, reply);
+      //      System.out.println(at + " " + end);
+    }
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    // skip the first node
+    Node firstNode = this.startNodes.item(0);
+    firstNode = this.getNextNode(firstNode, reply);
+    
+    // actually, we don't, as otherwise the indexed copy of 
+    // the document would be incomplete...
+    this.recurseStringify(this.startNodes.item(0), this.endNodes.item(0), reply);
+    return reply.toString();
+  }
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java	(revision 12189)
@@ -0,0 +1,281 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import java.io.File; 
+
+import java.net.URL;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocumentTools;
+import org.greenstone.gsdl3.gs3build.util.HTMLTidy;
+
+public abstract class XPointer
+{
+  Document document;  
+  String   xpointer;
+
+  public XPointer(org.w3c.dom.Document document, String xpointer)
+  { this.document = document;
+    this.xpointer = xpointer;
+    // now process the type of description
+  }
+
+  public static String expandXPath(String xpath)
+  {
+    int          dashAt;
+    String       pathElement;
+    StringBuffer expandedPtr = new StringBuffer();
+
+    if (xpath.startsWith("//")) {
+      xpath = xpath.substring(2);
+      expandedPtr.append("//");
+    }
+    else if (xpath.startsWith("/")) {
+      xpath = xpath.substring(1);
+      expandedPtr.append("/");
+    }
+
+    while (xpath.length() > 0) {
+      dashAt = xpath.indexOf('/');
+      if (dashAt >= 0) {
+	pathElement = xpath.substring(0, dashAt);
+	xpath = xpath.substring(dashAt+1);
+      }
+      else {
+	pathElement = xpath;
+	xpath = "";
+      }
+
+      int c = 0;
+      while (c < pathElement.length()) {
+	if (!Character.isDigit(pathElement.charAt(c))) 
+	  break;
+	c ++;
+      }
+
+      if (c == pathElement.length()) {
+	expandedPtr.append("*[");
+	expandedPtr.append(pathElement);
+	expandedPtr.append("]");
+      }
+      else {
+	expandedPtr.append(pathElement);
+      }
+
+      if (dashAt >= 0) {
+	expandedPtr.append('/');
+      }
+    }
+    xpath = expandedPtr.toString();
+
+    return xpath;
+  }
+
+  public static XPointer processXPointer(Document document, String xpointer)
+  { // do a "cheap and dirty" detection of range-to
+    if (xpointer.indexOf("range-to") >= 0) {
+      return new RangeToXPointer(document, xpointer);
+    } 
+    return new IdentifierXPointer(document, xpointer);
+  }
+
+  public static XPointer processXPointer(Document document, URL url)
+  { String reference = url.getRef();
+
+    reference.trim();
+    if (reference.startsWith("xpointer(")) {
+      reference = reference.substring(9, reference.length() - 1);
+    }
+    else {
+      return null;
+    }
+    //    System.out.println(reference);
+    
+    return processXPointer(document, reference);
+  }
+
+  public static void printNode(Node node, StringBuffer to, boolean close)
+  {
+    if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+      if (!close) {
+	to.append(node.getNodeValue());
+	to.append(" ");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+      if (!close) {
+	to.append("&");
+	to.append(node.getNodeValue());
+	to.append(";");
+	to.append(" ");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      if (!close) {
+	to.append("<!-- ");
+	to.append(node.getNodeValue());
+	to.append(" -->");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      boolean hasChildren = (node.getChildNodes().getLength() > 0);
+
+      if (close && !hasChildren) {
+	return;
+      }
+      to.append("<");
+      if (close) {
+	to.append("/");
+      }
+      to.append(node.getNodeName());
+
+      if (!close) {
+	NamedNodeMap attributes = ((Element) node).getAttributes();
+	for (int a = 0; a < attributes.getLength(); a ++) {
+	  Node attributeNode = attributes.item(a);
+	  to.append(" ");
+	  to.append(attributeNode.getNodeName());
+	  
+	  String value = attributeNode.getNodeValue();
+	  if (value != null && value.length() > 0) {
+	    to.append("=\"");
+	    to.append(value);
+	    to.append("\"");
+	  }
+	}
+      }
+
+      if (!hasChildren) {
+	to.append(" /");
+      }
+      to.append(">");
+      if (close && node.getNodeName()=="Section") {
+	  to.append((char) 3);
+      }
+    }
+  }
+
+    public static Node getNextNode(Node currentNode) {
+	
+	// Try to get the first valid child
+	NodeList children = currentNode.getChildNodes();
+	for (int c = 0; c < children.getLength(); c++) {
+	    Node childNode = children.item(c);
+	    
+	    if (childNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.COMMENT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.TEXT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+		return childNode;
+	    }
+	}	
+	
+	// now try a siblings and parents siblings
+	Node siblingNode = null;
+	do {
+	    siblingNode = currentNode.getNextSibling();
+	    if (siblingNode != null) {
+		// should we be checking type??
+		return siblingNode;
+	    }
+	    
+	    currentNode = currentNode.getParentNode();
+	} while (currentNode != null);
+	
+	return null;
+    }
+
+    
+  public static Node getNextNode(Node currentNode, StringBuffer buffer)
+  { Node reply;
+
+    // Try for children only the once...
+    NodeList children = currentNode.getChildNodes();
+    for (int c = 0; c < children.getLength(); c++) {
+      Node childNode = children.item(c);
+
+      if (childNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.COMMENT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.TEXT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+	return childNode;
+      }
+    }
+    
+    // Repeat looking for a sibling & going up until done...
+    do {
+      reply = currentNode.getNextSibling();
+      if (reply != null) {
+	return reply;
+      }
+      
+      currentNode = currentNode.getParentNode();
+      if (buffer != null && currentNode != null) {
+	printNode(currentNode, buffer, true);
+      }
+    } while (currentNode != null);
+
+    return reply;
+  }
+
+  public abstract Node getStartNode();
+
+
+  public static void main(String args[])
+  {	
+    try
+    { File file = new File(args[0]);
+
+      Document document;
+      if (args[0].endsWith(".html") || args[0].endsWith(".htm")) {
+	HTMLTidy tidier = new HTMLTidy(file);
+	document = tidier.getDocument();
+
+	HTMLDocumentTools docTools = new HTMLDocumentTools(document);
+	docTools.findSections();
+	System.out.println("Sections found");
+      }
+      else {
+	DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	DocumentBuilder builder = factory.newDocumentBuilder();
+	document = builder.parse(file);
+      }
+      
+      XPointer pointer = null;
+      if (args.length < 2) {
+      }
+      else if (args.length == 2) {
+	pointer = XPointer.processXPointer(document, args[1]);
+      }
+      else {
+	pointer = XPointer.processXPointer(document, args[1]+"/range-to("+args[2]+")");
+      }
+      if (pointer != null) {
+	System.out.println(pointer.toString());
+      }
+    }
+    catch (FactoryConfigurationError e) {
+      System.err.println(e);
+    }
+    catch (ParserConfigurationException ex) {
+      System.err.println(ex);
+    }
+    catch (SAXException ex) {
+      System.err.println(ex);
+    }
+    catch (java.io.IOException ex) {
+      System.err.println(ex);
+    }
+  }
+}
+
+
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 12189)
@@ -0,0 +1,218 @@
+/*
+ *    AbstractGS3DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Implements the generic retrieval and classifier services for GS3 
+ * collections.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractGS3DocumentRetrieve
+    extends AbstractDocumentRetrieve {
+    protected static final String INDEX_STEM_ELEM = "indexStem";
+
+    protected SQLQuery database = null;
+    protected String index_stem = null;
+
+
+    /** constructor */
+    protected AbstractGS3DocumentRetrieve()
+    {
+	this.database = new SQLQuery();
+	// set up a macro resolver
+    }
+    
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractGS3DocumentRetrieve...");
+
+	// the index stem is either specified in the config file or is "index"
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    System.err.println("GS3MGRetrieve.configure(): indexStem element not found, stem will default to 'index'");
+	    this.index_stem = "index";
+	}
+
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+    
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getTop(node_id);
+    }
+    
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	if (GS3OID.isDocTop(node_id)) { // hack
+	    node_id =  GS3OID.createOID(node_id, "1");
+	}
+	ArrayList documents = database.getDocumentChildren(node_id);
+	if (documents == null || documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+    
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata,
+				      ArrayList metadata_names)   
+	throws GSException {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getDocumentMetadata(node_id, metadata);
+		if (value != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null; // not implemented yet
+    }
+
+    
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+    }
+
+    protected String resolveRelativeLinks(String doc_content, String doc_id) {
+
+	String http_path  = this.site_http_address + "/collect/"+this.cluster_name;
+	doc_content = doc_content.replaceAll("_httpcollection_", http_path);
+	return doc_content;
+	
+    }
+
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id) throws GSException;
+
+}   
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3Browse.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3Browse.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3Browse.java	(revision 12189)
@@ -0,0 +1,193 @@
+/*
+ *    GS3Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+import java.io.File;
+
+/** Greenstone 3 collection classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+public class GS3Browse
+    extends AbstractBrowse
+{
+
+    protected SQLQuery database = null;
+
+    /** constructor */
+    public GS3Browse()
+    {
+	this.database = new SQLQuery();
+
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+    
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.err.println("Configuring GS3Browse...");
+	
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+	
+	return true;
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+
+        /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+   }
+    
+    /** returns the id of the root node of the document containing node 
+     * node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getDocID(node_id);
+    }
+    
+   /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	ArrayList documents = database.getClassifierDocChildren(node_id);
+	if (documents == null) {
+	    documents = new ArrayList();
+	}
+	ArrayList children = database.getClassifierChildren(node_id);
+	if (children != null) {
+	    documents.addAll(children);
+	}
+	if (documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+    
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata, ArrayList metadata_names) {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getClassifierMetadata(node_id, metadata);
+		if (value != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null;
+    }
+
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    protected boolean isDocumentId(String node_id){
+	if (node_id.startsWith("CL")) {
+	    return false;
+	}
+	return true;
+    }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 12189)
@@ -0,0 +1,121 @@
+/*
+ *    GS3MGPPRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGPPRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+
+    private MGPPWrapper mgpp_src = null;
+    
+    private String default_level = null;
+    private String mgpp_textdir = null;
+	
+    public GS3MGPPRetrieve() {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS3MGPPRetrieve...");
+
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    System.err.println("Error: default level not specified!");
+	    return false;
+	}
+	// System.out.println("Default level: " + default_level_);
+	
+	// The location of the MGPP text files
+	mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionTextPath(this.index_stem);
+
+	return true;
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mgpp_src.getDocument(this.mgpp_textdir, 
+						    this.default_level,
+						    doc_int);
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+	} catch (Exception e) {
+	    System.out.println("exception happended with mgpp_src.getDocument()" + e);
+	    doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_int+"\n";
+	}
+
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPSearch.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPSearch.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGPPSearch.java	(revision 12189)
@@ -0,0 +1,141 @@
+/*
+ *    GS3MGPPSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.io.File;
+
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public class GS3MGPPSearch 
+    extends AbstractMGPPSearch 
+{
+    protected SQLQuery database = null;
+    /** the base index prefix */
+    protected String base_index_prefix = null;
+
+
+    /** constructor */
+    public GS3MGPPSearch()
+    {
+	this.database = new SQLQuery();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// get the base prefix
+	Element def = (Element) GSXML.getChildByTagName(info, "baseIndexPrefix");
+	if (def != null) {
+	    this.base_index_prefix = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.base_index_prefix == null || this.base_index_prefix.equals("")) {
+	    System.err.println("Error: base index prefix not specified!");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Search Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Search Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	return database.documentHasChildren(node_id);
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MGPP internal id to Greenstone oid */
+    protected String MGPPNum2OID(long docnum)
+    {
+	String mg_id = this.base_index_prefix+"."+docnum;
+	String doc_id = database.MGNum2OID(mg_id);
+	// hack !!!
+	if (doc_id.endsWith("-All")) {
+	    // get rid of the All bit
+	    doc_id = doc_id.substring(0,doc_id.length()-4);
+	    //doc_id=doc_id.replaceAll("All","1");
+	}
+	return doc_id;
+	
+    }
+
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGRetrieve.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 12189)
@@ -0,0 +1,133 @@
+/*
+ *    GS3MGRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+
+    private MGWrapper mg_src = null;
+    private String mg_basedir = null;
+    private String mg_textdir = null;
+    private String default_index = null;
+    
+    public GS3MGRetrieve() {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS3MGRetrieve...");
+	// System.out.println("info:\n" + converter_.getString(info));
+	// System.out.println("extra_info:\n" + converter_.getString(extra_info));
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_index == null || this.default_index.equals("")) {
+	    System.err.println("Error: default index not specified!");
+	    return false;
+	}
+
+	// System.out.println("Default index: " + this.default_index);
+
+	
+	// The location of the MG index and text files
+	mg_basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+	mg_textdir = GSFile.collectionTextPath(this.index_stem);
+	// index is only needed to start up MG, not used so just use the default index
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mg_src.setIndex(indexpath);
+	return true;
+    }
+    
+    /** returns the content of a node.
+     * node_id should already have been translated if necessary
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mg_src.getDocument(this.mg_basedir, 
+						  this.mg_textdir, 
+						  doc_int);
+	    // remove any ctrl-c or ctrl-b 
+	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+
+	} catch (Exception e) {
+	    System.out.println("exception happended with mg_src.getDocument()");
+	    throw new GSException("Couldn't get document content for id: "+ doc_int+"\n"+e.getMessage());
+	    //doc_content = "this is the content for section hash id "+ doc_id+", mg doc num "+doc_int+"\n";
+	}
+	
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGSearch.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGSearch.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/service/GS3MGSearch.java	(revision 12189)
@@ -0,0 +1,141 @@
+/*
+ *    GS3MGSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.io.File;
+
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public class GS3MGSearch 
+    extends AbstractMGSearch 
+{
+    protected SQLQuery database = null;
+    /** the base index prefix */
+    protected String base_index_prefix = null;
+
+
+    /** constructor */
+    public GS3MGSearch()
+    {
+	this.database = new SQLQuery();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// get the base prefix
+	Element def = (Element) GSXML.getChildByTagName(info, "baseIndexPrefix");
+	if (def != null) {
+	    this.base_index_prefix = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.base_index_prefix == null || this.base_index_prefix.equals("")) {
+	    System.err.println("Error: base index prefix not specified!");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Search Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Search Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	return database.documentHasChildren(node_id);
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MG internal id to Greenstone oid */
+    protected String MGNum2OID(long docnum)
+    {
+	String mg_id = this.base_index_prefix+"."+docnum;
+	String doc_id = database.MGNum2OID(mg_id);
+	// hack !!!
+	if (doc_id.endsWith("-All")) {
+	    // get rid of the All bit
+	    doc_id = doc_id.substring(0,doc_id.length()-4);
+	    //doc_id=doc_id.replaceAll("All","1");
+	}
+	return doc_id;
+	
+    }
+
+}
+
+
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/GS3OID.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/GS3OID.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/GS3OID.java	(revision 12189)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.util;
+
+public class GS3OID extends OID {
+
+    /**  public static boolean isTop(String oid) {
+	if (oid.equals("")) {
+	    return false;
+	}
+	int pos = oid.indexOf('.');
+	if (pos == -1) { // its the top document, not the top structure
+	    return false;
+	}
+	pos = pod.indexOf('.', pos+1);
+	if (pos == -1) {
+	    return true; // there is only one dot
+	}
+	return false; // there is more than one dot
+		
+	}*/
+
+    public static boolean isDocTop(String oid) {
+	return oid.indexOf('-')==-1;
+    }
+
+    public static String createOID(String doc_id, String section_label) {
+	return doc_id + "-"+section_label;
+    }
+
+    public static String getDocID(String oid) {
+	if (isDocTop(oid)) {
+	    return oid;
+	}
+	return oid.substring(0, oid.indexOf('-'));
+    }
+    public static String getSectionLabel(String oid) {
+
+	if (isDocTop(oid)) {
+	    return "";
+	}
+	return oid.substring(oid.indexOf('-')+1);
+
+    }
+
+    // put this here for now - may need to go with SQL stuff later
+    public static String translateOID(String oid) {
+	
+	int p = oid.lastIndexOf('.');
+	if (p != oid.length()-3) {
+	    System.out.println("translateoid error: '.' is not the third to last char!!");
+	    return oid;
+	}
+	
+	String top = oid.substring(0, p);
+	String suff = oid.substring(p+1);
+	if (suff.equals("pr")) {
+	    return getParent(top);
+	} else if (suff.equals("rt")) {
+	    return getTop(top);
+	} 
+	return oid;
+
+    }
+
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/GSSQL.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/GSSQL.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/GSSQL.java	(revision 12189)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.util;
+
+public class GSSQL {
+
+    // table names
+    public static final String DOCUMENT_TABLE = "document";
+    public static final String STRUCTURE_TABLE = "structure";
+    public static final String DIVISION_TABLE = "divisions";
+    public static final String DIVISION_FILE_TABLE = "divisionfilerefs";
+    public static final String DIVISION_METADATA_TABLE = "divisionmetarefs";
+    public static final String FILE_SECTION_TABLE = "filesection";
+    public static final String FILE_GROUP_TABLE = "filegroups";
+    public static final String FILE_TABLE = "files";
+    public static final String METADATA_TABLE = "metadata"; 
+    public static final String NAMESPACE_TABLE = "namespaces";
+    public static final String METADATA_VALUE_TABLE = "mdvalues";
+    public static final String CLASSIFIER_TABLE = "classifiers";
+    public static final String CLASS_DATA_TABLE = "classdata";
+    public static final String CLASS_DOCUMENT_TABLE = "classdocuments";
+
+    // unique key names - references to the tables
+    public static final String METADATA_REF = "MetadataRef";
+    public static final String NAMESPACE_REF = "NamespaceRef";
+    public static final String STRUCTURE_REF = "StructureRef";
+    public static final String DIVISION_REF = "DivisionRef";
+    public static final String FILE_SECTION_REF = "FileSectionRef";
+    public static final String FILE_GROUP_REF = "FileGroupRef";
+    public static final String CLASSIFIER_REF = "ClassifyRef";
+    
+    // combined refs
+    public static final String PARENT_REF = "ParentRef";
+    public static final String PARENT_TYPE = "ParentType";
+    
+    // id fields - these are unique ids per document - ie combined with docid
+    public static final String DOCUMENT_ID = "DocID";
+    public static final String METADATA_ID = "MetaID";
+    public static final String FILE_ID = "FileID";
+    public static final String STRUCTURE_ID = "StructureID";
+    public static final String SECTION_ID = "SectionID";
+    public static final String FILE_GROUP_ID = "FileID";
+    public static final String FILE_SECTION_ID = "FileID";
+    public static final String CLASSIFIER_ID = "ClassifyID";
+    public static final String PARENT_ID = "ParentID";
+    
+    // type fields
+    public static final String DOC_TYPE = "DocType";
+    public static final String NAMESPACE_TYPE = "NamespaceType";
+    public static final String STRUCTURE_TYPE = "StructureType";
+    public static final String DIVISION_TYPE = "DivisionType";
+    public static final String FILE_LOCATION_TYPE = "FileLocType";
+    public static final String MIME_TYPE = "MIMEType";
+
+    // other stuff
+    public static final String LABEL = "Label";
+    public static final String VALUE = "Value";
+    public static final String LABEL_ORDER = "LabelOrder";
+    public static final String SHORT_LABEL = "ShortLabel";
+    public static final String USER_LABEL = "UserLabel";
+    public static final String NAME = "Name";
+    public static final String DESCRIPTION = "Description";
+    public static final String FILE_LOCATION = "FileLocation";
+    public static final String CREATOR = "Creator";
+    public static final String DOCUMENT_ORDER = "DocOrder";
+    public static final String NUM_LEAF_DOCUMENTS = "NumLeafDocs";
+
+    public static final String COMMA = ",";
+    public static final String EQUALS = "=";
+    public static final String QUOTE = "'";
+    public static final String EQUALS_QUOTE = "='";
+    public static final String DOT = ".";
+    public static final String END = ";";
+    public static final String NOT="!";
+
+    // some values in the fields
+    public static final String STRUCTURE = "Structure";
+    public static final String DIVISION = "Division";
+    public static final String ALL = "All";
+    public static final String MG_NUM = "mgseqno";
+    
+}
Index: /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/SQLQuery.java
===================================================================
--- /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/SQLQuery.java	(revision 12189)
+++ /branches/kjdon/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/util/SQLQuery.java	(revision 12189)
@@ -0,0 +1,416 @@
+package org.greenstone.gsdl3.util;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnectionFactory;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import java.util.ArrayList;
+
+public class SQLQuery {
+    GS3SQLConnection connection = null;
+    
+    public SQLQuery() {
+
+    }
+
+    public boolean setDatabase(String db_name) {
+	connection = GS3SQLConnectionFactory.getGS3SQLConnection(db_name);
+	try {
+
+	    // test the connection
+	    String query = "select * from "+GSSQL.DOCUMENT_TABLE+GSSQL.END;
+	    Statement s = connection.createStatement();
+	    s.execute(query);
+	    s.close();
+	    return true;
+	} catch (Exception e) {
+	    System.err.println("SQLQuery.setDatabase():"+e);
+	}
+	closeConnection();
+	return false;
+    }
+
+    public void closeConnection() {
+	if (connection != null) {
+	    connection.close();
+	    connection=null;
+	}
+    }
+
+    public String MGNum2OID(String mg_num) {
+	// get doc id and meta id
+	String query = "select "+ 
+	    GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+
+	    GSSQL.COMMA+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+
+	    " from "+GSSQL.METADATA_VALUE_TABLE+GSSQL.COMMA+GSSQL.NAMESPACE_TABLE+GSSQL.COMMA+GSSQL.METADATA_TABLE+
+	    " where "+
+	    GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.LABEL+GSSQL.EQUALS_QUOTE+GSSQL.MG_NUM+GSSQL.QUOTE+
+	    " and "+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.VALUE+GSSQL.EQUALS_QUOTE+mg_num+GSSQL.QUOTE+
+	    " and "+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+GSSQL.EQUALS+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+
+	    " and "+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+GSSQL.EQUALS+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+
+	    GSSQL.END;
+	
+	Statement statement = null;
+	ResultSet results = null;
+	String doc_id = null;
+	String meta_id = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		doc_id = results.getString(GSSQL.DOCUMENT_ID);
+		meta_id = results.getString(GSSQL.METADATA_ID);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+	    System.err.println("SQLQuery.MGNum2OID(): "+e);
+	    return null;
+	}
+
+	// now get division label
+	query = "select "+
+	    GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.SHORT_LABEL+
+	    " from "+GSSQL.DIVISION_TABLE+GSSQL.COMMA+GSSQL.DIVISION_METADATA_TABLE+
+	    " where "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+GSSQL.EQUALS_QUOTE+meta_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+GSSQL.EQUALS+GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+
+	    GSSQL.END;
+
+	
+	String short_label = null;
+	try {
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		short_label = results.getString(GSSQL.SHORT_LABEL);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	    statement.close();
+	} catch (SQLException e) {
+
+	    System.err.println("SQLQuery.MGNum2OID() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return GS3OID.createOID(doc_id, short_label);
+    }
+
+    public String OID2MGNum(String oid) {
+	String id = getDocumentMetadata(oid, "gsdl3.mgseqno");
+	return id;
+
+    }
+
+    // does the document have sections, or is it just a single page document?
+    public boolean isHierarchicalDocument(String oid) {
+	// first just check the OID - it may already have section info in it
+	if (!GS3OID.isDocTop(oid)) {
+	    return true;
+	}
+
+	// now we need to check the database
+	String query = "select * "+ 
+	    " from "+GSSQL.STRUCTURE_TABLE +
+	    " where "+ GSSQL.DOCUMENT_ID +GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and "+ GSSQL.STRUCTURE_ID +GSSQL.EQUALS_QUOTE + "Section" + GSSQL.QUOTE +
+	    GSSQL.END;
+	boolean is_hierarchical = false;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    is_hierarchical = results.first();
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    return false;
+	}
+	
+	return is_hierarchical;
+    }
+    
+    // classifier metadata is not actually metadata
+    public String getClassifierMetadata(String oid, String full_meta_name) {
+
+	if (!full_meta_name.equals("Title") && !full_meta_name.equals("numleafdocs")) {
+	    return null;
+	}
+	// get the description
+	String field_name="";
+	if (full_meta_name.equals("Title")) {
+	    field_name = GSSQL.DESCRIPTION;
+	} else {
+	    field_name = GSSQL.NUM_LEAF_DOCUMENTS;
+	}
+	
+	String query = "select "+ field_name+
+	    " from " + GSSQL.CLASSIFIER_TABLE +
+	    " where " + GSSQL.CLASSIFIER_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    GSSQL.END;
+	
+	String value = null;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    if (results.first()) {
+		value = results.getString(field_name);
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getClassifierMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	return value;
+	
+    }
+    
+    public ArrayList getDocumentChildren(String oid) {
+
+	ArrayList children = new ArrayList();
+	String doc_id = oid;
+	String section_id = "All"; // if we are just passed a doc id
+	if (!GS3OID.isDocTop(oid)) {
+	    doc_id = GS3OID.getDocID(oid);
+	    section_id = GS3OID.getSectionLabel(oid);
+	}
+//  	int sep_index = oid.indexOf(".");
+//  	if (sep_index != -1) {
+//  	    doc_id = oid.substring(0, sep_index);
+//  	    section_id = oid.substring(sep_index+1);
+//  	}
+
+	// get the division ref for this section
+	String query = "select "+GSSQL.DIVISION_REF +
+	    " from "+GSSQL.DIVISION_TABLE+
+	    " where "+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.SHORT_LABEL+GSSQL.EQUALS_QUOTE+section_id+GSSQL.QUOTE+
+	    GSSQL.END;
+
+	ResultSet results = null;
+	Statement statement = null;
+	String div_ref = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		div_ref = results.getString(GSSQL.DIVISION_REF);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getDocumentChildren() Error1: "+e.getMessage());
+	    return null;
+	}
+
+	// now get the list of children
+	query = "select "+GSSQL.SHORT_LABEL +
+	    " from "+GSSQL.DIVISION_TABLE+
+	    " where "+ GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.PARENT_TYPE+GSSQL.EQUALS_QUOTE+GSSQL.DIVISION+GSSQL.QUOTE+
+	    " and "+GSSQL.PARENT_REF+GSSQL.EQUALS_QUOTE+div_ref+GSSQL.QUOTE+
+	    GSSQL.END;
+	try {
+	    results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.SHORT_LABEL);
+		children.add(GS3OID.createOID(doc_id, id));
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getDocumentChildren() Error2: "+e.getMessage());
+	    return null;
+	}
+
+	return children;
+	
+    }
+    
+    public boolean documentHasChildren(String oid) {
+	// for now, use getChildren, but can we do this a better way??
+	ArrayList children = getDocumentChildren(oid);
+	if (children==null || children.size()==0) {
+	    return false;
+	}
+	return true;
+	
+    }
+    public ArrayList getClassifierChildren(String oid) {
+
+	ArrayList children = new ArrayList();
+
+	String query = "select "+ GSSQL.CLASSIFIER_ID +
+	    " from " + GSSQL.CLASSIFIER_TABLE +
+	    " where " + GSSQL.PARENT_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and " + GSSQL.NUM_LEAF_DOCUMENTS + GSSQL.NOT+GSSQL.EQUALS_QUOTE+ "0"+GSSQL.QUOTE +
+	    GSSQL.END; 
+	
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.CLASSIFIER_ID);
+		children.add(id);
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getClassifierChildren Error: "+e.getMessage());
+	    return null;
+	}
+	return children;
+    }
+
+    public ArrayList getClassifierDocChildren(String oid) {
+	ArrayList children = new ArrayList();
+
+	String query = "select "+ GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+ GSSQL.DOCUMENT_ID+
+	    " from " + GSSQL.CLASSIFIER_TABLE + GSSQL.COMMA+GSSQL.CLASS_DOCUMENT_TABLE+
+	    " where " + GSSQL.CLASSIFIER_TABLE +GSSQL.DOT+ GSSQL.CLASSIFIER_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and " + GSSQL.CLASSIFIER_TABLE +GSSQL.DOT+ GSSQL.CLASSIFIER_REF+GSSQL.EQUALS+GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+ GSSQL.CLASSIFIER_REF+
+	    " order by "+GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ORDER+
+	    GSSQL.END;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.DOCUMENT_ID);
+		children.add(id);
+	    }
+	} catch (java.sql.SQLException e) {
+		
+	    System.err.println("SQLQuery.getClassifierDocChildren() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return children;
+    }
+    
+    public String getDocumentMetadata(String oid, String full_meta_name) {
+
+	String doc_id = oid;
+	String section_id = "All"; // if we are just passed a doc id
+	if (!GS3OID.isDocTop(oid)) {
+	    doc_id = GS3OID.getDocID(oid);
+	    section_id = GS3OID.getSectionLabel(oid);
+	}
+//  	int sep_index = oid.indexOf(".");
+//  	if (sep_index != -1) {
+//  	    doc_id = oid.substring(0, sep_index);
+//  	    section_id = oid.substring(sep_index+1);
+//  	}
+
+
+	// get the metadata block id
+	String query = "select "+ 
+	    GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID +
+	    " from " + GSSQL.DIVISION_METADATA_TABLE+GSSQL.COMMA+GSSQL.DIVISION_TABLE+
+	    " where " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+GSSQL.EQUALS+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+
+	    " and " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.SHORT_LABEL+GSSQL.EQUALS_QUOTE+section_id+GSSQL.QUOTE+
+	    " and " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+ 
+	    GSSQL.END;
+	
+	Statement statement = null;
+	ResultSet results = null;
+	String meta_id = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		meta_id = results.getString(GSSQL.METADATA_ID);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+
+	    System.err.println("SQLQuery.getDocumentMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	// now get the list of namespace refs for the right namespace
+	int sep_index = full_meta_name.indexOf('.');
+	String meta_ns = "gsdl3";
+	String meta_name = full_meta_name;
+	if (sep_index != -1) {
+
+	    meta_ns = full_meta_name.substring(0, sep_index);
+	    meta_name = full_meta_name.substring(sep_index+1);
+	}
+
+	query = "select * "+ 
+	    " from "+GSSQL.NAMESPACE_TABLE+GSSQL.COMMA+GSSQL.METADATA_TABLE+GSSQL.COMMA+GSSQL.METADATA_VALUE_TABLE+
+	    " where "+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+GSSQL.EQUALS_QUOTE+meta_id+GSSQL.QUOTE+
+	    " and "+ GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and " + GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+GSSQL.EQUALS+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+
+	    " and "+ GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_TYPE+GSSQL.EQUALS_QUOTE+meta_ns+GSSQL.QUOTE+
+	    " and "+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+GSSQL.EQUALS+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+
+	    GSSQL.END;
+	
+	String meta_value = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    while (results.next()) {
+		String m_name = results.getString(GSSQL.LABEL);
+		if (meta_name.equals(m_name)) {
+		    meta_value = results.getString(GSSQL.VALUE);
+		    break;
+		}
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+
+	    System.err.println("SQLQuery.getDocumentMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return meta_value;
+    }
+    public static void main (String [] args) {
+
+	SQLQuery self = new SQLQuery();
+	self.setDatabase("gs3test");
+	System.out.println("getting oid for mg num 3");
+	String oid = self.MGNum2OID("stx.3");
+	System.out.println("oid was "+oid);
+	
+	System.out.println("title metadata for mg num 3:");
+	String meta = self.getDocumentMetadata(oid, "gsdl3.title");
+	if (meta != null) {System.out.println(meta);}
+	System.out.println("subject metadata for mg num 3:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.Subject");
+	if (meta != null) {System.out.println(meta);}
+
+	System.out.println("converting back to mg, oid "+oid);
+	String id = self.OID2MGNum(oid);
+	System.out.println(""+id);
+
+	System.out.println("getting oid for mg num 8");
+	oid = self.MGNum2OID("stx.8");
+	System.out.println("oid was "+oid);
+
+	System.out.println("title metadata for mg num 8:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.title");
+	if (meta != null) {System.out.println(meta);}
+	System.out.println("subject metadata for mg num 8:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.Subject");
+	if (meta != null) {System.out.println(meta);}
+
+	System.out.println("converting back to mg, oid "+oid);
+	id = self.OID2MGNum(oid);
+	System.out.println(""+id);
+    }
+
+
+
+
+}
