Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 12191)
@@ -62,8 +62,8 @@
   CollectionMetadata metadata;       // collection-level metadata
   GS3SQLConnection   database;       // the database to store everything in
-  String             collectionHome;
-  String             siteHome;
-  String             collectionName;
-  String             qualifiedCollectionName; // used as the database name 
+  public String             collectionHome;
+  public String             siteHome;
+  public String             collectionName;
+  public String             qualifiedCollectionName; // used as the database name 
   String             notifyHost;
 
@@ -341,4 +341,5 @@
 		RecogniserInterface ri = this.buildManager.getRecogniserManager().addRecogniser(type);
 		if (ri != null) {
+		    ri.setCollectionManager(this);
 		    ri.configure(doc_type);
 		}
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 12191)
@@ -419,4 +419,21 @@
   }
   
+    public int getNumSections() {
+	return 3;
+	/*	String query = "SELECT count(*) FROM divisions WHERE DocID="+this.id+" AND ParentType='Division'";
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    
+	    if (results.first()) {
+		int count = results.getInt(0);
+		System.err.println("count = "+count);
+		return count;
+	    }
+	} catch (Exception e) {
+	    System.err.println("AbstractDocument.getNumSections(): "+e);
+	}
+	return -1;*/
+    }
   /**
    *  @see DocumentInterface:isMETSCompatible
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 12191)
@@ -5,4 +5,5 @@
 import org.w3c.dom.Element;
 
+import org.greenstone.gsdl3.gs3build.CollectionManager;
 import org.greenstone.gsdl3.gs3build.metadata.*;
 import org.greenstone.gsdl3.gs3build.util.HTTPTools;
@@ -22,5 +23,6 @@
     ArrayList filename_extensions = null;
     String document_type = "SET THIS IN THE CONCRETE CLASS";
-    
+    CollectionManager coll_manager = null;
+
     /** The constructor should set the variables 
      * preferredMimeType, filename_extensions and documentType
@@ -30,4 +32,8 @@
     }
     
+    /** set the collection manager */
+    public void setCollectionManager(CollectionManager coll_man) {
+	this.coll_manager = coll_man;
+    }
     /** configure by default does nothing */
     public boolean configure(Element config_elem){
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2PerlRecogniser.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2PerlRecogniser.java	(revision 12191)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/GS2PerlRecogniser.java	(revision 12191)
@@ -0,0 +1,136 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+import java.io.File;
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.Processing;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+
+public class GS2PerlRecogniser extends AbstractRecogniser
+{
+    public String plugin_name;
+    public GS2PerlRecogniser()
+    { 
+	//this.preferred_mime_type = "application/pdf";
+	//this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	//this.filename_extensions.add(".pdf");
+	//this.filename_extensions.add(".txt");
+	this.document_type = METSDocument.METS_DOCUMENT_TYPE;
+    }
+
+    public boolean configure(Element config_elem) {
+	String mime_type = config_elem.getAttribute("mimeType");
+	if (mime_type.equals("")) {
+	    System.err.println("GS2PerlRecogniser configure Error: no mimetype specified");
+	    return false;
+	}
+	this.preferred_mime_type = mime_type;
+	String plugin = config_elem.getAttribute("plugin");
+	if (plugin.equals("")) {
+	    System.err.println("GS2PerlRecogniser configure Error: no plugin specified");
+	    return false;
+	}
+	this.plugin_name = plugin;
+	
+	String extensions = config_elem.getAttribute("filenameExtensions");
+	if (extensions.equals("")) {
+	    System.err.println("GS2PerlRecogniser configure Error: no filename extensions specified");
+	    return false;
+	}
+	String [] exts = extensions.split("|");
+	for (int i=0; i<exts.length; i++) {
+	    this.filename_extensions.add(exts[i]);
+	}
+	return true;
+	
+    }
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if ( isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting "+document_type+" Document " + filename);
+		METSDocument doc = processFile(url, filename);
+		
+		if (doc != null) {
+		    this.list_repository.addDocument(doc);
+		    return true;
+		}
+		return false;
+	    }
+	} else {
+	    // Get Mime type remotely, and then proceed if required
+	    String mimeType = HTTPTools.getMIMEType(url);
+	    
+	    if (mimeType == this.preferred_mime_type) {
+		System.out.println("Posting "+document_type+" Document " + url.toString());
+		METSDocument doc = processFile(url, null);
+		if (doc != null) {
+		
+		    this.list_repository.addDocument(doc);
+		    return true;
+		}
+		return false;
+	    }
+	}
+	return false;
+    }
+
+
+    protected METSDocument processFile(URL url, String filename) {
+
+	// we can only do files at this stage
+	if (filename == null) return null;
+	File file = new File(filename);
+	String local_dir = file.getParent();
+	System.err.println("local dir = "+local_dir);
+	
+	//String command = "importone.pl -saveas METS -collectdir /research/kjdon/home/gsdl3/web/sites/localsite/collect -importdir "+local_dir+" -archivedir "+local_dir+" -- kath TEXTPlug "+ file.getName();
+	String command = "importone.pl -saveas METS -collectdir "+GSFile.collectDir(this.coll_manager.siteHome)+" -importdir "+local_dir+" -archivedir "+local_dir+" -- "+this.coll_manager.collectionName +" " +this.plugin_name+" "+ file.getName();
+	System.err.println("Command = "+command);
+	
+	int exit_value = Processing.runProcess(command);
+	if (exit_value == 0) {
+	    System.out.println("converted success");
+	} else {
+	    System.out.println("Unable to convert to mets");
+	    
+	    return null ;
+	}
+	
+	//read in the archives.inf file to see the directory for the new mets
+	File arcinfo = new File(local_dir, "archives.inf");
+	if (!arcinfo.isFile()) {
+	    System.err.println("cant find archives.inf file ("+arcinfo.getPath()+", aborting");
+	    return null;
+	}
+	try {
+	String content = new String(GSFile.readFile(arcinfo.getPath()));
+	content = content.trim();
+	System.err.println("content =  "+content);
+	String items[] = content.split("\\s+");
+	String filepath = items[1];
+	// delete the archive file
+	arcinfo.delete();
+	File mets_file = new File(local_dir, filepath);
+	System.err.println("mets file = "+mets_file.getPath());
+	return new METSDocument(mets_file.toURL());
+	} catch (Exception e) {
+	    System.err.println("processfile: "+e);
+	    return null;
+	}
+    }
+}
+
+    
+
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 12191)
@@ -26,4 +26,6 @@
     public static final String METS_DOCUMENT_TYPE = "METS";
     Document domDocument;
+    String original_location;
+    
     public METSDocument(DocumentID id) {
     	super(id);
@@ -50,6 +52,9 @@
 	    domDocument = builder.parse(file);
 
-	    int filePosition = file.getPath().indexOf("import/")+7;
-	    parseFilePath = file.getPath().substring(0, filePosition);
+	    //int filePosition = file.getPath().indexOf("import/")+7;
+	    //parseFilePath = file.getPath().substring(0, filePosition);
+	    parseFilePath = file.getParent(); // all refs should be relative to the current doc, or absolute
+	    parseFilePath += File.separator;
+	    this.original_location = parseFilePath;
 	    // TODO: get all the types in the tree
       
@@ -139,8 +144,14 @@
 	URL url = (URL) this.fileSet.getFile(0).getLocation();
 
-	this.getSectionText("1");
+	//this.getSectionText("1");
 	
 	if (url.getProtocol().equals("file")) {
-	    metsDoc = new HTMLDoc(url,url.getPath());
+	    String path = url.getPath();
+	    File f = new File(path);
+	    if (!f.isAbsolute() && this.original_location != null) {
+		path = this.original_location + path;
+		System.err.println("new path ="+path);
+	    }
+	    metsDoc = new HTMLDoc(url,path);
 	} else {
 	    metsDoc = new HTMLDoc(url);
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 12191)
@@ -4,4 +4,5 @@
 import org.w3c.dom.Element;
 
+import org.greenstone.gsdl3.gs3build.CollectionManager;
 import org.greenstone.gsdl3.gs3build.metadata.*;
 
@@ -13,4 +14,5 @@
 public interface RecogniserInterface
 {
+    public void setCollectionManager(CollectionManager coll_man); 
     public boolean configure(Element config_elem);
     public void setListRepository(DocumentList docList);
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 12191)
@@ -18,4 +18,5 @@
 import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
 import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
 import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
 import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
@@ -24,4 +25,5 @@
 import org.greenstone.gsdl3.gs3build.util.DOMUtils;
 import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
 import org.greenstone.gsdl3.util.Misc;
 import org.greenstone.gsdl3.util.Processing;
@@ -293,5 +295,5 @@
 		//metsDoc.setModified(true);
 		//	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
-	    } // section level
+	    } // first pass
 	
 	    // append an 'end of section' marker
@@ -408,12 +410,5 @@
     public boolean indexDocument(DocumentID docID, DocumentInterface document)
     {
-	
-	if (!this.firstDocument) {
-	    this.indexBuffer.append(END_OF_DOCUMENT);
-	    mgPasses.processDocument(indexBuffer.toString());
-	    this.indexBuffer.delete(0, this.indexBuffer.length());
-	    
-	}
-
+	int count = ((AbstractDocument)document).getNumSections();
 	String docText = null;
 	// set the mgseqno if first pass
@@ -426,14 +421,19 @@
 	
 	//long start = System.currentTimeMillis();
-	Document domDocument = document.getDOMDocument();
-	if (domDocument != null) {
-	    System.err.println("dom doc is not null");
-	    METSStructure sections = document.getDocumentStructure().getStructure("Section");
-	    if (sections != null) {
-		System.err.println("sections are not null");
-		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
-		//	System.out.println(docText);
-	    }
-	}
+	if (this.current_index.getLevel().equals("section")) {
+
+	    Document domDocument = document.getDOMDocument();
+	    if (domDocument != null) {
+		System.err.println("dom doc is not null");
+		METSStructure sections = document.getDocumentStructure().getStructure("Section");
+		if (sections != null) {
+		    System.err.println("sections are not null");
+		    docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+		    //	System.out.println(docText);
+		}
+	    }
+	} 
+	    
+	
 	//long finish = System.currentTimeMillis();
 	//System.err.println("dom doc = "+ Long.toString(finish-start));
@@ -447,4 +447,5 @@
 		if (field.equals("text")) {
 		    doc_text_buffer.append(document.getDocumentText());
+		    doc_text_buffer.append(" ");
 		}  else {
 		    // its a metadata - do namespace properly!!
@@ -455,4 +456,5 @@
 			    String value = valueIter.next().toString();
 			    doc_text_buffer.append(value);
+			    doc_text_buffer.append(" ");
 			}
 		    }
@@ -461,4 +463,5 @@
 	    docText = doc_text_buffer.toString();
 	    sectionSeqNo ++;
+	    int num_secs = 0;
 	}
 	//finish = System.currentTimeMillis();
@@ -467,6 +470,20 @@
 	this.indexBuffer.append(docText);
 	// remember that we're not on the first document, 
-	this.firstDocument = false;
+	//this.firstDocument = false;
 	this.documentSeqNo ++;
+	//if (!this.firstDocument) {
+	this.indexBuffer.append(END_OF_DOCUMENT);
+	mgPasses.processDocument(indexBuffer.toString());
+	String filename="";
+	try {
+	    filename = "pass"+this.pass+"doc"+this.documentSeqNo+".txt";
+	    System.err.println("trying to write to "+filename);
+	    GSFile.writeFile(indexBuffer.toString().getBytes(), filename);
+	} catch (Exception e) {
+	    System.err.println("COUldn't write to file, "+filename);
+	}
+	this.indexBuffer.delete(0, this.indexBuffer.length());
+	    
+	
 
 	return true;
@@ -487,5 +504,4 @@
 	this.indexBuffer = new StringBuffer();
 	int indexNo = this.pass/2;
-	this.current_index = null;
 	
 	this.current_index = (MGIndex) this.indexes.get(indexNo);
@@ -512,4 +528,5 @@
 	    this.textStem = this.indexStem;
 	}
+	
 	mgPasses.setFileName(this.indexStem);
 	if (!Misc.isWindows()) {
@@ -573,5 +590,5 @@
 	} catch (Exception e) {}
 	
-	int exit_value = 0;
+	int exit_value = mgPasses.exitValue();
 	System.out.println("Pass " + this.pass + " completed with " + exit_value);
 	if (exit_value !=0) {
Index: trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java
===================================================================
--- trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 12188)
+++ trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 12191)
@@ -8,8 +8,11 @@
 import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
 import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.GSFile;
 import org.greenstone.gsdl3.util.GSXML;
 import org.greenstone.gsdl3.util.Processing;
 import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
 import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -22,39 +25,36 @@
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 import org.w3c.dom.Document;
-
-public class MGPPIndexer extends AbstractIndexer
+import org.w3c.dom.NamedNodeMap;
+
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGPPIndexer //extends AbstractIndexer
+    implements IndexerInterface
 { 
     int          pass;
     int          documentSeqNo;
     int          sectionSeqNo;
-    String       name;
     boolean      firstDocument;
     File indexDirectory;
-    File textDirectory;
+    //    File textDirectory;
     String       indexStem;
-    String       textStem;
+    //    String       textStem;
     StringBuffer indexBuffer;
     String       outputDirectory;
-    //String       outputStem;
-//     String       passExtra;
-//     InputStream	 indexerFeedback;
-//     InputStream  indexerErrors;
-//     OutputStream indexerTextfeed;
-//     Process      mgpp_passes;
-    //String       overallName; 
-    String       currentIndexName;
-    String       currentIndexLevel;
-       String       currentIndexField; 
+    String       overallName; 
+    
+    List indexes;
+    MGPPIndex current_index = null;
     MGPPPassesWrapper mgppPasses;
     
 
-    static final String documentSeparator = "<Document>";
-    static final String sectionSeparator = "<Section>";
-
-    static final String START_OF_DOCUMENT = "<Document>";
-    static final String END_OF_DOCUMENT = "</Document>";
-    static final String START_OF_SECTION = "<Section>";
-    static final String END_OF_SECTION = "</Section>";
+    static final String DOCUMENT = "Doc";
+    static final String SECTION = "Sec";
+    static final String START_OF_DOCUMENT = "<"+DOCUMENT+">";
+    static final String END_OF_DOCUMENT = "</"+DOCUMENT+">";
+    static final String START_OF_SECTION = "<"+SECTION+">";
+    static final String END_OF_SECTION = "</"+SECTION+">";
     
     
@@ -67,13 +67,43 @@
 	public String name = null;
 	public String doc_level = null; 
-	public ArrayList levels = null;
-	public ArrayList fields = null;
+	public List levels = null;
+	public List fields = null;
 	boolean error = false;// assume built until we get an error
+	
+	public MGPPIndex(Element index_element) {
+	    
+	    this.fields = new ArrayList();
+	    this.levels = new ArrayList();
+	    this.name = index_element.getAttribute(GSXML.NAME_ATT);
+	    if (this.name.equals("")) {
+		// TODO make this dynamic
+		this.name = "xx";
+	    }
+	    NodeList children = index_element.getChildNodes();
+	    for (int c = 0; c < children.getLength(); c ++) {
+		Node child = children.item(c);
+		
+		if (child.getNodeType() == Node.ELEMENT_NODE) {
+		    String name = child.getNodeName();
+		    
+		    if (name.equals(GSXML.LEVEL_ELEM)) {
+			String level = DOMUtils.getNodeChildText(children.item(c));
+			this.levels.add(level);
+		    }
+		    else if (name.equals(GSXML.FIELD_ELEM)) {
+			String fieldName = DOMUtils.getNodeChildText(children.item(c));
+			this.fields.add(fieldName);
+		    }
+		}
+	    }
+	}
 	
 	public MGPPIndex(String name) {
 	    this.name = name;
-	    doc_level = "Document";
-	}
-
+	    this.doc_level = DOCUMENT;
+	    this.fields = new ArrayList();
+	    this.levels = new ArrayList();
+	}
+	
 	public void setDocLevel(String doc_level) {
 	    this.doc_level = doc_level;
@@ -88,5 +118,13 @@
 	    this.fields.add(field);
 	}
-
+	public List getLevels() {
+	    return this.levels;
+	}
+	public List getFields() {
+	    return this.fields;
+	}
+	public String getName() {
+	    return this.name;
+	}
 	public boolean hasError() {
 	    return this.error;
@@ -101,5 +139,6 @@
     public MGPPIndexer(String name)
     { 
-	this.name = name;
+	this.overallName = name;
+	this.indexes = new ArrayList();
 	//this.passExtra = "";
     }
@@ -107,5 +146,38 @@
     public String getName()
     { 
-	return this.name;
+	return this.overallName;
+    }
+    
+    public String getIndexType()
+    { 
+	return MGPP_INDEX_TYPE;
+    }
+
+    // for now make all indexes use document and section levels. 
+    // then when writing the buildconfig, only display the levels 
+    // that the user has specified (likely to be both doc and sec).
+    public boolean configure(Node search_node)
+    {
+	NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+	
+	// add a text 'index' - we should be able to turn this off in the config file? 
+	MGPPIndex text_index = new MGPPIndex("text");
+	text_index.addField("text");
+	// always do eveything at doc and sec level at the moment
+	text_index.addLevel(DOCUMENT);
+	text_index.addLevel(SECTION);
+	indexes.add(text_index);
+	
+	for (int i = 0; i < index_children.getLength(); i ++) {
+	    Element index_elem = (Element)index_children.item(i);
+	    MGPPIndex index = new MGPPIndex(index_elem);
+	    if (index.getName() != null && index.getLevels() != null && index.getFields()!= null) {
+		indexes.add(index);
+	    } else {
+		System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+	    }
+	}
+	// TODO make sure all index names are unique
+	return true;
     }
 
@@ -120,30 +192,30 @@
 	    
 	    // attempt to ensure that the text subdirectory exists
-	    this.textDirectory = new File(outputDirectory, "text");
-	    if (!textDirectory.exists()) {
-		if (!textDirectory.mkdir()) {
-		    return false;
-		}
-	    }
-	    else if (!textDirectory.isDirectory()) {
-		return false;
-	    }
-	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
-	    
-	    // attempt to ensure that the index subdir exists
-	    this.indexDirectory = new File(outputDirectory, "idx");
-	    if (!indexDirectory.exists()) {
-		if (!indexDirectory.mkdir()) {
-		    return false;
-		}
-	    }
-	    else if (!indexDirectory.isDirectory()) {
-		return false;
-	    }
-	    this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+// 	    this.textDirectory = new File(outputDirectory, "text");
+// 	    if (!textDirectory.exists()) {
+// 		if (!textDirectory.mkdir()) {
+// 		    return false;
+// 		}
+// 	    }
+// 	    else if (!textDirectory.isDirectory()) {
+// 		return false;
+// 	    }
+// 	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+// 	    // attempt to ensure that the index subdir exists
+// 	    this.indexDirectory = new File(outputDirectory, "idx");
+// 	    if (!indexDirectory.exists()) {
+// 		if (!indexDirectory.mkdir()) {
+// 		    return false;
+// 		}
+// 	    }
+// 	    else if (!indexDirectory.isDirectory()) {
+// 		return false;
+// 	    }
+// 	    this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
 	    
 	    // Sign to the user which mg directory is being used...
-	    System.out.println("Output MGPP text directory is " + this.textStem);
-	    System.out.println("Output MGPP index directory is " + this.indexStem); 
+// 	    System.out.println("Output MGPP text directory is " + this.textStem);
+// 	    System.out.println("Output MGPP index directory is " + this.indexStem); 
 	}
 	this.pass = 0;
@@ -151,19 +223,15 @@
     }
 
-    public String getIndexType()
-    { 
-	return MGPP_INDEX_TYPE;
-    }
-
-    public boolean addIndex(String name, String level, String field)
-    { 
-// 	if (level == "doc_level") {
-// 	    passExtra = " -J " + level;
-// 	}
-// 	else {
-// 	    passExtra = " -K " + level;
-// 	}
-	return true;
-    }
+
+//     public boolean addIndex(String name, String level, String field)
+//     { 
+// // 	if (level == "doc_level") {
+// // 	    passExtra = " -J " + level;
+// // 	}
+// // 	else {
+// // 	    passExtra = " -K " + level;
+// // 	}
+// 	return true;
+//     }
 
     /**
@@ -176,20 +244,12 @@
 	if (this.pass == 0) {
 	    document.removeAllMetadata("gsdl3", "mgseqno");
-	}
-
-	// why do this at the start and not at the end???
-	if (!this.firstDocument) {
-	    // Send a '</Document>' at the end of the doc
-	    this.indexBuffer.append(END_OF_DOCUMENT);
-	    mgppPasses.processDocument(indexBuffer.toString());
-	    this.indexBuffer.delete(0, this.indexBuffer.length());
-	}
-
+	    document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+	}
+	
+	
 	String docText = null;
 	
-	//int startSeqNo = this.sectionSeqNo;
-	//this.sectionSeqNo ++;
-	int startSeqNo = this.documentSeqNo;
-
+	this.sectionSeqNo ++;
+	
 	Document domDocument = document.getDOMDocument();
 	if (domDocument != null) {
@@ -198,113 +258,45 @@
 	    if (sections != null) {
 		System.err.println("sections are not null");
-		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
-		//	System.out.println(docText);
+		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); 
 	    }
 	}
 	if (docText == null) {
 	    System.err.println("dom doc or sections was null - asking for doc text");
-	    //if (this.currentIndexField.equals("text")) {
-		//docText = Character.toString(END_OF_DOCUMENT) + document.getDocumentText(); 
-		docText = document.getDocumentText();
-		//}
-// 	    else {
-// 		StringBuffer textBuffer = new StringBuffer();
-// 		//textBuffer.append(END_OF_DOCUMENT);
-// 		List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
-// 		if (values != null) {
-// 		    Iterator valueIter = values.iterator();
-// 		    while (valueIter.hasNext()) {
-// 			String value = valueIter.next().toString();
-			
-// 			textBuffer.append(value);
-// 			if (valueIter.hasNext()) {
-// 			    //textBuffer.append(END_OF_SECTION);
-// 			    //	      sectionSeqNo ++;
-// 			}
-// 		    }
-// 		}
-// 		else {
-// 		    textBuffer.append("No data");
-// 		}
-// 		docText = textBuffer.toString();
-// 	    }
+	    StringBuffer doc_text_buffer = new StringBuffer();
+	    List fields = this.current_index.getFields();
+	    for (int i=0; i<fields.size(); i++) {
+		String field = (String)fields.get(i);
+		if (field.equals("text")) {
+		    doc_text_buffer.append(document.getDocumentText());
+		    doc_text_buffer.append(" ");
+		}  else {
+		    // its a metadata - do namespace properly!!
+		    List values = document.getDocumentMetadataItem("gsdl3", field);
+		    if (values != null) {
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    doc_text_buffer.append(value);
+			    doc_text_buffer.append(" ");
+			}
+		    }
+		}
+	    } // for each field
+	    docText = doc_text_buffer.toString();
 	    sectionSeqNo ++;
 	}
-
-	    //try {					
-	    //	this.indexerTextfeed.write(documentSeparator.getBytes(), 0, documentSeparator.getBytes().length);
-	    // }
-// 	    catch (IOException ex) {
-// 		System.out.println("Bad output on end of document" + ex);
-// 		ex.printStackTrace();
-// 		return false;
-// 	    }
-// 	}
-
+	
 	this.indexBuffer.append(START_OF_DOCUMENT);
-	//String docText = document.getDocumentText();
+	this.indexBuffer.append(START_OF_SECTION);
 	this.indexBuffer.append(docText);
-	//int startSeqNo = this.documentSeqNo;
-	
-// 	byte [] bytes = docText.getBytes();
-// 	int pos = 0, end = bytes.length;
-	
-// 	try {
-// 	    while (pos < end) {
-// 		this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
-// 		pos = pos + 512;
-	
-// 		try {
-// 		    while (this.indexerFeedback.available() > 0) {
-// 			byte b[] = new byte[this.indexerFeedback.available()];
-// 			System.out.println("Feedback of " + this.indexerFeedback.available());
-// 			this.indexerFeedback.read(b);
-// 			System.out.println(b);
-// 		    }
-// 		}
-// 		catch (IOException ex) {
-		    
-// 		}
-
-
-// 		try {
-// 		    while (this.indexerErrors.available() > 0) {
-// 			byte b[] = new byte[this.indexerErrors.available()];
-// 			System.out.println("Feedback of " + this.indexerErrors.available());
-// 			this.indexerErrors.read(b);
-// 			System.out.println(new String(b));
-// 		    }
-// 		}
-// 		catch (IOException ex){
-		    
-// 		}
-// 	    }
-// 	}
-// 	catch (IOException ex) {
-// 	    System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
-// 	    ex.printStackTrace();
-// 	    return false;
-// 	}
+	this.indexBuffer.append(END_OF_SECTION);
+	this.indexBuffer.append(END_OF_DOCUMENT);
+	this.mgppPasses.processDocument(indexBuffer.toString());
+	this.indexBuffer.delete(0, this.indexBuffer.length());
+
 	this.firstDocument = false;
 
-	if (this.pass == 0) {    
-	    document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(startSeqNo));
-	}
 	this.documentSeqNo++;
 	
-// 	try {
-// 	    while (this.indexerErrors.available() > 0) {
-// 		char c = (char) this.indexerErrors.read();
-// 		System.out.println(c);
-// 	    }
-// 	    while (this.indexerFeedback.available() > 0) {
-// 		byte b[] = new byte[this.indexerFeedback.available()];
-// 		System.out.println("Feedback of " + this.indexerFeedback.available());
-// 		this.indexerFeedback.read(b);
-// 	    }
-// 	}
-// 	catch (IOException ex) {
-	    
-// 	}
 	return true;
     }
@@ -323,25 +315,50 @@
 	this.indexBuffer = new StringBuffer();
 	
-	MGPPIndex index = null; // do something with this!!
-	
+	int indexNo = this.pass/2;
+	this.current_index = (MGPPIndex) this.indexes.get(indexNo);
+
+	if (this.current_index.hasError()) {
+	    // an error has already occurred for this index, don't continue
+	    System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+	    return false;
+	}
+	
+	// attempt to ensure that the text/index subdirectory exists
+	this.indexDirectory = new File(outputDirectory, current_index.getName());
+	if (!indexDirectory.exists()) {
+	    if (!indexDirectory.mkdir()) {
+		return false;
+	    }
+	}
+	else if (!indexDirectory.isDirectory()) {
+	    return false;
+	}
+
+	this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+// 	if (this.pass == 0) {
+// 	    // first pass, also set up the textStem
+// 	    this.textDirectory = this.indexDirectory;
+// 	    this.textStem = this.indexStem;
+// 	}
+
 	// get the parameters for this execution of mg_passes
-	mgppPasses.setFileName((this.pass < 2 ? this.textStem : this.indexStem ));
+	mgppPasses.setFileName(this.indexStem);
 	if (!Misc.isWindows()) {
 	    mgppPasses.setBasePath("/");
 	}
 	
-	mgppPasses.setDocumentTag("Document");
-	//mgppPasses.addLevelTag("Section");
-
-	this.currentIndexLevel = "Document";// index.getLevel();
-	this.currentIndexField = "text";//index.getField();
-	this.currentIndexName = "idx"; //index.getName();
-	    
-
-	switch (this.pass) {
+	// always use Doc and Sec for now
+	mgppPasses.setDocumentTag(DOCUMENT);
+	mgppPasses.addLevelTag(SECTION);
+	
+	//this.currentIndexLevel = "Document";// index.getLevel();
+	//this.currentIndexField = "text";//index.getField();
+	//this.currentIndexName = "idx"; //index.getName();
+	    
+	int mgppPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	switch (mgppPass) {
 	case 0:
 	    // -T1
 	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_1);
-	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem + " -T1");
 	    break;
 	    
@@ -349,5 +366,4 @@
 	    // -T2
 	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_2);
-	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -T2");
 	    break;
 	    
@@ -355,30 +371,12 @@
 	    // -I1
 	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_1);
-	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I1");
 	    break;
 	    
 	case 3:
-	    //Process p = Runtime.getRuntime().exec("mgpp_perf_hash_build -f " + this.outputStem);
-	    //p.waitFor();
 	    // -I2
 	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_2);
-	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I2");
 	    break;
 	}
-	    
-	//this.indexerFeedback = mgpp_passes.getInputStream();
-	//   this.indexerErrors   = mgpp_passes.getErrorStream();
-	//   this.indexerTextfeed = mgpp_passes.getOutputStream();
-	//	}
-	//catch (IOException ex) {
-	//   System.out.println(ex);
-	//   ex.printStackTrace();
-	//   return false;
-	//}/	
-	//catch (InterruptedException ex) {
-	//   System.out.println(ex);
-	//   ex.printStackTrace();
-	//   return false;
-	//}
+	
 	mgppPasses.init();
 	System.out.println("Pass " + this.pass);
@@ -391,7 +389,4 @@
     public boolean endPass(int passNumber)
     { 
-	// TODO: end pass
-	Process p;
-	MGPPIndex index = null; // do something with this!!
 	try {
 	    this.indexBuffer.append(END_OF_DOCUMENT);
@@ -403,5 +398,7 @@
 	    System.out.println(ex);
 	}
+	
 	mgppPasses.finish();
+	
 	try {
 	    Thread.sleep(1000);
@@ -411,10 +408,9 @@
 	System.out.println("Pass " + this.pass + " completed with " + exit_value);
 	if (exit_value !=0) {
-	    //assume something has gone wrong, don't continue
-// 	    if (index != null) {
-// 		index.setError(true);
-// 		return false;
-// 	    }
-	}
+	    this.current_index.setError(true);
+	    return false;
+	}
+	
+	int mgppPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
 	
 	String osextra = "";
@@ -423,9 +419,8 @@
 	}
 
-	switch (this.pass) {
+	switch (mgppPass) {
 	case 0:
-	    //System.exit(1);
 	    System.out.println("Compressing dictionary");
-	    exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.textStem + " -S -H -2 -k 5120"+ osextra);
+	    exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.indexStem + " -S -H -2 -k 5120"+ osextra);
 	    
 	    if (exit_value == 0) {
@@ -433,5 +428,5 @@
 	    } else {
 		System.err.println("Error from mgpp_compression_dict: " + exit_value);
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -445,5 +440,5 @@
 	    } else {
 		System.err.println("Unable to build the perfect hash");
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -457,5 +452,5 @@
 	    } else {
 		System.err.println("Unable to create weights file");
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -467,5 +462,5 @@
 	    } else {
 		System.out.println("Unable to create inverted dictionary file");
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -477,5 +472,5 @@
 	    } else {
 		System.out.println("Unable to create stemmed index 1");
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -486,5 +481,5 @@
 	    } else {
 		System.out.println("Unable to create stemmed index 2");
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -494,5 +489,5 @@
 	    } else {
 		System.out.println("Unable to create stemmed index 3");
-		//index.setError(true);
+		this.current_index.setError(true);
 		return false;
 	    }
@@ -516,67 +511,61 @@
     public int getNumberOfPasses()
     { 
+	//return this.indexes.size()*2;
 	return 4;
     }
 
     public boolean addServiceDescriptions(Element service_rack_list) {
+	
+	// we only have one real index at the moment, - the first index in the list will be the text one.
+	MGPPIndex index = (MGPPIndex)this.indexes.get(1);
+	if (index.hasError()) {
+	    // we weren't able to create any indexes - don't add a search service
+	    return false;
+	}
+	
 	Document doc = service_rack_list.getOwnerDocument();
-
-	// generate the list of indexes
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	service_rack_list.appendChild(search_service_elem);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);		
+	service_rack_list.appendChild(retrieve_service_elem);
+	
+	// generate the list of indexes - with only one index in it
 	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
-	Element e = doc.createElement(GSXML.INDEX_ELEM);
-	e.setAttribute(GSXML.NAME_ATT, "idx");
-	index_list.appendChild(e);
-	String def_index = "idx";
-	
-// 	boolean found_index = false;
-// 	String def_index = ""; // the default index will just be the first one created for now.
-// 	for (int i=0; i<this.indexes.size(); i++) {
-// 	    MGIndex index = (MGIndex)this.indexes.get(i);
-// 	    if (!index.hasError()) {
-// 		Element e = doc.createElement(GSXML.INDEX_ELEM);
-// 		e.setAttribute(GSXML.NAME_ATT, index.getName());
-// 		index_list.appendChild(e);
-// 		if (found_index == false) {
-// 		    // this is the first index
-// 		    found_index = true;
-// 		    def_index = index.getName();
-// 		}
-// 	    }
-// 	}
-	
-// 	if (!found_index) {
-// 	    // no indexes were able to be created, so we can't use them or the text
-// 	    return false;
-// 	}
-
-	Element f = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	Element index_elem = doc.createElement(GSXML.INDEX_ELEM);
+	index_elem.setAttribute(GSXML.NAME_ATT, index.getName());
+	index_list.appendChild(index_elem);
 	
 	Element default_index = doc.createElement("defaultIndex");
-	default_index.setAttribute(GSXML.NAME_ATT, def_index);
+	default_index.setAttribute(GSXML.NAME_ATT, index.getName());
 
 	Element base_index_name = doc.createElement("baseIndexPrefix");
-	base_index_name.setAttribute(GSXML.NAME_ATT, "dtx");  //overallName);
+	base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
 
 	Element index_stem = doc.createElement("indexStem");
-	index_stem.setAttribute(GSXML.NAME_ATT, "index");
-
-	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
-	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
+	
 	Element default_level = doc.createElement("defaultLevel");
-	default_level.setAttribute(GSXML.NAME_ATT, "Document");
-
-	Element level_list = doc.createElement("levelList");
-	Element level = doc.createElement("level");
-	level.setAttribute(GSXML.NAME_ATT, "Document");
+	default_level.setAttribute(GSXML.NAME_ATT, SECTION);
+
+	// always have doc and sec at the moment
+	Element level_list = doc.createElement(GSXML.LEVEL_ELEM+GSXML.LIST_MODIFIER);
+	Element level = doc.createElement(GSXML.LEVEL_ELEM);
+	level.setAttribute(GSXML.NAME_ATT, DOCUMENT);
 	level_list.appendChild(level);
 
-	Element field_list = doc.createElement("fieldList");
-	Element field = doc.createElement("field");
-	field.setAttribute(GSXML.NAME_ATT, "ZZ");
-	field_list.appendChild(field);
-
-	service_rack_list.appendChild(search_service_elem);
-	service_rack_list.appendChild(retrieve_service_elem);
-
+	level = doc.createElement(GSXML.LEVEL_ELEM);
+	level.setAttribute(GSXML.NAME_ATT, SECTION);
+	level_list.appendChild(level);
+	
+	Element field_list = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	Element field;
+	List fields = index.getFields();
+	for (int i=0; i<fields.size(); i++) {
+	    String f = (String) fields.get(i);
+	    field = doc.createElement(GSXML.FIELD_ELEM);
+	    field.setAttribute(GSXML.NAME_ATT, f);
+	    field_list.appendChild(field);
+	}
+	
 	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPSearch");  
 	search_service_elem.appendChild(index_list);
@@ -584,8 +573,8 @@
 	search_service_elem.appendChild(level_list);
 	search_service_elem.appendChild(default_level);
-search_service_elem.appendChild(field_list); // do we need this??
+	search_service_elem.appendChild(field_list); 
 	search_service_elem.appendChild(base_index_name);
 	search_service_elem.appendChild(index_stem);
-
+	
 	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPRetrieve");
 	retrieve_service_elem.appendChild(default_level.cloneNode(true));
@@ -597,13 +586,224 @@
  
  
-  private Node recurseDOM(DocumentInterface metsDoc, Node node, 
-			  AbstractStructure structure, StringBuffer textBuffer,
-			  StringBuffer extraBuffer, String namespace) 
-			  //String name, String namespace, String field)
-  {
-    // send out the ctrl-c...if this is
-    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+    private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	    // try doing this for all index types
+	    // actually we should only need to do this once ????
+	    if (this.pass == 0) {
+	    System.err.println("division structure");
+	    //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+		METSDivision division = (METSDivision) structure;
+
+		// get the division metadata block
+		METSDescriptive descriptive;
+		String metadataId = division.getDefaultMetadataReference();
+		if (metadataId == null) {
+		    descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		    division.addMetadataReference(descriptive.getID());
+		}
+		else {
+		    // Get the descriptive item...
+		    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		}
+	    
+		descriptive.addMetadata("gsdl3", "mgseqno", this.overallName + "." + Integer.toString(this.sectionSeqNo));
+	    
+		metsDoc.setChanged(true);
+		//metsDoc.setModified(true);
+		//	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	    } // first pass
+	
+	    // append an 'end of section' marker
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+	    textBuffer.append(END_OF_SECTION);
+	    textBuffer.append(START_OF_SECTION);
+	    // for document-level indexes, always append an 'end of document' tag at the
+	    // end of the document for each section.  Otherwise, each section is followed
+	    // by an end of document character.  This ensures that all indexes use the
+	    // same document numbering...
+	    
+// 	    if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
+// 		extraBuffer.append(END_OF_DOCUMENT);
+// 	    }
+// 	    else {
+// 		textBuffer.append(END_OF_DOCUMENT);
+// 		this.documentSeqNo ++;
+// 	    }
+	    
+	    // produce the body here for metadata output of divisions - in the case of
+	    // text output, that will happen below...
+	    
+	    if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+		// if there is only text, don't do this
+		METSDescriptive descriptive;
+	
+		METSDivision division = (METSDivision) structure;
+
+		String metadataId = division.getDefaultMetadataReference();
+		// are there other metadata refs to get??
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		if (descriptive != null) {
+		    for (int i=0; i<fields.size(); i++) {
+			String field = (String)fields.get(i);
+			if (field.equals("text")) {
+			    continue;
+			}
+			List values = descriptive.getMetadata(namespace, field);
+			if (values != null) {	
+			    Iterator valueIter = values.iterator();
+			    while (valueIter.hasNext()) {
+				String value = valueIter.next().toString();
+				textBuffer.append("<"+field+">");
+				textBuffer.append(value);
+				textBuffer.append("</"+field+">");
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	
+	// go through our children as required...
+	Iterator children = structure.getChildIterator();
+	Node startNode;
+	boolean index_text = fields.contains("text");
+	while (children.hasNext()) {
+	    AbstractStructure child = (AbstractStructure) children.next();
+      
+	    // get xpointer for child 
+	    // get start position node
+	    if (metsDoc.getDocumentType() == "METS"){
+		startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+	    } else {
+		startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+	    }
+	    
+	    // while this node isn't the child's start node, produce the 
+	    // HTML node text, if in text field mode...
+	    if (index_text) {
+		while (node != startNode) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+	    
+	    // recurse to child
+	    node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+	} // while next child
+
+	// close a document - the actual closing \B will be done by the main
+	// loop, so only a required \C is printed here...
+	// why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
+	if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+	    if (this.pass == 0) {
+		System.err.println("structure structure");
+	    }
+	    if (index_text) {
+		while (node != null) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+      
+	    //textBuffer.append(END_OF_SECTION);
+	    //this.sectionSeqNo ++;
+      
+	}
+	return node;
+    }
+
+    
+    private void printNode(Node node, StringBuffer buffer, 
+			   boolean preserve_format, boolean close) {
+	
+	if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	    if (!close) {
+		buffer.append(node.getNodeValue());
+		buffer.append(" ");
+	    }
+	    return;
+	}
+	
+	if (node.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+	   if (!close) {
+	       buffer.append("&");
+	       buffer.append(node.getNodeValue());
+	       buffer.append(";");
+	       buffer.append(" ");
+	   }
+	   return;
+	}
+	if (!preserve_format) return; // we don't want any xml output
+	/*
+	else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      if (!close) {
+	buffer.append("<!-- ");
+	buffer.append(node.getNodeValue());
+	buffer.append(" -->");
+      }
+      }*/
+	if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	    
+	    boolean hasChildren = (node.getChildNodes().getLength() > 0);
+	    
+	    if (close) {
+		if (hasChildren) {
+		    // put the close node
+		    buffer.append("</");
+		    buffer.append(node.getNodeName());
+		    buffer.append(">");
+		}
+		// else it was closed off previously
+		return;
+	    }
+	    
+	    
+	    buffer.append("<");
+	    buffer.append(node.getNodeName());
+	    NamedNodeMap attributes = ((Element) node).getAttributes();
+	    for (int a = 0; a < attributes.getLength(); a ++) {
+		Node attributeNode = attributes.item(a);
+		buffer.append(" ");
+		buffer.append(attributeNode.getNodeName());
+		
+		String value = attributeNode.getNodeValue();
+		if (value != null && value.length() > 0) {
+		    buffer.append("=\"");
+		    buffer.append(value);
+		    buffer.append("\"");
+		}
+	    }
+	    
+
+	    if (!hasChildren) {
+		buffer.append(" /");
+	    }
+	    buffer.append(">");
+	    
+	    //if (close && node.getNodeName()=="Section") {
+	    //buffer.append((char) 3);
+	    // }
+	}
+	
+    }
+    
+    /*
+    private Node recurseDOMold(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
 	// try doing this for all index types
-	if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+	    if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
 	    METSDivision division = (METSDivision) structure;
 
@@ -718,15 +918,16 @@
     return node;
   }
-
+    */
     private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
-    //  String name, String namespace, String field)
-  { StringBuffer extraBuffer = new StringBuffer();
-    Node node = document.getDocumentElement();
-    StringBuffer textBuffer = new StringBuffer();
-
-    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
-    textBuffer.append(extraBuffer.toString());
-    return textBuffer.toString();
-  }
-
+	//  String name, String namespace, String field)
+    { 
+	StringBuffer extraBuffer = new StringBuffer();
+	Node node = document.getDocumentElement();
+	StringBuffer textBuffer = new StringBuffer();
+	
+	this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); 
+	textBuffer.append(extraBuffer.toString());
+	return textBuffer.toString();
+    }
+    
 }
