Changeset 12191 for trunk/greenstone3-extensions/gs3build/src/org

Timestamp:

2006-07-13T10:29:55+12:00 (18 years ago)

Author:

kjdon

Message:

committed some changes that I had made ages ago. Not sure if it still compiles - I need to write an ant build file for this, and check compilation. Will do it once I need to - its unclear whether anyone will ever use this again

Location:

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build

Files:

• CollectionManager.java (modified) (2 diffs)
• doctypes/AbstractDocument.java (modified) (1 diff)
• doctypes/AbstractRecogniser.java (modified) (3 diffs)
• doctypes/GS2PerlRecogniser.java (added)
• doctypes/METSDocument.java (modified) (3 diffs)
• doctypes/RecogniserInterface.java (modified) (2 diffs)
• indexers/MGIndexer.java (modified) (12 diffs)
• indexers/MGPPIndexer.java (modified) (28 diffs)

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/CollectionManager.java

@@ -62 +62 @@
   CollectionMetadata metadata;       // collection-level metadata
   GS3SQLConnection   database;       // the database to store everything in
-  String             collectionHome;
-  String             siteHome;
-  String             collectionName;
-  String             qualifiedCollectionName; // used as the database name 
+  public String             collectionHome;
+  public String             siteHome;
+  public String             collectionName;
+  public String             qualifiedCollectionName; // used as the database name 
   String             notifyHost;
 
@@ -341 +341 @@
         RecogniserInterface ri = this.buildManager.getRecogniserManager().addRecogniser(type);
         if (ri != null) {
+            ri.setCollectionManager(this);
             ri.configure(doc_type);
         }

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java

@@ -419 +419 @@
   }
   
+    public int getNumSections() {
+    return 3;
+    /*  String query = "SELECT count(*) FROM divisions WHERE DocID="+this.id+" AND ParentType='Division'";
+    try {
+        Statement statement = connection.createStatement();
+        ResultSet results = statement.executeQuery(query);
+        
+        if (results.first()) {
+        int count = results.getInt(0);
+        System.err.println("count = "+count);
+        return count;
+        }
+    } catch (Exception e) {
+        System.err.println("AbstractDocument.getNumSections(): "+e);
+    }
+    return -1;*/
+    }
   /**
    *  @see DocumentInterface:isMETSCompatible

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java

@@ -5 +5 @@
 import org.w3c.dom.Element;
 
+import org.greenstone.gsdl3.gs3build.CollectionManager;
 import org.greenstone.gsdl3.gs3build.metadata.*;
 import org.greenstone.gsdl3.gs3build.util.HTTPTools;
@@ -22 +23 @@
     ArrayList filename_extensions = null;
     String document_type = "SET THIS IN THE CONCRETE CLASS";
-    
+    CollectionManager coll_manager = null;
+
     /** The constructor should set the variables 
      * preferredMimeType, filename_extensions and documentType
@@ -30 +32 @@
     }
     
+    /** set the collection manager */
+    public void setCollectionManager(CollectionManager coll_man) {
+    this.coll_manager = coll_man;
+    }
     /** configure by default does nothing */
     public boolean configure(Element config_elem){

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java

@@ -26 +26 @@
     public static final String METS_DOCUMENT_TYPE = "METS";
     Document domDocument;
+    String original_location;
+    
     public METSDocument(DocumentID id) {
         super(id);
@@ -50 +52 @@
         domDocument = builder.parse(file);
 
-        int filePosition = file.getPath().indexOf("import/")+7;
-        parseFilePath = file.getPath().substring(0, filePosition);
+        //int filePosition = file.getPath().indexOf("import/")+7;
+        //parseFilePath = file.getPath().substring(0, filePosition);
+        parseFilePath = file.getParent(); // all refs should be relative to the current doc, or absolute
+        parseFilePath += File.separator;
+        this.original_location = parseFilePath;
         // TODO: get all the types in the tree
       
@@ -139 +144 @@
     URL url = (URL) this.fileSet.getFile(0).getLocation();
 
-    this.getSectionText("1");
+    //this.getSectionText("1");
     
     if (url.getProtocol().equals("file")) {
-        metsDoc = new HTMLDoc(url,url.getPath());
+        String path = url.getPath();
+        File f = new File(path);
+        if (!f.isAbsolute() && this.original_location != null) {
+        path = this.original_location + path;
+        System.err.println("new path ="+path);
+        }
+        metsDoc = new HTMLDoc(url,path);
     } else {
         metsDoc = new HTMLDoc(url);

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java

@@ -4 +4 @@
 import org.w3c.dom.Element;
 
+import org.greenstone.gsdl3.gs3build.CollectionManager;
 import org.greenstone.gsdl3.gs3build.metadata.*;
 
@@ -13 +14 @@
 public interface RecogniserInterface
 {
+    public void setCollectionManager(CollectionManager coll_man); 
     public boolean configure(Element config_elem);
     public void setListRepository(DocumentList docList);

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java

@@ -18 +18 @@
 import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
 import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
 import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
 import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
@@ -24 +25 @@
 import org.greenstone.gsdl3.gs3build.util.DOMUtils;
 import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
 import org.greenstone.gsdl3.util.Misc;
 import org.greenstone.gsdl3.util.Processing;
@@ -293 +295 @@
         //metsDoc.setModified(true);
         //  System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
-        } // section level
+        } // first pass
     
         // append an 'end of section' marker
@@ -408 +410 @@
     public boolean indexDocument(DocumentID docID, DocumentInterface document)
     {
-    
-    if (!this.firstDocument) {
-        this.indexBuffer.append(END_OF_DOCUMENT);
-        mgPasses.processDocument(indexBuffer.toString());
-        this.indexBuffer.delete(0, this.indexBuffer.length());
-        
-    }
-
+    int count = ((AbstractDocument)document).getNumSections();
     String docText = null;
     // set the mgseqno if first pass
@@ -426 +421 @@
     
     //long start = System.currentTimeMillis();
-    Document domDocument = document.getDOMDocument();
-    if (domDocument != null) {
-        System.err.println("dom doc is not null");
-        METSStructure sections = document.getDocumentStructure().getStructure("Section");
-        if (sections != null) {
-        System.err.println("sections are not null");
-        docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
-        //  System.out.println(docText);
-        }
-    }
+    if (this.current_index.getLevel().equals("section")) {
+
+        Document domDocument = document.getDOMDocument();
+        if (domDocument != null) {
+        System.err.println("dom doc is not null");
+        METSStructure sections = document.getDocumentStructure().getStructure("Section");
+        if (sections != null) {
+            System.err.println("sections are not null");
+            docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+            //  System.out.println(docText);
+        }
+        }
+    } 
+        
+    
     //long finish = System.currentTimeMillis();
     //System.err.println("dom doc = "+ Long.toString(finish-start));
@@ -447 +447 @@
         if (field.equals("text")) {
             doc_text_buffer.append(document.getDocumentText());
+            doc_text_buffer.append(" ");
         }  else {
             // its a metadata - do namespace properly!!
@@ -455 +456 @@
                 String value = valueIter.next().toString();
                 doc_text_buffer.append(value);
+                doc_text_buffer.append(" ");
             }
             }
@@ -461 +463 @@
         docText = doc_text_buffer.toString();
         sectionSeqNo ++;
+        int num_secs = 0;
     }
     //finish = System.currentTimeMillis();
@@ -467 +470 @@
     this.indexBuffer.append(docText);
     // remember that we're not on the first document, 
-    this.firstDocument = false;
+    //this.firstDocument = false;
     this.documentSeqNo ++;
+    //if (!this.firstDocument) {
+    this.indexBuffer.append(END_OF_DOCUMENT);
+    mgPasses.processDocument(indexBuffer.toString());
+    String filename="";
+    try {
+        filename = "pass"+this.pass+"doc"+this.documentSeqNo+".txt";
+        System.err.println("trying to write to "+filename);
+        GSFile.writeFile(indexBuffer.toString().getBytes(), filename);
+    } catch (Exception e) {
+        System.err.println("COUldn't write to file, "+filename);
+    }
+    this.indexBuffer.delete(0, this.indexBuffer.length());
+        
+    
 
     return true;
@@ -487 +504 @@
     this.indexBuffer = new StringBuffer();
     int indexNo = this.pass/2;
-    this.current_index = null;
     
     this.current_index = (MGIndex) this.indexes.get(indexNo);
@@ -512 +528 @@
         this.textStem = this.indexStem;
     }
+    
     mgPasses.setFileName(this.indexStem);
     if (!Misc.isWindows()) {
@@ -573 +590 @@
     } catch (Exception e) {}
     
-    int exit_value = 0;
+    int exit_value = mgPasses.exitValue();
     System.out.println("Pass " + this.pass + " completed with " + exit_value);
     if (exit_value !=0) {

trunk/greenstone3-extensions/gs3build/src/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java

@@ -8 +8 @@
 import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
 import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.GSFile;
 import org.greenstone.gsdl3.util.GSXML;
 import org.greenstone.gsdl3.util.Processing;
 import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
 import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -22 +25 @@
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 import org.w3c.dom.Document;
-
-public class MGPPIndexer extends AbstractIndexer
+import org.w3c.dom.NamedNodeMap;
+
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGPPIndexer //extends AbstractIndexer
+    implements IndexerInterface
 { 
     int          pass;
     int          documentSeqNo;
     int          sectionSeqNo;
-    String       name;
     boolean      firstDocument;
     File indexDirectory;
-    File textDirectory;
+    //    File textDirectory;
     String       indexStem;
-    String       textStem;
+    //    String       textStem;
     StringBuffer indexBuffer;
     String       outputDirectory;
-    //String       outputStem;
-//     String       passExtra;
-//     InputStream   indexerFeedback;
-//     InputStream  indexerErrors;
-//     OutputStream indexerTextfeed;
-//     Process      mgpp_passes;
-    //String       overallName; 
-    String       currentIndexName;
-    String       currentIndexLevel;
-       String       currentIndexField; 
+    String       overallName; 
+    
+    List indexes;
+    MGPPIndex current_index = null;
     MGPPPassesWrapper mgppPasses;
     
 
-    static final String documentSeparator = "<Document>";
-    static final String sectionSeparator = "<Section>";
-
-    static final String START_OF_DOCUMENT = "<Document>";
-    static final String END_OF_DOCUMENT = "</Document>";
-    static final String START_OF_SECTION = "<Section>";
-    static final String END_OF_SECTION = "</Section>";
+    static final String DOCUMENT = "Doc";
+    static final String SECTION = "Sec";
+    static final String START_OF_DOCUMENT = "<"+DOCUMENT+">";
+    static final String END_OF_DOCUMENT = "</"+DOCUMENT+">";
+    static final String START_OF_SECTION = "<"+SECTION+">";
+    static final String END_OF_SECTION = "</"+SECTION+">";
     
     
@@ -67 +67 @@
     public String name = null;
     public String doc_level = null; 
-    public ArrayList levels = null;
-    public ArrayList fields = null;
+    public List levels = null;
+    public List fields = null;
     boolean error = false;// assume built until we get an error
+    
+    public MGPPIndex(Element index_element) {
+        
+        this.fields = new ArrayList();
+        this.levels = new ArrayList();
+        this.name = index_element.getAttribute(GSXML.NAME_ATT);
+        if (this.name.equals("")) {
+        // TODO make this dynamic
+        this.name = "xx";
+        }
+        NodeList children = index_element.getChildNodes();
+        for (int c = 0; c < children.getLength(); c ++) {
+        Node child = children.item(c);
+        
+        if (child.getNodeType() == Node.ELEMENT_NODE) {
+            String name = child.getNodeName();
+            
+            if (name.equals(GSXML.LEVEL_ELEM)) {
+            String level = DOMUtils.getNodeChildText(children.item(c));
+            this.levels.add(level);
+            }
+            else if (name.equals(GSXML.FIELD_ELEM)) {
+            String fieldName = DOMUtils.getNodeChildText(children.item(c));
+            this.fields.add(fieldName);
+            }
+        }
+        }
+    }
     
     public MGPPIndex(String name) {
         this.name = name;
-        doc_level = "Document";
-    }
-
+        this.doc_level = DOCUMENT;
+        this.fields = new ArrayList();
+        this.levels = new ArrayList();
+    }
+    
     public void setDocLevel(String doc_level) {
         this.doc_level = doc_level;
@@ -88 +118 @@
         this.fields.add(field);
     }
-
+    public List getLevels() {
+        return this.levels;
+    }
+    public List getFields() {
+        return this.fields;
+    }
+    public String getName() {
+        return this.name;
+    }
     public boolean hasError() {
         return this.error;
@@ -101 +139 @@
     public MGPPIndexer(String name)
     { 
-    this.name = name;
+    this.overallName = name;
+    this.indexes = new ArrayList();
     //this.passExtra = "";
     }
@@ -107 +146 @@
     public String getName()
     { 
-    return this.name;
+    return this.overallName;
+    }
+    
+    public String getIndexType()
+    { 
+    return MGPP_INDEX_TYPE;
+    }
+
+    // for now make all indexes use document and section levels. 
+    // then when writing the buildconfig, only display the levels 
+    // that the user has specified (likely to be both doc and sec).
+    public boolean configure(Node search_node)
+    {
+    NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+    
+    // add a text 'index' - we should be able to turn this off in the config file? 
+    MGPPIndex text_index = new MGPPIndex("text");
+    text_index.addField("text");
+    // always do eveything at doc and sec level at the moment
+    text_index.addLevel(DOCUMENT);
+    text_index.addLevel(SECTION);
+    indexes.add(text_index);
+    
+    for (int i = 0; i < index_children.getLength(); i ++) {
+        Element index_elem = (Element)index_children.item(i);
+        MGPPIndex index = new MGPPIndex(index_elem);
+        if (index.getName() != null && index.getLevels() != null && index.getFields()!= null) {
+        indexes.add(index);
+        } else {
+        System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+        }
+    }
+    // TODO make sure all index names are unique
+    return true;
     }
 
@@ -120 +192 @@
         
         // attempt to ensure that the text subdirectory exists
-        this.textDirectory = new File(outputDirectory, "text");
-        if (!textDirectory.exists()) {
-        if (!textDirectory.mkdir()) {
-            return false;
-        }
-        }
-        else if (!textDirectory.isDirectory()) {
-        return false;
-        }
-        this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
-        
-        // attempt to ensure that the index subdir exists
-        this.indexDirectory = new File(outputDirectory, "idx");
-        if (!indexDirectory.exists()) {
-        if (!indexDirectory.mkdir()) {
-            return false;
-        }
-        }
-        else if (!indexDirectory.isDirectory()) {
-        return false;
-        }
-        this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+//      this.textDirectory = new File(outputDirectory, "text");
+//      if (!textDirectory.exists()) {
+//      if (!textDirectory.mkdir()) {
+//          return false;
+//      }
+//      }
+//      else if (!textDirectory.isDirectory()) {
+//      return false;
+//      }
+//      this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+        
+//      // attempt to ensure that the index subdir exists
+//      this.indexDirectory = new File(outputDirectory, "idx");
+//      if (!indexDirectory.exists()) {
+//      if (!indexDirectory.mkdir()) {
+//          return false;
+//      }
+//      }
+//      else if (!indexDirectory.isDirectory()) {
+//      return false;
+//      }
+//      this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
         
         // Sign to the user which mg directory is being used...
-        System.out.println("Output MGPP text directory is " + this.textStem);
-        System.out.println("Output MGPP index directory is " + this.indexStem); 
+//      System.out.println("Output MGPP text directory is " + this.textStem);
+//      System.out.println("Output MGPP index directory is " + this.indexStem); 
     }
     this.pass = 0;
@@ -151 +223 @@
     }
 
-    public String getIndexType()
-    { 
-    return MGPP_INDEX_TYPE;
-    }
-
-    public boolean addIndex(String name, String level, String field)
-    { 
-//  if (level == "doc_level") {
-//      passExtra = " -J " + level;
-//  }
-//  else {
-//      passExtra = " -K " + level;
-//  }
-    return true;
-    }
+
+//     public boolean addIndex(String name, String level, String field)
+//     { 
+// //   if (level == "doc_level") {
+// //       passExtra = " -J " + level;
+// //   }
+// //   else {
+// //       passExtra = " -K " + level;
+// //   }
+//  return true;
+//     }
 
     /**
@@ -176 +244 @@
     if (this.pass == 0) {
         document.removeAllMetadata("gsdl3", "mgseqno");
-    }
-
-    // why do this at the start and not at the end???
-    if (!this.firstDocument) {
-        // Send a '</Document>' at the end of the doc
-        this.indexBuffer.append(END_OF_DOCUMENT);
-        mgppPasses.processDocument(indexBuffer.toString());
-        this.indexBuffer.delete(0, this.indexBuffer.length());
-    }
-
+        document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+    }
+    
+    
     String docText = null;
     
-    //int startSeqNo = this.sectionSeqNo;
-    //this.sectionSeqNo ++;
-    int startSeqNo = this.documentSeqNo;
-
+    this.sectionSeqNo ++;
+    
     Document domDocument = document.getDOMDocument();
     if (domDocument != null) {
@@ -198 +258 @@
         if (sections != null) {
         System.err.println("sections are not null");
-        docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
-        //  System.out.println(docText);
+        docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); 
         }
     }
     if (docText == null) {
         System.err.println("dom doc or sections was null - asking for doc text");
-        //if (this.currentIndexField.equals("text")) {
-        //docText = Character.toString(END_OF_DOCUMENT) + document.getDocumentText(); 
-        docText = document.getDocumentText();
-        //}
-//      else {
-//      StringBuffer textBuffer = new StringBuffer();
-//      //textBuffer.append(END_OF_DOCUMENT);
-//      List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
-//      if (values != null) {
-//          Iterator valueIter = values.iterator();
-//          while (valueIter.hasNext()) {
-//          String value = valueIter.next().toString();
-            
-//          textBuffer.append(value);
-//          if (valueIter.hasNext()) {
-//              //textBuffer.append(END_OF_SECTION);
-//              //        sectionSeqNo ++;
-//          }
-//          }
-//      }
-//      else {
-//          textBuffer.append("No data");
-//      }
-//      docText = textBuffer.toString();
-//      }
+        StringBuffer doc_text_buffer = new StringBuffer();
+        List fields = this.current_index.getFields();
+        for (int i=0; i<fields.size(); i++) {
+        String field = (String)fields.get(i);
+        if (field.equals("text")) {
+            doc_text_buffer.append(document.getDocumentText());
+            doc_text_buffer.append(" ");
+        }  else {
+            // its a metadata - do namespace properly!!
+            List values = document.getDocumentMetadataItem("gsdl3", field);
+            if (values != null) {
+            Iterator valueIter = values.iterator();
+            while (valueIter.hasNext()) {
+                String value = valueIter.next().toString();
+                doc_text_buffer.append(value);
+                doc_text_buffer.append(" ");
+            }
+            }
+        }
+        } // for each field
+        docText = doc_text_buffer.toString();
         sectionSeqNo ++;
     }
-
-        //try {                 
-        //  this.indexerTextfeed.write(documentSeparator.getBytes(), 0, documentSeparator.getBytes().length);
-        // }
-//      catch (IOException ex) {
-//      System.out.println("Bad output on end of document" + ex);
-//      ex.printStackTrace();
-//      return false;
-//      }
-//  }
-
+    
     this.indexBuffer.append(START_OF_DOCUMENT);
-    //String docText = document.getDocumentText();
+    this.indexBuffer.append(START_OF_SECTION);
     this.indexBuffer.append(docText);
-    //int startSeqNo = this.documentSeqNo;
-    
-//  byte [] bytes = docText.getBytes();
-//  int pos = 0, end = bytes.length;
-    
-//  try {
-//      while (pos < end) {
-//      this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
-//      pos = pos + 512;
-    
-//      try {
-//          while (this.indexerFeedback.available() > 0) {
-//          byte b[] = new byte[this.indexerFeedback.available()];
-//          System.out.println("Feedback of " + this.indexerFeedback.available());
-//          this.indexerFeedback.read(b);
-//          System.out.println(b);
-//          }
-//      }
-//      catch (IOException ex) {
-            
-//      }
-
-
-//      try {
-//          while (this.indexerErrors.available() > 0) {
-//          byte b[] = new byte[this.indexerErrors.available()];
-//          System.out.println("Feedback of " + this.indexerErrors.available());
-//          this.indexerErrors.read(b);
-//          System.out.println(new String(b));
-//          }
-//      }
-//      catch (IOException ex){
-            
-//      }
-//      }
-//  }
-//  catch (IOException ex) {
-//      System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
-//      ex.printStackTrace();
-//      return false;
-//  }
+    this.indexBuffer.append(END_OF_SECTION);
+    this.indexBuffer.append(END_OF_DOCUMENT);
+    this.mgppPasses.processDocument(indexBuffer.toString());
+    this.indexBuffer.delete(0, this.indexBuffer.length());
+
     this.firstDocument = false;
 
-    if (this.pass == 0) {    
-        document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(startSeqNo));
-    }
     this.documentSeqNo++;
     
-//  try {
-//      while (this.indexerErrors.available() > 0) {
-//      char c = (char) this.indexerErrors.read();
-//      System.out.println(c);
-//      }
-//      while (this.indexerFeedback.available() > 0) {
-//      byte b[] = new byte[this.indexerFeedback.available()];
-//      System.out.println("Feedback of " + this.indexerFeedback.available());
-//      this.indexerFeedback.read(b);
-//      }
-//  }
-//  catch (IOException ex) {
-        
-//  }
     return true;
     }
@@ -323 +315 @@
     this.indexBuffer = new StringBuffer();
     
-    MGPPIndex index = null; // do something with this!!
-    
+    int indexNo = this.pass/2;
+    this.current_index = (MGPPIndex) this.indexes.get(indexNo);
+
+    if (this.current_index.hasError()) {
+        // an error has already occurred for this index, don't continue
+        System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+        return false;
+    }
+    
+    // attempt to ensure that the text/index subdirectory exists
+    this.indexDirectory = new File(outputDirectory, current_index.getName());
+    if (!indexDirectory.exists()) {
+        if (!indexDirectory.mkdir()) {
+        return false;
+        }
+    }
+    else if (!indexDirectory.isDirectory()) {
+        return false;
+    }
+
+    this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+//  if (this.pass == 0) {
+//      // first pass, also set up the textStem
+//      this.textDirectory = this.indexDirectory;
+//      this.textStem = this.indexStem;
+//  }
+
     // get the parameters for this execution of mg_passes
-    mgppPasses.setFileName((this.pass < 2 ? this.textStem : this.indexStem ));
+    mgppPasses.setFileName(this.indexStem);
     if (!Misc.isWindows()) {
         mgppPasses.setBasePath("/");
     }
     
-    mgppPasses.setDocumentTag("Document");
-    //mgppPasses.addLevelTag("Section");
-
-    this.currentIndexLevel = "Document";// index.getLevel();
-    this.currentIndexField = "text";//index.getField();
-    this.currentIndexName = "idx"; //index.getName();
-        
-
-    switch (this.pass) {
+    // always use Doc and Sec for now
+    mgppPasses.setDocumentTag(DOCUMENT);
+    mgppPasses.addLevelTag(SECTION);
+    
+    //this.currentIndexLevel = "Document";// index.getLevel();
+    //this.currentIndexField = "text";//index.getField();
+    //this.currentIndexName = "idx"; //index.getName();
+        
+    int mgppPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+    switch (mgppPass) {
     case 0:
         // -T1
         mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_1);
-        //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem + " -T1");
         break;
         
@@ -349 +366 @@
         // -T2
         mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_2);
-        //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -T2");
         break;
         
@@ -355 +371 @@
         // -I1
         mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_1);
-        //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I1");
         break;
         
     case 3:
-        //Process p = Runtime.getRuntime().exec("mgpp_perf_hash_build -f " + this.outputStem);
-        //p.waitFor();
         // -I2
         mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_2);
-        //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I2");
         break;
     }
-        
-    //this.indexerFeedback = mgpp_passes.getInputStream();
-    //   this.indexerErrors   = mgpp_passes.getErrorStream();
-    //   this.indexerTextfeed = mgpp_passes.getOutputStream();
-    //  }
-    //catch (IOException ex) {
-    //   System.out.println(ex);
-    //   ex.printStackTrace();
-    //   return false;
-    //}/    
-    //catch (InterruptedException ex) {
-    //   System.out.println(ex);
-    //   ex.printStackTrace();
-    //   return false;
-    //}
+    
     mgppPasses.init();
     System.out.println("Pass " + this.pass);
@@ -391 +389 @@
     public boolean endPass(int passNumber)
     { 
-    // TODO: end pass
-    Process p;
-    MGPPIndex index = null; // do something with this!!
     try {
         this.indexBuffer.append(END_OF_DOCUMENT);
@@ -403 +398 @@
         System.out.println(ex);
     }
+    
     mgppPasses.finish();
+    
     try {
         Thread.sleep(1000);
@@ -411 +408 @@
     System.out.println("Pass " + this.pass + " completed with " + exit_value);
     if (exit_value !=0) {
-        //assume something has gone wrong, don't continue
-//      if (index != null) {
-//      index.setError(true);
-//      return false;
-//      }
-    }
+        this.current_index.setError(true);
+        return false;
+    }
+    
+    int mgppPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
     
     String osextra = "";
@@ -423 +419 @@
     }
 
-    switch (this.pass) {
+    switch (mgppPass) {
     case 0:
-        //System.exit(1);
         System.out.println("Compressing dictionary");
-        exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.textStem + " -S -H -2 -k 5120"+ osextra);
+        exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.indexStem + " -S -H -2 -k 5120"+ osextra);
         
         if (exit_value == 0) {
@@ -433 +428 @@
         } else {
         System.err.println("Error from mgpp_compression_dict: " + exit_value);
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -445 +440 @@
         } else {
         System.err.println("Unable to build the perfect hash");
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -457 +452 @@
         } else {
         System.err.println("Unable to create weights file");
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -467 +462 @@
         } else {
         System.out.println("Unable to create inverted dictionary file");
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -477 +472 @@
         } else {
         System.out.println("Unable to create stemmed index 1");
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -486 +481 @@
         } else {
         System.out.println("Unable to create stemmed index 2");
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -494 +489 @@
         } else {
         System.out.println("Unable to create stemmed index 3");
-        //index.setError(true);
+        this.current_index.setError(true);
         return false;
         }
@@ -516 +511 @@
     public int getNumberOfPasses()
     { 
+    //return this.indexes.size()*2;
     return 4;
     }
 
     public boolean addServiceDescriptions(Element service_rack_list) {
+    
+    // we only have one real index at the moment, - the first index in the list will be the text one.
+    MGPPIndex index = (MGPPIndex)this.indexes.get(1);
+    if (index.hasError()) {
+        // we weren't able to create any indexes - don't add a search service
+        return false;
+    }
+    
     Document doc = service_rack_list.getOwnerDocument();
-
-    // generate the list of indexes
+    Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    service_rack_list.appendChild(search_service_elem);
+    Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);        
+    service_rack_list.appendChild(retrieve_service_elem);
+    
+    // generate the list of indexes - with only one index in it
     Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
-    Element e = doc.createElement(GSXML.INDEX_ELEM);
-    e.setAttribute(GSXML.NAME_ATT, "idx");
-    index_list.appendChild(e);
-    String def_index = "idx";
-    
-//  boolean found_index = false;
-//  String def_index = ""; // the default index will just be the first one created for now.
-//  for (int i=0; i<this.indexes.size(); i++) {
-//      MGIndex index = (MGIndex)this.indexes.get(i);
-//      if (!index.hasError()) {
-//      Element e = doc.createElement(GSXML.INDEX_ELEM);
-//      e.setAttribute(GSXML.NAME_ATT, index.getName());
-//      index_list.appendChild(e);
-//      if (found_index == false) {
-//          // this is the first index
-//          found_index = true;
-//          def_index = index.getName();
-//      }
-//      }
-//  }
-    
-//  if (!found_index) {
-//      // no indexes were able to be created, so we can't use them or the text
-//      return false;
-//  }
-
-    Element f = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+    Element index_elem = doc.createElement(GSXML.INDEX_ELEM);
+    index_elem.setAttribute(GSXML.NAME_ATT, index.getName());
+    index_list.appendChild(index_elem);
     
     Element default_index = doc.createElement("defaultIndex");
-    default_index.setAttribute(GSXML.NAME_ATT, def_index);
+    default_index.setAttribute(GSXML.NAME_ATT, index.getName());
 
     Element base_index_name = doc.createElement("baseIndexPrefix");
-    base_index_name.setAttribute(GSXML.NAME_ATT, "dtx");  //overallName);
+    base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
 
     Element index_stem = doc.createElement("indexStem");
-    index_stem.setAttribute(GSXML.NAME_ATT, "index");
-
-    Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
-    Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
+    
     Element default_level = doc.createElement("defaultLevel");
-    default_level.setAttribute(GSXML.NAME_ATT, "Document");
-
-    Element level_list = doc.createElement("levelList");
-    Element level = doc.createElement("level");
-    level.setAttribute(GSXML.NAME_ATT, "Document");
+    default_level.setAttribute(GSXML.NAME_ATT, SECTION);
+
+    // always have doc and sec at the moment
+    Element level_list = doc.createElement(GSXML.LEVEL_ELEM+GSXML.LIST_MODIFIER);
+    Element level = doc.createElement(GSXML.LEVEL_ELEM);
+    level.setAttribute(GSXML.NAME_ATT, DOCUMENT);
     level_list.appendChild(level);
 
-    Element field_list = doc.createElement("fieldList");
-    Element field = doc.createElement("field");
-    field.setAttribute(GSXML.NAME_ATT, "ZZ");
-    field_list.appendChild(field);
-
-    service_rack_list.appendChild(search_service_elem);
-    service_rack_list.appendChild(retrieve_service_elem);
-
+    level = doc.createElement(GSXML.LEVEL_ELEM);
+    level.setAttribute(GSXML.NAME_ATT, SECTION);
+    level_list.appendChild(level);
+    
+    Element field_list = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+    Element field;
+    List fields = index.getFields();
+    for (int i=0; i<fields.size(); i++) {
+        String f = (String) fields.get(i);
+        field = doc.createElement(GSXML.FIELD_ELEM);
+        field.setAttribute(GSXML.NAME_ATT, f);
+        field_list.appendChild(field);
+    }
+    
     search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPSearch");  
     search_service_elem.appendChild(index_list);
@@ -584 +573 @@
     search_service_elem.appendChild(level_list);
     search_service_elem.appendChild(default_level);
-search_service_elem.appendChild(field_list); // do we need this??
+    search_service_elem.appendChild(field_list); 
     search_service_elem.appendChild(base_index_name);
     search_service_elem.appendChild(index_stem);
-
+    
     retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPRetrieve");
     retrieve_service_elem.appendChild(default_level.cloneNode(true));
@@ -597 +586 @@
  
  
-  private Node recurseDOM(DocumentInterface metsDoc, Node node, 
-              AbstractStructure structure, StringBuffer textBuffer,
-              StringBuffer extraBuffer, String namespace) 
-              //String name, String namespace, String field)
-  {
-    // send out the ctrl-c...if this is
-    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+    private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+                AbstractStructure structure, StringBuffer textBuffer,
+                StringBuffer extraBuffer, String namespace) 
+    //String name, String namespace, String field)
+    {
+    List fields = current_index.getFields();
+    // send out the ctrl-c...if this is
+    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+        // try doing this for all index types
+        // actually we should only need to do this once ????
+        if (this.pass == 0) {
+        System.err.println("division structure");
+        //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+        METSDivision division = (METSDivision) structure;
+
+        // get the division metadata block
+        METSDescriptive descriptive;
+        String metadataId = division.getDefaultMetadataReference();
+        if (metadataId == null) {
+            descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+            division.addMetadataReference(descriptive.getID());
+        }
+        else {
+            // Get the descriptive item...
+            descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+        }
+        
+        descriptive.addMetadata("gsdl3", "mgseqno", this.overallName + "." + Integer.toString(this.sectionSeqNo));
+        
+        metsDoc.setChanged(true);
+        //metsDoc.setModified(true);
+        //  System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+        } // first pass
+    
+        // append an 'end of section' marker
+        //textBuffer.append(END_OF_SECTION);
+        this.sectionSeqNo ++;
+        textBuffer.append(END_OF_SECTION);
+        textBuffer.append(START_OF_SECTION);
+        // for document-level indexes, always append an 'end of document' tag at the
+        // end of the document for each section.  Otherwise, each section is followed
+        // by an end of document character.  This ensures that all indexes use the
+        // same document numbering...
+        
+//      if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
+//      extraBuffer.append(END_OF_DOCUMENT);
+//      }
+//      else {
+//      textBuffer.append(END_OF_DOCUMENT);
+//      this.documentSeqNo ++;
+//      }
+        
+        // produce the body here for metadata output of divisions - in the case of
+        // text output, that will happen below...
+        
+        if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+        // if there is only text, don't do this
+        METSDescriptive descriptive;
+    
+        METSDivision division = (METSDivision) structure;
+
+        String metadataId = division.getDefaultMetadataReference();
+        // are there other metadata refs to get??
+        descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+        if (descriptive != null) {
+            for (int i=0; i<fields.size(); i++) {
+            String field = (String)fields.get(i);
+            if (field.equals("text")) {
+                continue;
+            }
+            List values = descriptive.getMetadata(namespace, field);
+            if (values != null) {   
+                Iterator valueIter = values.iterator();
+                while (valueIter.hasNext()) {
+                String value = valueIter.next().toString();
+                textBuffer.append("<"+field+">");
+                textBuffer.append(value);
+                textBuffer.append("</"+field+">");
+                }
+            }
+            }
+        }
+        }
+    }
+    
+    // go through our children as required...
+    Iterator children = structure.getChildIterator();
+    Node startNode;
+    boolean index_text = fields.contains("text");
+    while (children.hasNext()) {
+        AbstractStructure child = (AbstractStructure) children.next();
+      
+        // get xpointer for child 
+        // get start position node
+        if (metsDoc.getDocumentType() == "METS"){
+        startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);   
+        } else {
+        startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+        }
+        
+        // while this node isn't the child's start node, produce the 
+        // HTML node text, if in text field mode...
+        if (index_text) {
+        while (node != startNode) {
+            XPointer.printNode(node, textBuffer, false);
+            node = XPointer.getNextNode(node);
+        }
+        }
+        
+        // recurse to child
+        node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+    } // while next child
+
+    // close a document - the actual closing \B will be done by the main
+    // loop, so only a required \C is printed here...
+    // why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
+    if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+        if (this.pass == 0) {
+        System.err.println("structure structure");
+        }
+        if (index_text) {
+        while (node != null) {
+            XPointer.printNode(node, textBuffer, false);
+            node = XPointer.getNextNode(node);
+        }
+        }
+      
+        //textBuffer.append(END_OF_SECTION);
+        //this.sectionSeqNo ++;
+      
+    }
+    return node;
+    }
+
+    
+    private void printNode(Node node, StringBuffer buffer, 
+               boolean preserve_format, boolean close) {
+    
+    if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+        if (!close) {
+        buffer.append(node.getNodeValue());
+        buffer.append(" ");
+        }
+        return;
+    }
+    
+    if (node.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+       if (!close) {
+           buffer.append("&");
+           buffer.append(node.getNodeValue());
+           buffer.append(";");
+           buffer.append(" ");
+       }
+       return;
+    }
+    if (!preserve_format) return; // we don't want any xml output
+    /*
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      if (!close) {
+    buffer.append("<!-- ");
+    buffer.append(node.getNodeValue());
+    buffer.append(" -->");
+      }
+      }*/
+    if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+        
+        boolean hasChildren = (node.getChildNodes().getLength() > 0);
+        
+        if (close) {
+        if (hasChildren) {
+            // put the close node
+            buffer.append("</");
+            buffer.append(node.getNodeName());
+            buffer.append(">");
+        }
+        // else it was closed off previously
+        return;
+        }
+        
+        
+        buffer.append("<");
+        buffer.append(node.getNodeName());
+        NamedNodeMap attributes = ((Element) node).getAttributes();
+        for (int a = 0; a < attributes.getLength(); a ++) {
+        Node attributeNode = attributes.item(a);
+        buffer.append(" ");
+        buffer.append(attributeNode.getNodeName());
+        
+        String value = attributeNode.getNodeValue();
+        if (value != null && value.length() > 0) {
+            buffer.append("=\"");
+            buffer.append(value);
+            buffer.append("\"");
+        }
+        }
+        
+
+        if (!hasChildren) {
+        buffer.append(" /");
+        }
+        buffer.append(">");
+        
+        //if (close && node.getNodeName()=="Section") {
+        //buffer.append((char) 3);
+        // }
+    }
+    
+    }
+    
+    /*
+    private Node recurseDOMold(DocumentInterface metsDoc, Node node, 
+                AbstractStructure structure, StringBuffer textBuffer,
+                StringBuffer extraBuffer, String namespace) 
+    //String name, String namespace, String field)
+    {
+    List fields = current_index.getFields();
+    // send out the ctrl-c...if this is
+    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
     // try doing this for all index types
-    if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+        if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
         METSDivision division = (METSDivision) structure;
 
@@ -718 +918 @@
     return node;
   }
-
+    */
     private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
-    //  String name, String namespace, String field)
-  { StringBuffer extraBuffer = new StringBuffer();
-    Node node = document.getDocumentElement();
-    StringBuffer textBuffer = new StringBuffer();
-
-    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
-    textBuffer.append(extraBuffer.toString());
-    return textBuffer.toString();
-  }
-
+    //  String name, String namespace, String field)
+    { 
+    StringBuffer extraBuffer = new StringBuffer();
+    Node node = document.getDocumentElement();
+    StringBuffer textBuffer = new StringBuffer();
+    
+    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); 
+    textBuffer.append(extraBuffer.toString());
+    return textBuffer.toString();
+    }
+    
 }