Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 6283)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 6284)
@@ -13,4 +13,5 @@
 
 import org.greenstone.gsdl3.gs3build.metadata.NamespaceFactory;
+import org.greenstone.gsdl3.gs3build.metadata.StructureIdentifierFactory;
 import org.greenstone.gsdl3.gs3build.metadata.GSDL3Namespace;
 import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
@@ -22,4 +23,5 @@
 import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
 import org.greenstone.gsdl3.gs3build.metadata.METSNamespace;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
 
 import org.greenstone.gsdl3.gs3build.util.MultiMap;
@@ -39,4 +41,5 @@
   DocumentID       id;
   boolean          isModified;
+  StructureIdentifierFactory structureIdFactory;
 
   /**
@@ -53,4 +56,5 @@
     this.structureSet = new METSStructureSet();
     this.id           = id;
+    this.structureIdFactory = new StructureIdentifierFactory();
   }
   
@@ -152,18 +156,7 @@
    *   @see DocumentInterface:addDocumentMetadata
    */
-  public void addDocumentMetadata(String name, String value)
-  { int colonAt = name.indexOf(":");
-    String namespace;
-		
-    if (colonAt > 0) {
-      namespace = name.substring(0, colonAt);
-      name = name.substring(colonAt+1);
-    }
-    else {
-      namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
-    }
-
-    // no need to set isModified, as the following call will do it anyway!
-    this.addDocumentMetadata(namespace, name, value);
+  public void addDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.addDocumentMetadata(label.getNamespace(), label.getLabel(), value);
   }
 
@@ -286,4 +279,11 @@
   { this.fileSet = fileSet;
   }
+
+  /**
+   *  This is just a dummy function that does nothing at this level...
+   */
+  public org.w3c.dom.Document getDOMDocument()
+  { return null;
+  }
   
   /**
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 6283)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 6284)
@@ -57,4 +57,9 @@
    */
   public String getDocumentText();
+
+  /**
+   *  The document as a dom object
+   */
+  public org.w3c.dom.Document getDOMDocument();
 
   /**
@@ -113,5 +118,5 @@
    *  the label.
    *
-   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
    *                             sub-component structures.  The label may commence
    *                             with a namespace followed by a colon. 
@@ -119,5 +124,5 @@
    *  @param <code>String</code> value of the metadata
    */
-  public void addDocumentMetadata(String label, String value);
+  public void addDocumentMetadata(MetadataLabel label, String value);
   
   /**
@@ -132,5 +137,4 @@
   public void addDocumentMetadata(String namespace, String label, String value);
 
-  
   /**
    *  Post metadata to a file in this document - the appropriate changes
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 6283)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 6284)
@@ -9,13 +9,18 @@
 import java.util.ArrayList;
 
-import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.*;
 import org.greenstone.gsdl3.gs3build.util.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+
+import org.w3c.dom.*;
 
 public class HTMLDocument extends AbstractDocument
 {
   public static final String HTML_DOCUMENT_TYPE = "HTML";
+  Document domDocument;
 
   public HTMLDocument(DocumentID id)
   { super(id);
+    this.domDocument = null;
   }
 
@@ -29,23 +34,84 @@
   { super(url);
 
-    HTMLDoc htmlDoc;
+    this.loadDocument(url);
+
+    this._extractDocumentFiles();
+    this._extractDocumentMetadata();
+
+    HTMLDocumentTools docTools = new HTMLDocumentTools(this.domDocument);
+    docTools.setMetsDocument(this);
+    docTools.setUrl(this.fileSet.getFile(0).getLocation());
+    docTools.findSections();
+    METSStructure sectionStruct = docTools.getStructure();
+
+    if (sectionStruct.size() > 0) {
+      METSStructureSet structureSet = this.getDocumentStructure();
+      structureSet.addStructure(sectionStruct);
+    }
+  }
+
+  private void loadDocument(URL url)
+  {
+    //    HTMLDoc htmlDoc;
+    HTMLTidy tidyDoc;
     if (url.toString().startsWith("file://"))
-    { htmlDoc = new HTMLDoc(url, url.toString().substring(7));
+    { //htmlDoc = new HTMLDoc(url, url.toString().substring(7));
+      tidyDoc = new HTMLTidy(new File(url.toString().substring(7)));
     }
     else if (url.toString().startsWith("file:/"))
-    { htmlDoc = new HTMLDoc(url, url.toString().substring(5));
+    { //htmlDoc = new HTMLDoc(url, url.toString().substring(5));
+      tidyDoc = new HTMLTidy(new File( url.toString().substring(5)));
     }
     else
-    { htmlDoc = new HTMLDoc(url);
+    { //htmlDoc = new HTMLDoc(url);
+      tidyDoc = new HTMLTidy(url);
     }
     
-    this._extractDocumentFiles(htmlDoc);
-    this._extractDocumentMetadata(htmlDoc);
-  }
-
-  private void _extractDocumentMetadata(HTMLDoc htmlDoc)
-  { HTMLBlock codedContent = htmlDoc.getCodedContent();
+    this.domDocument = tidyDoc.getDocument(); 
+ }
+
+  private void _extractDocumentMetadata()
+  { 
+    NodeList metadata = this.domDocument.getElementsByTagName("META");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String name = element.getAttribute("name");
+      if (name == null || name.length() == 0) {
+	continue;
+      }
+
+      String value = element.getAttribute("content");
+      if (value == null || value.length() == 0) {
+	value = name;
+      }
+
+      this.addDocumentMetadata(new MetadataLabel(name), value);
+    }
+    
+    NodeList titles = this.domDocument.getElementsByTagName("TITLE");
+    StringBuffer title = new StringBuffer();
+    for (int n = 0; n < titles.getLength(); n ++) {
+      Node node = titles.item(n);
+      Element element = (Element) node;
+
+      NodeList childNodes = node.getChildNodes();
+      for (int c = 0; c < childNodes.getLength(); c ++) {
+	Node child = childNodes.item(c);
+	if (child.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	  title.append(child.getNodeValue());
+	}
+      }
+    }
+
+    if (title.length() > 0)
+    { this.addDocumentMetadata(new MetadataLabel("title"), title.toString());
+    }
+
+    /* Old HTMLDoc based parsing...
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
     boolean inTitle = false;
-    StringBuffer title = new StringBuffer();
+    title = new StringBuffer();
 
     for (int e = 0; e < codedContent.size(); e ++)
@@ -62,16 +128,13 @@
 	  // get the value, if it exists
 	  String value = tag.idValue("content");
-	  if (value != null && value.length() > 0) {
-	    System.out.println("  " + value);
-	  }
+
 	  // if value does not exist, default it to being the same
 	  // as the name.
-	  else {
+	  if (value == null || value.length() > 0) {
 	    value = name;
 	  }
-
-	  this.addDocumentMetadata(name, value);
-	}
-	else if (tag.tagName().equals("title"))
+	}
+	else 
+	if (tag.tagName().equals("title"))
 	{ inTitle = true;
 	}
@@ -92,12 +155,34 @@
       }
     }
-    if (title.length() > 0)
-    { this.addDocumentMetadata("title", title.toString());
-    }
-  }
-
-  private void _extractDocumentFiles(HTMLDoc htmlDoc)
+      */
+  }
+
+  private void _extractDocumentFiles()
   { URL homeUrl = this.fileSet.getFile(0).getLocation();
 
+    NodeList metadata = this.domDocument.getElementsByTagName("img");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String location = element.getAttribute("src");
+      if (location == null || location.length() == 0) {
+	System.out.println("No name");
+	continue;
+      }
+
+      try 
+      { // make the url for the image, and then add it to the document list of
+	// 
+	URL imgUrl = new URL(homeUrl, location);
+	METSFile file = this.fileSet.addFile(imgUrl);
+	this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+      }
+      catch (MalformedURLException ex)
+      { // TODO: report exception/failure to resolve...
+      }
+    }
+
+    /**    
     HTMLBlock codedContent = htmlDoc.getCodedContent();
     for (int e = 0; e < codedContent.size(); e ++)
@@ -121,4 +206,5 @@
       }
     }
+    */
   }
 
@@ -132,4 +218,6 @@
     URL     url =(URL) this.fileSet.getFile(0).getLocation();
 
+    this.getSectionText("1");
+
     if (url.toString().startsWith("file://"))
     { htmlDoc = new HTMLDoc(url, url.toString().substring(7));
@@ -144,6 +232,76 @@
   }
 
-  public String getSectionText(String document)
-  { return "";
+  public Document getDOMDocument()
+  {
+    if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+    }
+    return this.domDocument;
+  }
+
+  private XPointer getXPointer(METSDivision division)
+  { String fileId = division.getDefaultFileReference();
+
+    if (fileId == null) {
+      System.err.println("Unable to obtain file reference for section " + division.getLabel());
+      return null;
+    }
+
+    METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+    if (fileGroup == null) {
+      System.err.println("Unable to obtain file reference for filegroup " + fileId);
+      return null;
+    }
+
+    METSFile file = fileGroup.getFile(0);
+    if (file == null) {
+      System.err.println("Unable to obtain any files within filegroup " + fileId);
+      return null;
+    }
+
+    URL url = file.getLocation();
+    XPointer xpointer = XPointer.processXPointer(this.domDocument, url);
+
+    return xpointer;
+  }
+
+  private XPointer getXPointer(String sectionId)
+  { if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+    }
+
+    METSDivision division = 
+      this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					      sectionId);
+    if (division == null) {
+      //      System.err.println("Unable to locate section " + sectionId);
+      return null; 
+    }
+
+    return this.getXPointer(division);
+  }
+  
+  public Node getSectionStartNode(METSDivision division)
+  { XPointer xpointer = this.getXPointer(division);
+    return xpointer.getStartNode();
+  }
+
+  public Node getSectionStartNode(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    return xpointer.getStartNode();
+  }
+
+  public String getSectionText(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    if (xpointer == null) {
+      return "";
+    }
+
+    // get the XML content of the xpointer...
+    return xpointer.toString();
   }
 }
Index: /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java
===================================================================
--- /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 6284)
+++ /trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 6284)
@@ -0,0 +1,328 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.w3c.dom.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.AbstractStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFilePos;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileID;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
+import org.greenstone.gsdl3.gs3build.util.*;
+
+public class HTMLDocumentTools
+{
+  Document          document;
+  URL               documentUrl;
+  int               sectionLevel;
+  StringBuffer      sectionLabel;
+  List              sectionSeq;
+  HTMLSection       rootSection;
+  HTMLSection       currentSection;
+  int               sectionSeqNo;
+  METSStructure     metsStructure;
+  AbstractStructure currentMets;
+  AbstractDocument  metsDocument;
+
+  /**
+   *
+   */
+  class HTMLSection
+  { String startLocation;
+    String endLocation;
+    String label;
+    List   children;
+    HTMLSection parent;
+    int         seqNo;
+
+    public HTMLSection(String label, int seqNo)
+    { this.label = label;
+      this.seqNo = seqNo;
+      this.parent = null;
+      this.children = new ArrayList();
+    }
+    
+    public void setStartLocation(String location)
+    { this.startLocation = location;
+    }
+    
+    public void setEndLocation(String location)
+    { this.endLocation = location;
+    }
+
+    public void addChild(HTMLSection child)
+    { this.children.add(child);
+      child.parent = this;
+    }
+    
+    public HTMLSection getParent()
+    { return this.parent;
+    }
+
+    public String getLabel()
+    { return this.label;
+    }
+
+    public String getXPointer()
+    { StringBuffer xPointerBuffer = new StringBuffer();
+    
+      xPointerBuffer.append("xpointer(");
+      xPointerBuffer.append(this.startLocation);
+      xPointerBuffer.append("/range-to(");
+      xPointerBuffer.append(this.endLocation);
+      xPointerBuffer.append("))");
+      
+      return xPointerBuffer.toString();
+    }
+  }
+
+  public HTMLDocumentTools(Document document)
+  { this.document = document;
+    this.sectionLevel = 0;
+    this.sectionSeq   = new ArrayList();
+    this.sectionSeq.add(new Integer(0));
+    this.sectionLabel = new StringBuffer();
+    this.rootSection = new HTMLSection("", 0);
+    this.currentSection = this.rootSection;
+    this.metsStructure  = new METSStructure(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					    "Section", "Section");
+    this.currentMets    = this.metsStructure;
+    this.metsDocument   = null;
+  }
+
+  public void setMetsDocument(AbstractDocument document)
+  { this.metsDocument = document;
+  }
+
+  public void setUrl(URL url)
+  { this.documentUrl = url;
+  }
+
+  public Object findSections()
+  { this.findSections(document.getDocumentElement(), "/1");
+    return this.rootSection;
+  }
+
+  public void findSections(Node node, String location)
+  { // if at an element, then we aren't going to find section information,
+    // but we do need to recurse into the children to find the section markers
+    // there
+    if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      NodeList children = node.getChildNodes();
+      StringBuffer locationBuffer = new StringBuffer();
+      locationBuffer.append(location);
+      locationBuffer.append("/");
+
+      int elementNo = 1;
+      int commentNo = 1;
+      for (int c = 0; c < children.getLength(); c++)
+      {
+	locationBuffer.setLength(location.length()+1);
+	if (children.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  locationBuffer.append(Integer.toString(elementNo));
+	  elementNo ++;
+	}
+	else if (children.item(c).getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+	  locationBuffer.append("comment()[");
+	  locationBuffer.append(Integer.toString(commentNo));
+	  locationBuffer.append("]");
+	  commentNo ++;
+	}
+	this.findSections(children.item(c), locationBuffer.toString());
+      }
+    }
+    // if at a comment node, then one needs to dissect it to discover if
+    // any section markings are included...
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      String commentText = node.getNodeValue();
+      
+      // parse the node now...checking first for any 'section' items in
+      // it - this is a crude check...
+      if (commentText.indexOf("<Section>") >= 0 ||
+	  commentText.indexOf("</Section>") >= 0) {
+	// parse the section roughly using the primitive HTMLDoc form...      
+	HTMLDoc innerDoc     = HTMLDoc.fromString("dummy", commentText);
+	HTMLBlock innerHtml  = innerDoc.getCodedContent();
+	boolean   inMetadata = false;
+	StringBuffer metaValue = new StringBuffer();
+	String       metaLabel = null;
+
+	// Now just go through the content in sequence...
+	for (int e = 0; e < innerHtml.size(); e ++) {
+	  Object element = innerHtml.elementAt(e);
+
+	  if (element instanceof HTMLTag) {
+	    HTMLTag tag = (HTMLTag) element;
+
+	    if (tag.tagName().equals("section")) {
+	      // get the counters sorted out for the sections...
+	      this.sectionSeq.set(this.sectionLevel, new Integer(((Integer) this.sectionSeq.get(this.sectionLevel)).intValue() + 1));
+	      this.sectionLevel ++;
+	      if (sectionSeq.size() == this.sectionLevel) {
+		this.sectionSeq.add(new Integer(0));
+	      }
+	      else {
+		this.sectionSeq.set(this.sectionLevel, new Integer(0));
+	      }
+
+	      // create the label for the new section...
+	      this.sectionLabel.setLength(0);
+	      for (int s = 0; s < sectionLevel; s ++) {
+		if (s > 0) {
+		  this.sectionLabel.append(".");
+		}
+		this.sectionLabel.append(this.sectionSeq.get(s).toString());
+	      }
+
+	      // note the new section in a new data structure...
+	      HTMLSection newSection = new HTMLSection(this.sectionLabel.toString(), this.sectionSeqNo);
+
+	      newSection.setStartLocation(location);
+	      this.currentSection.addChild(newSection);
+	      this.currentSection = newSection;
+
+	      // do the same for mets...
+	      METSDivision newDivision = new METSDivision("DS" + Integer.toString(this.sectionSeqNo), Integer.toString(this.sectionSeqNo),
+							  this.sectionLabel.toString(), this.sectionLabel.toString(), "Section");
+	      this.currentMets.addDivision(newDivision);
+	      this.currentMets = newDivision;
+
+	      this.sectionSeqNo ++;
+	    }
+	    else if (tag.tagName().equals("/section")) {
+	      // cope with poorly formed metadata...
+	      if (inMetadata) {
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+
+	      // note the end position of the section
+	      this.currentSection.setEndLocation(location);
+
+	      // now post that section to the file list...
+	      this.postFilePtr(this.currentSection);
+
+	      // backtrack up the section tree...
+	      this.currentSection = this.currentSection.getParent();
+	      this.currentMets = this.currentMets.getParent();
+	      this.sectionLevel --;
+
+	      // trim the section label
+	      int cutTo = this.sectionLabel.toString().lastIndexOf('.');
+	      if (cutTo >= 0) {
+		this.sectionLabel.setLength(cutTo);
+	      }
+	      else {
+		this.sectionLabel.setLength(0);
+	      }
+
+	      // we can't be in metadata either...
+	      inMetadata = false;
+	    }
+	    else if (tag.tagName().equals("metadata")) {
+	      if (inMetadata) {
+		// post the metadata
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+	      metaValue.setLength(0);
+	      metaLabel = tag.idValue("name");
+	      inMetadata = (metaLabel != null);
+	    }
+	    else if (tag.tagName().equals("/metadata")) {
+	      // post the metadata
+	      this.postMetadata(metaLabel, metaValue.toString());
+	      inMetadata = false;
+	    }
+	  }
+	  else {
+	    if (inMetadata) {
+	      metaValue.append(element.toString());
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  private String makeXPointer(HTMLSection section)
+  { StringBuffer xPointerBuffer = new StringBuffer();
+
+    if (this.documentUrl != null)
+    { xPointerBuffer.append(this.documentUrl.toString());
+      xPointerBuffer.append("#");
+    }
+    xPointerBuffer.append(section.getXPointer());
+
+    return xPointerBuffer.toString();
+  }
+
+  private void postFilePtr(HTMLSection section)
+  { String xPointer = this.makeXPointer(section);
+
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    try {
+      URL url = new URL(xPointer);
+
+      METSFileSet fileSet = this.metsDocument.getDocumentFiles();
+
+      METSFilePos filePos = new METSFilePos(url);
+      METSFile    file = fileSet.createFile(filePos, "text/html");
+      
+      METSFileGroup fileGroup;
+      String fileRef = ((METSDivision) this.currentMets).getDefaultFileReference();
+
+      if (fileRef == null) {
+	fileGroup = fileSet.createGroup();
+	fileSet.addGroup(fileGroup);
+	((METSDivision) this.currentMets).addFileReference(fileGroup.getId());
+      }
+      else {
+	// simple call - the group should be at the 'root' level, immediately below
+	// the FileSet itself...
+	fileGroup = fileSet.getGroup(fileRef);
+      }
+      fileGroup.addFile(file);
+
+      
+    }
+    catch (java.net.MalformedURLException ex) {
+      System.err.println(ex + " " + xPointer);
+    }
+  }
+
+  private void postMetadata(String label, String value)
+  {
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    String metadataRef = ((METSDivision) this.currentMets).getDefaultMetadataReference();
+    METSDescriptive descriptive;
+    if (metadataRef == null) {
+      descriptive = this.metsDocument.getDocumentMetadata().createDescriptive(this.currentSection.getLabel());
+      ((METSDivision) this.currentMets).addMetadataReference(descriptive.getID());
+    }
+    else {
+      descriptive = this.metsDocument.getDocumentMetadata().getDescriptiveById(metadataRef);
+    }
+    descriptive.addMetadata(MetadataLabel.getDefaultNamespace(), label, value);
+  }
+
+  public METSStructure getStructure()
+  { return this.metsStructure;
+  }
+}
+
