Changeset 6284

Timestamp:

2003-12-17T13:08:02+13:00 (20 years ago)

Author:

cs025

Message:

Added HTMLDocumentTools, also modifications to the abstract interfaces
and the HTMLDocument doctype to support indexing by section.

Location:

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes

Files:

• AbstractDocument.java (modified) (6 diffs)
• DocumentInterface.java (modified) (4 diffs)
• HTMLDocument.java (modified) (7 diffs)
• HTMLDocumentTools.java (added)

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java

@@ -13 +13 @@
 
 import org.greenstone.gsdl3.gs3build.metadata.NamespaceFactory;
+import org.greenstone.gsdl3.gs3build.metadata.StructureIdentifierFactory;
 import org.greenstone.gsdl3.gs3build.metadata.GSDL3Namespace;
 import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
@@ -22 +23 @@
 import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
 import org.greenstone.gsdl3.gs3build.metadata.METSNamespace;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
 
 import org.greenstone.gsdl3.gs3build.util.MultiMap;
@@ -39 +41 @@
   DocumentID       id;
   boolean          isModified;
+  StructureIdentifierFactory structureIdFactory;
 
   /**
@@ -53 +56 @@
     this.structureSet = new METSStructureSet();
     this.id           = id;
+    this.structureIdFactory = new StructureIdentifierFactory();
   }
   
@@ -152 +156 @@
    *   @see DocumentInterface:addDocumentMetadata
    */
-  public void addDocumentMetadata(String name, String value)
-  { int colonAt = name.indexOf(":");
-    String namespace;
-        
-    if (colonAt > 0) {
-      namespace = name.substring(0, colonAt);
-      name = name.substring(colonAt+1);
-    }
-    else {
-      namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
-    }
-
-    // no need to set isModified, as the following call will do it anyway!
-    this.addDocumentMetadata(namespace, name, value);
+  public void addDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.addDocumentMetadata(label.getNamespace(), label.getLabel(), value);
   }
 
@@ -286 +279 @@
   { this.fileSet = fileSet;
   }
+
+  /**
+   *  This is just a dummy function that does nothing at this level...
+   */
+  public org.w3c.dom.Document getDOMDocument()
+  { return null;
+  }
   
   /**

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java

@@ -57 +57 @@
    */
   public String getDocumentText();
+
+  /**
+   *  The document as a dom object
+   */
+  public org.w3c.dom.Document getDOMDocument();
 
   /**
@@ -113 +118 @@
    *  the label.
    *
-   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
    *                             sub-component structures.  The label may commence
    *                             with a namespace followed by a colon. 
@@ -119 +124 @@
    *  @param <code>String</code> value of the metadata
    */
-  public void addDocumentMetadata(String label, String value);
+  public void addDocumentMetadata(MetadataLabel label, String value);
   
   /**
@@ -132 +137 @@
   public void addDocumentMetadata(String namespace, String label, String value);
 
-  
   /**
    *  Post metadata to a file in this document - the appropriate changes

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java

@@ -9 +9 @@
 import java.util.ArrayList;
 
-import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.*;
 import org.greenstone.gsdl3.gs3build.util.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+
+import org.w3c.dom.*;
 
 public class HTMLDocument extends AbstractDocument
 {
   public static final String HTML_DOCUMENT_TYPE = "HTML";
+  Document domDocument;
 
   public HTMLDocument(DocumentID id)
   { super(id);
+    this.domDocument = null;
   }
 
@@ -29 +34 @@
   { super(url);
 
-    HTMLDoc htmlDoc;
+    this.loadDocument(url);
+
+    this._extractDocumentFiles();
+    this._extractDocumentMetadata();
+
+    HTMLDocumentTools docTools = new HTMLDocumentTools(this.domDocument);
+    docTools.setMetsDocument(this);
+    docTools.setUrl(this.fileSet.getFile(0).getLocation());
+    docTools.findSections();
+    METSStructure sectionStruct = docTools.getStructure();
+
+    if (sectionStruct.size() > 0) {
+      METSStructureSet structureSet = this.getDocumentStructure();
+      structureSet.addStructure(sectionStruct);
+    }
+  }
+
+  private void loadDocument(URL url)
+  {
+    //    HTMLDoc htmlDoc;
+    HTMLTidy tidyDoc;
     if (url.toString().startsWith("file://"))
-    { htmlDoc = new HTMLDoc(url, url.toString().substring(7));
+    { //htmlDoc = new HTMLDoc(url, url.toString().substring(7));
+      tidyDoc = new HTMLTidy(new File(url.toString().substring(7)));
     }
     else if (url.toString().startsWith("file:/"))
-    { htmlDoc = new HTMLDoc(url, url.toString().substring(5));
+    { //htmlDoc = new HTMLDoc(url, url.toString().substring(5));
+      tidyDoc = new HTMLTidy(new File( url.toString().substring(5)));
     }
     else
-    { htmlDoc = new HTMLDoc(url);
+    { //htmlDoc = new HTMLDoc(url);
+      tidyDoc = new HTMLTidy(url);
     }
     
-    this._extractDocumentFiles(htmlDoc);
-    this._extractDocumentMetadata(htmlDoc);
-  }
-
-  private void _extractDocumentMetadata(HTMLDoc htmlDoc)
-  { HTMLBlock codedContent = htmlDoc.getCodedContent();
+    this.domDocument = tidyDoc.getDocument(); 
+ }
+
+  private void _extractDocumentMetadata()
+  { 
+    NodeList metadata = this.domDocument.getElementsByTagName("META");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String name = element.getAttribute("name");
+      if (name == null || name.length() == 0) {
+    continue;
+      }
+
+      String value = element.getAttribute("content");
+      if (value == null || value.length() == 0) {
+    value = name;
+      }
+
+      this.addDocumentMetadata(new MetadataLabel(name), value);
+    }
+    
+    NodeList titles = this.domDocument.getElementsByTagName("TITLE");
+    StringBuffer title = new StringBuffer();
+    for (int n = 0; n < titles.getLength(); n ++) {
+      Node node = titles.item(n);
+      Element element = (Element) node;
+
+      NodeList childNodes = node.getChildNodes();
+      for (int c = 0; c < childNodes.getLength(); c ++) {
+    Node child = childNodes.item(c);
+    if (child.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+      title.append(child.getNodeValue());
+    }
+      }
+    }
+
+    if (title.length() > 0)
+    { this.addDocumentMetadata(new MetadataLabel("title"), title.toString());
+    }
+
+    /* Old HTMLDoc based parsing...
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
     boolean inTitle = false;
-    StringBuffer title = new StringBuffer();
+    title = new StringBuffer();
 
     for (int e = 0; e < codedContent.size(); e ++)
@@ -62 +128 @@
       // get the value, if it exists
       String value = tag.idValue("content");
-      if (value != null && value.length() > 0) {
-        System.out.println("  " + value);
-      }
+
       // if value does not exist, default it to being the same
       // as the name.
-      else {
+      if (value == null || value.length() > 0) {
         value = name;
       }
-
-      this.addDocumentMetadata(name, value);
-    }
-    else if (tag.tagName().equals("title"))
+    }
+    else 
+    if (tag.tagName().equals("title"))
     { inTitle = true;
     }
@@ -92 +155 @@
       }
     }
-    if (title.length() > 0)
-    { this.addDocumentMetadata("title", title.toString());
-    }
-  }
-
-  private void _extractDocumentFiles(HTMLDoc htmlDoc)
+      */
+  }
+
+  private void _extractDocumentFiles()
   { URL homeUrl = this.fileSet.getFile(0).getLocation();
 
+    NodeList metadata = this.domDocument.getElementsByTagName("img");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String location = element.getAttribute("src");
+      if (location == null || location.length() == 0) {
+    System.out.println("No name");
+    continue;
+      }
+
+      try 
+      { // make the url for the image, and then add it to the document list of
+    // 
+    URL imgUrl = new URL(homeUrl, location);
+    METSFile file = this.fileSet.addFile(imgUrl);
+    this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+      }
+      catch (MalformedURLException ex)
+      { // TODO: report exception/failure to resolve...
+      }
+    }
+
+    /**    
     HTMLBlock codedContent = htmlDoc.getCodedContent();
     for (int e = 0; e < codedContent.size(); e ++)
@@ -121 +206 @@
       }
     }
+    */
   }
 
@@ -132 +218 @@
     URL     url =(URL) this.fileSet.getFile(0).getLocation();
 
+    this.getSectionText("1");
+
     if (url.toString().startsWith("file://"))
     { htmlDoc = new HTMLDoc(url, url.toString().substring(7));
@@ -144 +232 @@
   }
 
-  public String getSectionText(String document)
-  { return "";
+  public Document getDOMDocument()
+  {
+    if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+    }
+    return this.domDocument;
+  }
+
+  private XPointer getXPointer(METSDivision division)
+  { String fileId = division.getDefaultFileReference();
+
+    if (fileId == null) {
+      System.err.println("Unable to obtain file reference for section " + division.getLabel());
+      return null;
+    }
+
+    METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+    if (fileGroup == null) {
+      System.err.println("Unable to obtain file reference for filegroup " + fileId);
+      return null;
+    }
+
+    METSFile file = fileGroup.getFile(0);
+    if (file == null) {
+      System.err.println("Unable to obtain any files within filegroup " + fileId);
+      return null;
+    }
+
+    URL url = file.getLocation();
+    XPointer xpointer = XPointer.processXPointer(this.domDocument, url);
+
+    return xpointer;
+  }
+
+  private XPointer getXPointer(String sectionId)
+  { if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+    }
+
+    METSDivision division = 
+      this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+                          sectionId);
+    if (division == null) {
+      //      System.err.println("Unable to locate section " + sectionId);
+      return null; 
+    }
+
+    return this.getXPointer(division);
+  }
+  
+  public Node getSectionStartNode(METSDivision division)
+  { XPointer xpointer = this.getXPointer(division);
+    return xpointer.getStartNode();
+  }
+
+  public Node getSectionStartNode(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    return xpointer.getStartNode();
+  }
+
+  public String getSectionText(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    if (xpointer == null) {
+      return "";
+    }
+
+    // get the XML content of the xpointer...
+    return xpointer.toString();
   }
 }