source: tags/ant-install-branch-merged-1/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java@ 9873

package org.greenstone.gsdl3.gs3build.doctypes;

import java.io.File;
import java.net.URL;
import java.net.MalformedURLException;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.ArrayList;

import org.greenstone.gsdl3.gs3build.metadata.*;
import org.greenstone.gsdl3.gs3build.util.*;
import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
import org.greenstone.gsdl3.util.XMLConverter;

import org.w3c.dom.*;

public class HTMLDocument extends AbstractDocument
{
  public static final String HTML_DOCUMENT_TYPE = "HTML";
  Document domDocument;

  public HTMLDocument(DocumentID id)
  { super(id);
    this.domDocument = null;
  }

  /**
   *  Create the HTMLDocument from a given URL - the URL may in fact be a reference
   *  to a local file.
   *
   *  @param <code>URL</code> The location from which to load the file
   */
  public HTMLDocument(URL url)
  { super(url);

    this.loadDocument(url);

    this._extractDocumentFiles();
    this._extractDocumentMetadata();
    // extract the files before resolving the images
    resolveImages();
    HTMLDocumentTools docTools = new HTMLDocumentTools(this.domDocument);
    docTools.setMetsDocument(this);
    docTools.setUrl(this.fileSet.getFile(0).getLocation());
    docTools.findSections();
    METSStructure sectionStruct = docTools.getStructure();

    if (sectionStruct.size() > 0) {
      METSStructureSet structureSet = this.getDocumentStructure();
      structureSet.addStructure(sectionStruct);
    }
  }

  private void loadDocument(URL url)
  {
    //    HTMLDoc htmlDoc;

    HTMLTidy tidyDoc;
    if (url.getProtocol().equals("file"))
    { tidyDoc = new HTMLTidy(new File(url.getPath()));
    }
    else
    { //htmlDoc = new HTMLDoc(url);
      tidyDoc = new HTMLTidy(url);
    }
    
    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());

    this.domDocument = tidyDoc.getDocument(); 
    Runtime.getRuntime().gc();
    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
 }

  private void _extractDocumentMetadata()
  { 
    NodeList metadata = this.domDocument.getElementsByTagName("META");
    for (int n = 0; n < metadata.getLength(); n ++) {
      Node node = metadata.item(n);
      Element element = (Element) node;

      String name = element.getAttribute("name");
      if (name == null || name.length() == 0) {
    continue;
      }

      String value = element.getAttribute("content");
      if (value == null || value.length() == 0) {
    value = name;
      }

      this.addDocumentMetadata(new MetadataLabel(name), value);
    }
    
    NodeList titles = this.domDocument.getElementsByTagName("TITLE");
    StringBuffer title = new StringBuffer();
    for (int n = 0; n < titles.getLength(); n ++) {
      Node node = titles.item(n);
      Element element = (Element) node;

      NodeList childNodes = node.getChildNodes();
      for (int c = 0; c < childNodes.getLength(); c ++) {
    Node child = childNodes.item(c);
    if (child.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
      title.append(child.getNodeValue());
    }
      }
    }

    if (title.length() > 0)
    { this.addDocumentMetadata(new MetadataLabel("Title"), title.toString());
    }

    /* Old HTMLDoc based parsing...
    HTMLBlock codedContent = htmlDoc.getCodedContent();
    boolean inTitle = false;
    title = new StringBuffer();

    for (int e = 0; e < codedContent.size(); e ++)
    { if (codedContent.elementAt(e) instanceof HTMLTag)
      { HTMLTag tag = (HTMLTag) codedContent.elementAt(e);

        if (tag.tagName().equals("meta"))
    { // check that the name of the metadata item exists
      String name = tag.idValue("name");
      if (name == null || name.length() == 0) {
        continue;
      }

      // get the value, if it exists
      String value = tag.idValue("content");

      // if value does not exist, default it to being the same
      // as the name.
      if (value == null || value.length() > 0) {
        value = name;
      }
    }
    else 
    if (tag.tagName().equals("title"))
    { inTitle = true;
    }
    else if (tag.tagName().equals("/title"))
    { inTitle = false;
    }
    // cut off when real body content appears - not a perfect
        // implementation, just cheap & cheerful
    else if (tag.tagName().equals("/head"))
    { break;
    }
    else if (tag.tagName().equals("body"))
    { break;
    }
      }
      else if (inTitle == true)
      { title.append(codedContent.elementAt(e).toString());
      }
    }
      */
  }

  private void _extractDocumentFiles()
  { URL homeUrl = this.fileSet.getFile(0).getLocation();

    NodeList metadata = this.domDocument.getElementsByTagName("img");
    for (int n = 0; n < metadata.getLength(); n ++) {
      Node node = metadata.item(n);
      Element element = (Element) node;

      String location = element.getAttribute("src");
      if (location == null || location.length() == 0) {
    System.out.println("No name");
    continue;
      }
      try 
      { // make the url for the image, and then add it to the document list of
    // 
    URL imgUrl = new URL(homeUrl, location);
    METSFile file = this.fileSet.addFile(imgUrl);
    this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
      }
      catch (MalformedURLException ex)
      { // TODO: report exception/failure to resolve...
      }
    }

  
    /**    
    HTMLBlock codedContent = htmlDoc.getCodedContent();
    for (int e = 0; e < codedContent.size(); e ++)
    { if (codedContent.elementAt(e) instanceof HTMLTag)
      { HTMLTag tag = (HTMLTag) codedContent.elementAt(e);

        if (tag.tagName().equals("img"))
    { String location = tag.idValue("src");

      try 
      { // make the url for the image, and then add it to the document list of
        // 
        URL imgUrl = new URL(homeUrl, location);
        METSFile file = this.fileSet.addFile(imgUrl);
        this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
      }
      catch (MalformedURLException ex)
      { // TODO: report exception/failure to resolve...
      }
    }
      }
    }
    */
  }

  public String getDocumentType()
  { return HTML_DOCUMENT_TYPE;
  }

    // I think this is used for single section documents, while getSectionText
    // is used for sectioned documents
    // we will use the domDocument rather than reading it in again to another HTMLDoc.
  public String getDocumentText()
  { 
      XMLConverter converter = new XMLConverter();
      return converter.getPrettyString(this.domDocument.getDocumentElement());
      /*  HTMLDoc htmlDoc;
    URL     url =(URL) this.fileSet.getFile(0).getLocation();

    this.getSectionText("1");

    if (url.getProtocol().equals("file"))
    { htmlDoc = new HTMLDoc(url, url.getPath());
    }
    else
    { htmlDoc = new HTMLDoc(url);
    }
    return htmlDoc.getContent();
      */ 
  }

    private void resolveImages() {

    // find the path of the url relative to the collection
    URL full_path = this.fileSet.getFile(0).getLocation();
       
    String base_url;
    if (full_path.getProtocol().equals("file")) {
        base_url = full_path.getPath();
        int import_pos = base_url.indexOf("import");
        base_url = base_url.substring(import_pos);
        base_url = "_httpcollection_/"+base_url;
    } else {
        base_url = full_path.toString();
    }

    // need to take off the last part
    base_url = base_url.substring(0, base_url.lastIndexOf("/")+1);

    NodeList metadata = this.domDocument.getElementsByTagName("img");
    for (int n = 0; n < metadata.getLength(); n ++) {
        Node node = metadata.item(n);
        Element element = (Element) node;
        
        String location = element.getAttribute("src");
        if (location != null && location.length() > 0 && isRelative(location)) {
        // modify the source url
        element.setAttribute("src", base_url+location);
        }
    }
    }

    private boolean isRelative(String location) {

    if ( location.startsWith("http:") || location.startsWith("file:")) {
        return false;
    }
    return true;
    }
    
    public Document getDOMDocument()
    {    
      if (this.domDocument == null) {
      URL     url =(URL) this.fileSet.getFile(0).getLocation();
      this.loadDocument(url);
      resolveImages();
    }
    return this.domDocument;
  }

  private XPointer getXPointer(METSDivision division)
  { String fileId = division.getDefaultFileReference();

    if (fileId == null) {
      System.err.println("Unable to obtain file reference for section " + division.getLabel());
      return null;
    }

    METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
    if (fileGroup == null) {
      System.err.println("Unable to obtain file reference for filegroup " + fileId);
      return null;
    }

    METSFile file = fileGroup.getFile(0);
    if (file == null) {
      System.err.println("Unable to obtain any files within filegroup " + fileId);
      return null;
    }

    URL url = file.getLocation();
    XPointer xpointer = XPointer.processXPointer(this.domDocument, url);

    return xpointer;
  }

  private XPointer getXPointer(String sectionId)
  { if (this.domDocument == null) {
      URL     url =(URL) this.fileSet.getFile(0).getLocation();
      this.loadDocument(url);
      resolveImages();
    }

    METSDivision division = 
      this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
                          sectionId);
    if (division == null) {
      //      System.err.println("Unable to locate section " + sectionId);
      return null; 
    }

    return this.getXPointer(division);
  }
  
  public Node getSectionStartNode(METSDivision division)
  { XPointer xpointer = this.getXPointer(division);
    return xpointer.getStartNode();
  }

  public Node getSectionStartNode(String sectionId)
  { XPointer xpointer = this.getXPointer(sectionId);

    return xpointer.getStartNode();
  }

  public String getSectionText(String sectionId)
  { XPointer xpointer = this.getXPointer(sectionId);

    if (xpointer == null) {
      return "";
    }

    // get the XML content of the xpointer...
    return xpointer.toString();
  }
}