source: trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java@ 6376

package org.greenstone.gsdl3.gs3build.indexers;

import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;

import org.w3c.dom.*;

import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
import org.greenstone.gsdl3.gs3build.metadata.*;
import org.greenstone.gsdl3.gs3build.xpointer.XPointer;

public class MGIndexer implements IndexerInterface
{ 
  int          pass;
  int          documentSeqNo;
  int          sectionSeqNo;
  boolean      firstDocument;
  String       outputDirectory;
  InputStream  indexerFeedback;
  InputStream  indexerErrors;
  OutputStream indexerTextfeed;
  Process      mg_passes;
  File         textDirectory;
  File         indexDirectory;
  String       indexStem;
  String       textStem;
  List         indexes;
  String       indexName;
  String       level;
  String       field;

  static final char END_OF_DOCUMENT = (char) 2;
  static final char END_OF_SECTION  = (char) 3;
  static final char END_OF_STREAM   = (char) 4;

  class MGIndex
  { String level;
    String field;

    public MGIndex(String level, String field)
    { this.level = level;
      this.field = field;
    }

    public MGIndex(String indexLabel)
    { int colonAt = indexLabel.indexOf(':');
    
      if (colonAt >= 0)
      { field = indexLabel.substring(colonAt+1);
        level = indexLabel.substring(0, colonAt);
      }
    }

    public String getLevel()
    { return this.level;
    }

    public String getField()
    { return this.field;
    }
  }

  public MGIndexer()
  { this.indexes = new ArrayList();
  }

  private String getIndexDirectory(String level, String field)
  { StringBuffer directory = new StringBuffer();
    directory.append(Character.toLowerCase((char) level.charAt(0)));

    int c, w;
    w = 0;
    c = 0;
    while (c < field.length() && w < 2) {
      char ch = field.charAt(c);

      ch = Character.toLowerCase(ch);
      if (Character.isLetter(ch)) {
    if (ch != 'a' && ch != 'e' && ch != 'i' &&
        ch != 'o' && ch != 'u') {
      directory.append(ch);
      w++;
    }
      }
      c ++;
    }
    return directory.toString();
  }

  /**
   *  The output directory should be (collection)/building/text/ for
   *  normal Greenstone builds.
   *
   *  @param <code>String</code> the label to configure
   *  @param <code>String</code> the value...
   */ 
  public boolean configure(String label, String value)
  { 
    if (label.equals(IndexerManager.outputDir)) {
      this.outputDirectory = value;
      this.textStem   = value + "/text/index";
      this.pass = 0;
    
      // attempt to ensure that the text subdirectory exists
      this.textDirectory = new File(outputDirectory, "text");
      if (!textDirectory.exists()) {
    if (!textDirectory.mkdir()) {
      return false;
    }
      }
      else if (!textDirectory.isDirectory()) {
    return false;
      }

      // Sign to the user which mg directory is being used...
      System.out.println("Output MG directory is " + this.textStem);
    }
    else if (label.equals(IndexerInterface.GS2_INDEX_LABEL)) {
      this.indexes.add(new MGIndex(value));
    }

    return true;
  }

  public boolean addIndex(String level, String field)
  {
    MGIndex index = new MGIndex(level, field);
    this.indexes.add(index);
    return true;
  }

  private Node recurseDOM(DocumentInterface metsDoc, Node node, 
              AbstractStructure structure, StringBuffer textBuffer,
              StringBuffer extraBuffer, String indexName,
              String namespace, String field)
  {
    // send out the ctrl-c...if this is
    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
      if ((indexName != null) && indexName.startsWith("s")) {
    METSDivision division = (METSDivision) structure;

    // get the division metadata block
    METSDescriptive descriptive;
    String metadataId = division.getDefaultMetadataReference();
    if (metadataId == null) {
      descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
      division.addMetadataReference(descriptive.getID());
    }
    else {
      // Get the descriptive item...
      descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
    }

    descriptive.addMetadata("gsdl3", "mgseqno", indexName + "." + Integer.toString(this.documentSeqNo));
    metsDoc.setModified(true);
    //  System.out.println("Assigning " + this.documentSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
      }
      
      // append an 'end of section' marker
      textBuffer.append(END_OF_SECTION);

      // for document-level indexes, always append an 'end of document' tag at the
      // end of the document for each section.  Otherwise, each section is followed
      // by an end of document character.  This ensures that all indexes use the
      // same document numbering...
      if (this.level == null ||
      this.level.equals(IndexerInterface.DOCUMENT_LEVEL)) {
    // extraBuffer.append(END_OF_DOCUMENT);
      }
      else {
    textBuffer.append(END_OF_DOCUMENT);
    this.documentSeqNo ++;
      }
      this.sectionSeqNo ++;

      // produce the body here for metadata output of divisions - in the case of
      // text output, that will happen below...
      if (!this.field.equals("text")) 
      { METSDescriptive descriptive;
    
    METSDivision division = (METSDivision) structure;

    String metadataId = division.getDefaultMetadataReference();
        
    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
    if (descriptive != null) {
      List values = descriptive.getMetadata(namespace, field);
      
      if (values != null) { 
        Iterator valueIter = values.iterator();
        while (valueIter.hasNext()) {
          String value = valueIter.next().toString();
          
          textBuffer.append(value);
          if (valueIter.hasNext()) {
        textBuffer.append(END_OF_SECTION);
          }
        }
      }
    }
      }
    }

    // go through our children as required...
    Iterator children = structure.getChildIterator();
    while (children.hasNext()) {
      AbstractStructure child = (AbstractStructure) children.next();
      
      // get xpointer for child 
      // get start position node
      Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
      
      // while this node isn't the child's start node, produce the HTML node text, if
      // in text field mode...
      if (field.equals("text")) {
    while (node != startNode) {
      XPointer.printNode(node, textBuffer, false);

      // print buffer to node
      node = XPointer.getNextNode(node, (field.equals("text") ? textBuffer : null));
    }
      }
      
      // recurse to child
      this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, indexName, namespace, field);
    }

    // close a document - the actual closing \B will be done by the main
    // loop, so only a required \C is printed here...
    if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
      while (node != null) {
    if (field.equals("text")) {
      XPointer.printNode(node, textBuffer, false);
    }
    node = XPointer.getNextNode(node, (field.equals("text") ? textBuffer : null));
      }
      /*
      textBuffer.append(END_OF_SECTION);
      this.sectionSeqNo ++;
      */
    }
    return node;
  }

  private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, 
                String indexName, String namespace, String field)
  { StringBuffer extraBuffer = new StringBuffer();
    Node node = document.getDocumentElement();
    StringBuffer textBuffer = new StringBuffer();

    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, indexName, namespace, field);
    textBuffer.append(extraBuffer.toString());
    return textBuffer.toString();
  }

  /**
   *  Index a single document; the document interface can be used to extract individual
   *  metadata items etc. as required or desired and index those instead or as well as
   *  the body text of the document.
   */
  public boolean indexDocument(DocumentID docID, DocumentInterface document)
  {
    if (!this.firstDocument)
    { // Send a 'CTRL-B' before the document itself
      try {
    this.indexerTextfeed.write(2);
      }
      catch (IOException ex)
      { System.out.println("Bad output on end of document" + ex);
    ex.printStackTrace();
    return false;
      }
    }
    String docText = null;

    int startSeqNo = this.sectionSeqNo;

    Document domDocument = document.getDOMDocument();
    if (domDocument != null) {
      METSStructure sections = document.getDocumentStructure().getStructure("Section");
      if (sections != null) {
    docText = this.prepareDOM(document, domDocument, sections, this.indexName, "gsdl3", this.field);
    //  System.out.println(docText);
      }
    }
    if (docText == null) {
      if (this.field.equals("text")) {
    docText = Character.toString(END_OF_DOCUMENT) + Character.toString(END_OF_SECTION) + 
      document.getDocumentText();
      }
      else {
    StringBuffer textBuffer = new StringBuffer();
    textBuffer.append(END_OF_DOCUMENT);
    textBuffer.append(END_OF_SECTION);
    List values = document.getDocumentMetadataItem("gsdl3", this.field);
    if (values != null) {
      Iterator valueIter = values.iterator();
      while (valueIter.hasNext()) {
        String value = valueIter.next().toString();
        
        textBuffer.append(value);
        if (valueIter.hasNext()) {
          textBuffer.append(END_OF_SECTION);
          sectionSeqNo ++;
        }
      }
    }
    else {
      textBuffer.append("No data");
    }
    docText = textBuffer.toString();
      }
      sectionSeqNo ++;
    }

    byte [] bytes = docText.getBytes();
    int pos = 0, end = bytes.length;

    try {
      while (pos < end) {
    this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
    pos = pos + 512;
      
    try {
      while (this.indexerFeedback.available() > 0)
          { byte b[] = new byte[this.indexerFeedback.available()];
        System.out.println("Feedback of " + this.indexerFeedback.available());
        this.indexerFeedback.read(b);
        System.out.println(b);
      }
    }
    catch (IOException ex)
        { System.out.println(ex);
    }


    try {
      while (this.indexerErrors.available() > 0)
      { byte b[] = new byte[this.indexerErrors.available()];
        System.out.println("Feedback of " + this.indexerErrors.available());
        this.indexerErrors.read(b);
        System.out.println(new String(b));
      }
    }
    catch (IOException ex)
        { System.out.println(ex);
    }
      }
    }
    catch (IOException ex)
    { System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
      ex.printStackTrace();
      return false;
    }

    // remember that we're not on the first document, assign the sequence number
    // on the first pass only, and increment the sequence number.
    this.firstDocument = false;
    if (this.pass == 0) {
      document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(this.documentSeqNo));
      //      System.out.println("Assigning " + startSeqNo + " to " + document.getID());
    }
    this.documentSeqNo += 1;

    try {
      while (this.indexerErrors.available() > 0)
      { char c = (char) this.indexerErrors.read();
        System.out.println(c);
      }
      while (this.indexerFeedback.available() > 0)
      { byte b[] = new byte[this.indexerFeedback.available()];
        System.out.println("Feedback of " + this.indexerFeedback.available());
    this.indexerFeedback.read(b);
      }
    }
    catch (IOException ex)
    {
    }
    return true;
  }

    /**
     *  Initialise the pass: open required files, check status
     */
    public boolean startPass(int passNumber)
    {   
      this.pass = passNumber;
      this.firstDocument = true;
      this.documentSeqNo = 1;
      this.sectionSeqNo  = 1;

      int indexNo = (this.pass - 2) / 2;
      if (this.pass >= 2) {
    MGIndex index = (MGIndex) this.indexes.get(indexNo);
      
    // attempt to ensure that the text subdirectory exists
    this.indexDirectory = new File(outputDirectory, this.getIndexDirectory(index.getLevel(), index.getField()));
    if (!indexDirectory.exists()) {
      if (!indexDirectory.mkdir()) {
        return false;
      }
    }
    else if (!indexDirectory.isDirectory()) {
      return false;
    }

    this.level = index.getLevel();
    this.field = index.getField();
    this.indexName = this.getIndexDirectory(index.getLevel(), index.getField());
    this.indexStem = this.outputDirectory + File.separatorChar + 
      this.indexName + File.separatorChar + "index"; // TODO: modify for index
    if (this.pass % 2 == 1) {
      this.indexName = null;
    }
      }
      else {
    this.field = "text";
    this.level = "document";
    this.indexName = null;
      }
      System.out.println("level is " + this.level);
      System.out.println("field is " + this.field);
      System.out.println("index name is " + this.indexName);
      
      // get the parameters for this execution of mg_passes
      String pathParams = "-f index -d " + (this.pass < 2 ? this.textDirectory.toString() : this.indexDirectory.toString());

      int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);

      try {
    switch (mgPass) {
      case 0:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -T1");
        break;

      case 1:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -T2");
        break;
        
      case 2:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -2 -m 32 -s 0 -G -t 10 -N1");
        break;
        
      case 3:
        Process p = Runtime.getRuntime().exec("mg_perf_hash_build -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Perfect hashes completed");
        }
        
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -2 -c 3 -G -t 10 -N2");
        break;
    }
        
    this.indexerFeedback = mg_passes.getInputStream();
    this.indexerErrors   = mg_passes.getErrorStream();
    this.indexerTextfeed = mg_passes.getOutputStream();
      }
      catch (IOException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      } 
      catch (InterruptedException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      }
      System.out.println("Pass " + this.pass);
      return true;
    }

    /**
     *  Complete a pass - reset file counters, close files, etc.
     */
    public boolean endPass(int passNumber)
    { Process p;
        
      try {
    this.indexerTextfeed.write(END_OF_DOCUMENT);
    this.indexerTextfeed.write(END_OF_STREAM);
    while (this.indexerErrors.available() > 0)
    { char c = (char) this.indexerErrors.read();
      System.out.print(c);
    }
    while (this.indexerFeedback.available() > 0)
    { byte b[] = new byte[this.indexerFeedback.available()];
      System.out.print("Feedback of " + this.indexerFeedback.available());
      this.indexerFeedback.read(b);
    }

    this.indexerTextfeed.close();
    Thread.sleep(1000);
    this.mg_passes.waitFor();
      }
      catch (IOException ex)
      { System.out.println(ex);
      }
      catch (InterruptedException ex)
      { System.out.println(ex);
      }
      System.out.println("Pass " + this.pass + " completed with " + this.mg_passes.exitValue());

      int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);

      try {
    switch (mgPass)
    { 
      case 0:
        System.out.println("Compressing dictionary");
        p = Runtime.getRuntime().exec("mg_compression_dict -f index -d " + this.textDirectory.toString() + " -S -H -2 -k 5120");
        p.waitFor();
        if (p.exitValue() != 0) {
          System.out.println("Error from mg_compression_dict: " + p.exitValue());
        }
        else {
          System.out.println("Compressed dictionary successfully written");
        }
      break;

      case 3:
        System.out.println("Writing weights file");
        p = Runtime.getRuntime().exec("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Weights file successfully written");
        }
        else {
          System.out.println("Unable to create weights file " + "mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
        }

        p = Runtime.getRuntime().exec("mg_invf_dict -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Inverted dictionary file successfully written");
        }
        else {
          System.out.println("Unable to create inverted dictionary file");
        }
        
        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s1 -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Stemmed index successfully written");
        }
        else {
          System.out.println("Unable to create stemmed index");
        }

        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s2 -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Stemmed index successfully written");
        }
        else {
          System.out.println("Unable to create stemmed index");
        }

        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s3 -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Stemmed index successfully written");
        }
        else {
          System.out.println("Unable to create stemmed index");
        }
      break;
    }
      }
      catch (IOException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      }
      catch (InterruptedException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      }
      return true;
    }

    /**
     *  Do any tidying up
     */
    public void tidyup()
    {
    }
    
    /**
     *  Return the number of passes required for this index.
     */
    public int getNumberOfPasses()
    { return 2 + this.indexes.size() * 2;
    }
}