source: trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java@ 6898

package org.greenstone.gsdl3.gs3build.indexers;

import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;

import org.w3c.dom.*;

import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
import org.greenstone.gsdl3.gs3build.metadata.*;
import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
import org.greenstone.gsdl3.util.GSXML;

public class MGIndexer extends AbstractIndexer
{ 
  int          pass;
  int          documentSeqNo;
  int          sectionSeqNo;
  boolean      firstDocument;
  String       outputDirectory;
  InputStream  indexerFeedback;
  InputStream  indexerErrors;
  OutputStream indexerTextfeed;
  Process      mg_passes;
  File         textDirectory;
  File         indexDirectory;
  String       indexStem;
  String       textStem;
  List         indexes;
  String       indexName;
  String       level;
  String       field;
  String       name;

  static final char END_OF_DOCUMENT = (char) 2;
  static final char END_OF_SECTION  = (char) 3;
  static final char END_OF_STREAM   = (char) 4;

  public static final String MG_INDEX_TYPE = "mg";
    public static final String INDEX_FILE_STEM = "index";
  class MGIndex
  { String name=null;
    String level=null;
    String field=null;
      boolean error = false;

    public MGIndex(String name, String level, String field)
    { this.name  = name;
      this.level = level;
      this.field = field;
      //this.error = false; // assume built until we get an error
    }

    public MGIndex(String indexLabel)
    { int colonAt = indexLabel.indexOf(':');
    
      if (colonAt >= 0)
      { this.field = indexLabel.substring(colonAt+1);
        this.level = indexLabel.substring(0, colonAt);
    createIndexName();
      }
      //this.name = null;
      //this.error = false;
    }

    public String getLevel()
    { return this.level;
    }

    public String getField()
    { return this.field;
    }

    public String getName()
    { 
    if (this.name==null || this.name.equals("")) {
        createIndexName();
    }
    return this.name;
    }

      public boolean hasError() {
      return this.error;
      }
      public void setError(boolean b) {
      this.error = b;
      }

      private void createIndexName() {
      StringBuffer new_name = new StringBuffer();
      new_name.append(Character.toLowerCase((char) this.level.charAt(0)));

      int c, w;
      w = 0;
      c = 0;
      while (c < this.field.length() && w < 2) {
          char ch = this.field.charAt(c);

          ch = Character.toLowerCase(ch);
          if (Character.isLetter(ch)) {
          if (ch != 'a' && ch != 'e' && ch != 'i' &&
              ch != 'o' && ch != 'u') {
              new_name.append(ch);
              w++;
          }
          }
          c ++;
      }
      this.name = new_name.toString();

      }
  }

  public MGIndexer(String name)
  { this.indexes = new ArrayList();
    this.name    = name;
  }

  public String getIndexType()
  { return MG_INDEX_TYPE;
  } 

  public String getName()
  { return this.name;
  }

//    private String getIndexDirectory(String level, String field)
//    { StringBuffer directory = new StringBuffer();
//      directory.append(Character.toLowerCase((char) level.charAt(0)));

//      int c, w;
//      w = 0;
//      c = 0;
//      while (c < field.length() && w < 2) {
//        char ch = field.charAt(c);

//        ch = Character.toLowerCase(ch);
//        if (Character.isLetter(ch)) {
//      if (ch != 'a' && ch != 'e' && ch != 'i' &&
//          ch != 'o' && ch != 'u') {
//        directory.append(ch);
//        w++;
//      }
//        }
//        c ++;
//      }
//      return directory.toString();
//    }

  /**
   *  The output directory should be (collection)/building/text/ for
    *  normal Greenstone builds.
   *
   *  @param <code>String</code> the label to configure
   *  @param <code>String</code> the value...
   */ 
  public boolean configure(String label, String value)
  { 
    if (label.equals(IndexerManager.outputDir)) {
      this.outputDirectory = value;
      this.pass = 0;
    
      // attempt to ensure that the text subdirectory exists
      this.textDirectory = new File(outputDirectory, "text");
      if (!textDirectory.exists()) {
    if (!textDirectory.mkdir()) {
      return false;
    }
      }
      else if (!textDirectory.isDirectory()) {
    return false;
      }
      this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;

      // Sign to the user which mg directory is being used...
      System.out.println("Output MG directory is " + this.textStem);
    }
    else if (label.equals(IndexerInterface.GS2_INDEX_LABEL)) {
      this.indexes.add(new MGIndex(value));
    }

    return true;
  }

  public boolean addIndex(String name, String level, String field)
  {
    MGIndex index = new MGIndex(name, level, field);
    this.indexes.add(index);
    return true;
  }

  private Node recurseDOM(DocumentInterface metsDoc, Node node, 
              AbstractStructure structure, StringBuffer textBuffer,
              StringBuffer extraBuffer, String namespace) 
              //String indexName, String namespace, String field)
  {
    // send out the ctrl-c...if this is
    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
    if ((this.indexName != null) && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //indexName.startsWith("s")) {
    METSDivision division = (METSDivision) structure;

    // get the division metadata block
    METSDescriptive descriptive;
    String metadataId = division.getDefaultMetadataReference();
    if (metadataId == null) {
      descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
      division.addMetadataReference(descriptive.getID());
    }
    else {
      // Get the descriptive item...
      descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
    }

    descriptive.addMetadata("gsdl3", "mgseqno", this.indexName + "." + Integer.toString(this.sectionSeqNo));
    metsDoc.setModified(true);
    //  System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
      }
      
      // append an 'end of section' marker
      textBuffer.append(END_OF_SECTION);
      this.sectionSeqNo ++;

      // for document-level indexes, always append an 'end of document' tag at the
      // end of the document for each section.  Otherwise, each section is followed
      // by an end of document character.  This ensures that all indexes use the
      // same document numbering...
      if (this.level == null ||
      this.level.equals(IndexerInterface.DOCUMENT_LEVEL)) {
    extraBuffer.append(END_OF_DOCUMENT);
      }
      else {
    textBuffer.append(END_OF_DOCUMENT);
    this.documentSeqNo ++;
      }

      // produce the body here for metadata output of divisions - in the case of
      // text output, that will happen below...
      if (!this.field.equals("text")) 
      { METSDescriptive descriptive;
    
    METSDivision division = (METSDivision) structure;

    String metadataId = division.getDefaultMetadataReference();
        
    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
    if (descriptive != null) {
      List values = descriptive.getMetadata(namespace, this.field);
      
      if (values != null) { 
        Iterator valueIter = values.iterator();
        while (valueIter.hasNext()) {
          String value = valueIter.next().toString();
          
          textBuffer.append(value);
          if (valueIter.hasNext()) {
        textBuffer.append(END_OF_SECTION);
          }
        }
      }
    }
      }
    }

    // go through our children as required...
    Iterator children = structure.getChildIterator();
    while (children.hasNext()) {
      AbstractStructure child = (AbstractStructure) children.next();
      
      // get xpointer for child 
      // get start position node
      Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
      
      // while this node isn't the child's start node, produce the HTML node text, if
      // in text field mode...
      if (this.field.equals("text")) {
    while (node != startNode) {
      XPointer.printNode(node, textBuffer, false);

      // print buffer to node
      node = XPointer.getNextNode(node, (this.field.equals("text") ? textBuffer : null));
    }
      }
      
      // recurse to child
      node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // indexName, namespace, field);
    }

    // close a document - the actual closing \B will be done by the main
    // loop, so only a required \C is printed here...
    if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
      while (node != null) {
    if (this.field.equals("text")) {
      XPointer.printNode(node, textBuffer, false);
    }
    node = XPointer.getNextNode(node, (this.field.equals("text") ? textBuffer : null));
      }
      /*
      textBuffer.append(END_OF_SECTION);
      this.sectionSeqNo ++;
      */
    }
    return node;
  }

    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
    //  String indexName, String namespace, String field)
  { StringBuffer extraBuffer = new StringBuffer();
    Node node = document.getDocumentElement();
    StringBuffer textBuffer = new StringBuffer();

    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //indexName, namespace, field);
    textBuffer.append(extraBuffer.toString());
    return textBuffer.toString();
  }

  /**
   *  Index a single document; the document interface can be used to extract individual
   *  metadata items etc. as required or desired and index those instead or as well as
   *  the body text of the document.
   */
  public boolean indexDocument(DocumentID docID, DocumentInterface document)
  {
    if (this.pass == 0) {
      document.removeAllMetadata("gsdl3", "mgseqno");
    }

    if (!this.firstDocument)
    { // Send a 'CTRL-B' before the document itself
      try {
    this.indexerTextfeed.write(END_OF_DOCUMENT);
      }
      catch (IOException ex)
      { System.out.println("Bad output on end of document" + ex);
    ex.printStackTrace();
    return false;
      }
    }

    String docText = null;

    int startSeqNo = this.sectionSeqNo;
    this.sectionSeqNo ++;

    Document domDocument = document.getDOMDocument();
    if (domDocument != null) {
      METSStructure sections = document.getDocumentStructure().getStructure("Section");
      if (sections != null) {
    docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.indexName, "gsdl3", this.field);
    //  System.out.println(docText);
      }
    }
    if (docText == null) {
      if (this.field.equals("text")) {
    docText = Character.toString(END_OF_DOCUMENT) + Character.toString(END_OF_SECTION) + 
      document.getDocumentText();
      }
      else {
    StringBuffer textBuffer = new StringBuffer();
    textBuffer.append(END_OF_DOCUMENT);
    textBuffer.append(END_OF_SECTION);
    List values = document.getDocumentMetadataItem("gsdl3", this.field);
    if (values != null) {
      Iterator valueIter = values.iterator();
      while (valueIter.hasNext()) {
        String value = valueIter.next().toString();
        
        textBuffer.append(value);
        if (valueIter.hasNext()) {
          textBuffer.append(END_OF_SECTION);
          //          sectionSeqNo ++;
        }
      }
    }
    else {
      textBuffer.append("No data");
    }
    docText = textBuffer.toString();
      }
      sectionSeqNo ++;
    }

    /*    if (this.pass == 0) {
      System.err.println(docText);
    }
    */

    byte [] bytes = docText.getBytes();
    int pos = 0, end = bytes.length;

    try {
      while (pos < end) {
    this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
    pos = pos + 512;
      
    try {
      while (this.indexerFeedback.available() > 0)
          { byte b[] = new byte[this.indexerFeedback.available()];
        System.out.println("Feedback of " + this.indexerFeedback.available());
        this.indexerFeedback.read(b);
        System.out.println(b);
      }
    }
    catch (IOException ex)
        { System.out.println(ex);
    }


    try {
      while (this.indexerErrors.available() > 0)
      { byte b[] = new byte[this.indexerErrors.available()];
        System.out.println("Feedback of " + this.indexerErrors.available());
        this.indexerErrors.read(b);
        System.out.println(new String(b));
      }
    }
    catch (IOException ex)
        { System.out.println(ex);
    }
      }
    }
    catch (IOException ex)
    { System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
      ex.printStackTrace();
      return false;
    }

    // remember that we're not on the first document, assign the sequence number
    // on the first pass only, and increment the sequence number.
    this.firstDocument = false;
    if (this.pass == 0) {
      document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(startSeqNo));
      //System.out.println("Assigning " + startSeqNo + " to " + document.getID());
    }
    this.documentSeqNo += 1;

    try {
      while (this.indexerErrors.available() > 0)
      { char c = (char) this.indexerErrors.read();
        System.out.println(c);
      }
      while (this.indexerFeedback.available() > 0)
      { byte b[] = new byte[this.indexerFeedback.available()];
        System.out.println("Feedback of " + this.indexerFeedback.available());
    this.indexerFeedback.read(b);
      }
    }
    catch (IOException ex)
    {
    }
    return true;
  }

    /**
     *  Initialise the pass: open required files, check status
     */
    public boolean startPass(int passNumber)
    {   
      this.pass = passNumber;
      this.firstDocument = true;
      this.documentSeqNo = 1;
      this.sectionSeqNo  = 1;

      int indexNo = (this.pass - 2) / 2;
      MGIndex index = null;
      if (this.pass >= 2) {
      index = (MGIndex) this.indexes.get(indexNo);
      if (index.hasError()) {
          // an error has already occurred for this index, don't continue
          System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
          return false;
      }
      // attempt to ensure that the text subdirectory exists
      //this.indexDirectory = new File(outputDirectory, this.getIndexDirectory(index.getLevel(), index.getField()));
      this.indexDirectory = new File(outputDirectory, index.getName());
      if (!indexDirectory.exists()) {
          if (!indexDirectory.mkdir()) {
          return false;
          }
      }
      else if (!indexDirectory.isDirectory()) {
          return false;
      }

    this.level = index.getLevel();
    this.field = index.getField();
    this.indexName = index.getName();
    if (this.level == null || this.field == null ) {
        System.out.println("invalid index - level or field was null");
        return false;
    }
    //if (this.indexName == null || this.indexName.length() == 0) {
        //this.indexName = this.getIndexDirectory(index.getLevel(), index.getField());
    //}
    this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
    if (this.pass % 2 == 1) {
        this.indexName = null; // why???
    }
      }
      else {
    this.field = "text";
    this.level = "section";
    this.indexName = null;
      }
      System.out.println("level is " + this.level);
      System.out.println("field is " + this.field);
      System.out.println("index name is " + this.indexName);
      
      // get the parameters for this execution of mg_passes
      String pathParams = "-f index -d " + (this.pass < 2 ? this.textDirectory.toString() : this.indexDirectory.toString());

      int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
      
      try {
    switch (mgPass) {
      case 0:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -T1");
        break;

      case 1:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -T2");
        break;
        
      case 2:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -2 -m 32 -s 0 -G -t 10 -N1");
        break;
        
      case 3:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -2 -c 3 -G -t 10 -N2");
        break;
    }
        
    this.indexerFeedback = mg_passes.getInputStream();
    this.indexerErrors   = mg_passes.getErrorStream();
    this.indexerTextfeed = mg_passes.getOutputStream();
      }
      catch (IOException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    index.setError(true);
    return false;
      } 
//        catch (InterruptedException ex)
//        { System.out.println(ex);
//          ex.printStackTrace();
//      index.setError(true);
//      return false;
//        }
      System.out.println("Pass " + this.pass);
      return true;
    }

    /**
     *  Complete a pass - reset file counters, close files, etc.
     */
    public boolean endPass(int passNumber)
    { Process p;
    
    int indexNo = (passNumber - 2) / 2;
    MGIndex index = null;
    if (passNumber >= 2) {
    index = (MGIndex) this.indexes.get(indexNo);
    }
    try {
    this.indexerTextfeed.write(END_OF_DOCUMENT);
    this.indexerTextfeed.write(END_OF_STREAM);
    while (this.indexerErrors.available() > 0)
    { char c = (char) this.indexerErrors.read();
      System.out.print(c);
    }
    while (this.indexerFeedback.available() > 0)
    { byte b[] = new byte[this.indexerFeedback.available()];
      System.out.print("Feedback of " + this.indexerFeedback.available());
      this.indexerFeedback.read(b);
    }

    this.indexerTextfeed.close();
    Thread.sleep(1000);
    this.mg_passes.waitFor();
      }
      catch (IOException ex)
      { System.out.println(ex);
      }
      catch (InterruptedException ex)
      { System.out.println(ex);
      }
    int exitValue = this.mg_passes.exitValue();
      System.out.println("Pass " + this.pass + " completed with " + exitValue);
      if (exitValue !=0) {
      //assume something has gone wrong, don't continue
      if (index != null) {
          index.setError(true);
          return false;
      }
      }
      int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
      try {
    switch (mgPass)
    { 
      case 0:
        System.out.println("Compressing dictionary");
        p = Runtime.getRuntime().exec("mg_compression_dict -f index -d " + this.textDirectory.toString() + " -S -H -2 -k 5120");
        p.waitFor();
        if (p.exitValue() != 0) {
          System.out.println("Error from mg_compression_dict: " + p.exitValue());
          
          return false;
        }
        else {
          System.out.println("Compressed dictionary successfully written");
        }
      break;

    case 2:
        System.out.println("Creating perfect hash");
        p = Runtime.getRuntime().exec("mg_perf_hash_build -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Perfect hashes completed");
        } else {
        System.out.println("Unable to build the perfect hash");
        index.setError(true);
        return false;
        }
        break;

      case 3:
        System.out.println("Writing weights file");
        p = Runtime.getRuntime().exec("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Weights file successfully written");
        }
        else {
          System.out.println("Unable to create weights file " + "mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
          index.setError(true);
          return false;

        }

        p = Runtime.getRuntime().exec("mg_invf_dict -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Inverted dictionary file successfully written");
        }
        else {
          System.out.println("Unable to create inverted dictionary file");
          index.setError(true);
          return false;

        }
        
        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s1 -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Stemmed index 1 successfully written");
        }
        else {
          System.out.println("Unable to create stemmed index 1");
          index.setError(true);
          return false;

        }

        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s2 -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Stemmed index 2 successfully written");
        }
        else {
          System.out.println("Unable to create stemmed index 2");
          index.setError(true);
          return false;
        }

        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s3 -f index -d " + this.indexDirectory.toString());
        p.waitFor();
        if (p.exitValue() == 0) {
          System.out.println("Stemmed index 3 successfully written");
        }
        else {
          System.out.println("Unable to create stemmed index 3");
          index.setError(true);
          return false;
        }
      break;
    }
      }
      catch (IOException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    index.setError(true);
    return false;
      }
      catch (InterruptedException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    index.setError(true);
    return false;
      }
      return true;
    }

    /**
     *  Do any tidying up
     */
    public void tidyup()
    {
    }
    
    /**
     *  Return the number of passes required for this index.
     */
    public int getNumberOfPasses()
    { return 2 + this.indexes.size() * 2;
    }

    public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list) {
    System.out.println("adding service description, MGIndexer");
    Document doc = service_rack_list.getOwnerDocument();

    // generate the list of indexes
    Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
    boolean found_index = false;
    String def_index = ""; // the default index will just be the first one created for now.
    for (int i=0; i<this.indexes.size(); i++) {
        MGIndex index = (MGIndex)this.indexes.get(i);
        if (!index.hasError()) {
        Element e = doc.createElement(GSXML.INDEX_ELEM);
        e.setAttribute(GSXML.NAME_ATT, index.getName());
        index_list.appendChild(e);
        if (found_index == false) {
            // this is the first index
            found_index = true;
            def_index = index.getName();
        }
        }
    }
    
    if (!found_index) {
        // no indexes were able to be created, so we can't use them or the text
        return false;
    }
    Element default_index = doc.createElement("defaultIndex");
    default_index.setAttribute(GSXML.NAME_ATT, def_index);

    Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
    Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
    service_rack_list.appendChild(search_service_elem);
    service_rack_list.appendChild(retrieve_service_elem);

    search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGSearch");
    
    search_service_elem.appendChild(index_list);
    search_service_elem.appendChild(default_index);
    retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGRetrieve");
    retrieve_service_elem.appendChild(default_index.cloneNode(true));
    return true;
    }

}