source: trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java@ 5952

package org.greenstone.gsdl3.gs3build.indexers;

import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;

public class MGIndexer implements IndexerInterface
{ 
  int          pass;
  int          documentSeqNo;
  boolean      firstDocument;
  String       outputDirectory;
  String       outputStem;
  InputStream    indexerFeedback;
  InputStream  indexerErrors;
  OutputStream indexerTextfeed;
  Process      mg_passes;
  File         textDirectory;
  File         dtxDirectory;
  String       textStem;

  public MGIndexer()
  { 
  }

  /**
   *  The output directory should be (collection)/building/text/ for
   *  normal Greenstone builds
   */ 
  public boolean configure(String outputDirectory)
  { this.outputDirectory = outputDirectory;
    this.outputStem = outputDirectory + "/dtx/index"; // TODO: modify for index
    this.textStem   = outputDirectory + "/text/index";
    this.pass = 0;
    
    // attempt to ensure that the text subdirectory exists
    this.textDirectory = new File(outputDirectory, "text");
    if (!textDirectory.exists()) {
      if (!textDirectory.mkdir()) {
    return false;
      }
    }
    else if (!textDirectory.isDirectory()) {
      return false;
    }
    
    // attempt to ensure that the text subdirectory exists
    this.dtxDirectory = new File(outputDirectory, "dtx");
    if (!dtxDirectory.exists()) {
      if (!dtxDirectory.mkdir()) {
    return false;
      }
    }
    else if (!dtxDirectory.isDirectory()) {
      return false;
    }
    
    System.out.println("Output MG directory is " + this.textStem);
    return true;
  }

    /**
     *  Index a single document; the document interface can be used to extract individual
     *  metadata items etc. as required or desired and index those instead or as well as
     *  the body text of the document.
     */
    public boolean indexDocument(DocumentID docID, DocumentInterface document)
    {
      if (!this.firstDocument)
      { // Send a 'CTRL-B' before the document itself
    try {
      this.indexerTextfeed.write(2);
    }
    catch (IOException ex)
    { System.out.println("Bad output on end of document" + ex);
      ex.printStackTrace();
      return false;
    }
      }
      String docText = document.getDocumentText();

      byte [] bytes = docText.getBytes();
      int pos = 0, end = bytes.length;

      try {
    while (pos < end) {
      this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
      pos = pos + 512;

      try {
        while (this.indexerFeedback.available() > 0)
        { byte b[] = new byte[this.indexerFeedback.available()];
          System.out.println("Feedback of " + this.indexerFeedback.available());
          this.indexerFeedback.read(b);
          System.out.println(b);
        }
      }
      catch (IOException ex)
      {
      }


      try {
        while (this.indexerErrors.available() > 0)
        { byte b[] = new byte[this.indexerErrors.available()];
          System.out.println("Feedback of " + this.indexerErrors.available());
          this.indexerErrors.read(b);
          System.out.println(new String(b));
        }
      }
      catch (IOException ex)
      {
      }
    }
      }
      catch (IOException ex)
      { System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
        ex.printStackTrace();
    return false;
      }

      // remember that we're not on the first document, assign the sequence number
      // on the first pass only, and increment the sequence number.
      this.firstDocument = false;
      if (this.pass == 0) {
    document.addDocumentMetadata("gsdl3", "mgseqno", Integer.toString(this.documentSeqNo));
      }
      this.documentSeqNo += 1;
      
      try {
    while (this.indexerErrors.available() > 0)
    { char c = (char) this.indexerErrors.read();
      System.out.println(c);
    }
    while (this.indexerFeedback.available() > 0)
    { byte b[] = new byte[this.indexerFeedback.available()];
      System.out.println("Feedback of " + this.indexerFeedback.available());
      this.indexerFeedback.read(b);
    }
      }
      catch (IOException ex)
      {
      }
      return true;
    }

    /**
     *  Initialise the pass: open required files, check status
     */
    public boolean startPass(int passNumber)
    {   
      this.pass = passNumber;
      this.firstDocument = true;
      this.documentSeqNo = 0;

      String pathParams = "-f index -d " + (pass < 2 ? this.textDirectory.toString() : this.dtxDirectory.toString());

      try {
    switch (this.pass) {
      case 0:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -T1");
        break;

      case 1:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -T2 -M 10");
        break;
        
      case 2:
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -2 -m 32 -s 0 -G -t 10 -N1");
        break;
        
      case 3:
        Process p = Runtime.getRuntime().exec("mg_perf_hash_build -f index -d " + this.dtxDirectory.toString());
        p.waitFor();
        System.out.println(p.exitValue());
        
        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -2 -c 3 -G -t 10 -N2");
        break;
    }
        
    this.indexerFeedback = mg_passes.getInputStream();
    this.indexerErrors   = mg_passes.getErrorStream();
    this.indexerTextfeed = mg_passes.getOutputStream();
      }
      catch (IOException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      } 
      catch (InterruptedException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      }
      System.out.println("Pass " + this.pass);
      return true;
    }

    /**
     *  Complete a pass - reset file counters, close files, etc.
     */
    public boolean endPass(int passNumber)
    { Process p;
        
      try {
    this.indexerTextfeed.write((char) 2);
    this.indexerTextfeed.write(4);
    while (this.indexerErrors.available() > 0)
    { char c = (char) this.indexerErrors.read();
      System.out.print(c);
    }
    while (this.indexerFeedback.available() > 0)
    { byte b[] = new byte[this.indexerFeedback.available()];
      System.out.print("Feedback of " + this.indexerFeedback.available());
      this.indexerFeedback.read(b);
    }

    this.indexerTextfeed.close();
    Thread.sleep(1000);
    this.mg_passes.waitFor();
      }
      catch (IOException ex)
      { System.out.println(ex);
      }
      catch (InterruptedException ex)
      { System.out.println(ex);
      }
      System.out.println("Completed with " + this.mg_passes.exitValue());

      try {
    switch (this.pass)
    {                   
      case 0:
        System.out.println("Compressing dictionary");
        p = Runtime.getRuntime().exec("mg_compression_dict -f index -d " + this.textDirectory.toString() + " -S -H -2 -k 5120");
        p.waitFor();
        if (p.exitValue() != 0) {
          System.out.println("Error from mg_compression_dict: " + p.exitValue());
        }
      break;

      case 3:
        p = Runtime.getRuntime().exec("mg_weights_build -f " + this.outputStem + " -t " + this.textStem + " -d /");
        p.waitFor();
        System.out.println(p.exitValue());

        p = Runtime.getRuntime().exec("mg_invf_dict -f index -d " + this.dtxDirectory.toString());
        p.waitFor();
        System.out.println(p.exitValue());
        
        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s1 -f index -d " + this.dtxDirectory.toString());
        p.waitFor();
        System.out.println(p.exitValue());
        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s2 -f index -d " + this.dtxDirectory.toString());
        p.waitFor();
        System.out.println(p.exitValue());
        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s3 -f index -d " + this.dtxDirectory.toString());
        p.waitFor();
        System.out.println(p.exitValue());
      break;
    }
      }
      catch (IOException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      }
      catch (InterruptedException ex)
      { System.out.println(ex);
        ex.printStackTrace();
    return false;
      }
      return true;
    }

    /**
     *  Do any tidying up
     */
    public void tidyup()
    {
    }
    
    /**
     *  Return the number of passes required for this index.
     */
    public int getNumberOfPasses()
    { return 4;
    }
}