source: trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java@ 6736

package org.greenstone.gsdl3.gs3build.indexers;

import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;

import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;

public class MGPPIndexer extends AbstractIndexer
{ 
  int          pass;
  String       name;
  boolean      firstDocument;
  String       outputDirectory;
  String       outputStem;
  InputStream    indexerFeedback;
  InputStream  indexerErrors;
  OutputStream indexerTextfeed;
  Process      mgpp_passes;
  static final String documentSeparator = "<Document>";

  public static final String MGPP_INDEX_TYPE = "mgpp";
  
  public MGPPIndexer(String name)
  { this.name = name;
  }

  public String getName()
  { return this.name;
  }

  /**
   *  The output directory should be (collection)/building/text/ for
   *  normal Greenstone builds
   */ 
  public boolean configure(String label, String value)
  { 
    if (label.equals("outputDir")) {
      this.outputDirectory = value;
      this.outputStem = value + "/index";
    }
    this.pass = 0;
    return true;
  }

  public String getIndexType()
  { return MGPP_INDEX_TYPE;
  }

  public boolean addIndex(String name, String level, String field)
  { return true;
  }

  /**
   *  Index a single document; the document interface can be used to extract individual
   *  metadata items etc. as required or desired and index those instead or as well as
   *  the body text of the document.
   */
  public boolean indexDocument(DocumentID docID, DocumentInterface document)
  {
    if (!this.firstDocument)
    { // Send a '<document>' before the document itself
      try {                 
    this.indexerTextfeed.write(documentSeparator.getBytes(), 0, documentSeparator.getBytes().length);
      }
      catch (IOException ex)
      { System.out.println("Bad output on end of document" + ex);
        ex.printStackTrace();
    return false;
      }
    }
    String docText = document.getDocumentText();
    
    byte [] bytes = docText.getBytes();
    int pos = 0, end = bytes.length;
    
    try {
      while (pos < end) {
    this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
    pos = pos + 512;
    
    try {
      while (this.indexerFeedback.available() > 0)
      { byte b[] = new byte[this.indexerFeedback.available()];
        System.out.println("Feedback of " + this.indexerFeedback.available());
        this.indexerFeedback.read(b);
        System.out.println(b);
      }
    }
    catch (IOException ex)
      {
      }


    try {
      while (this.indexerErrors.available() > 0)
      { byte b[] = new byte[this.indexerErrors.available()];
        System.out.println("Feedback of " + this.indexerErrors.available());
        this.indexerErrors.read(b);
        System.out.println(new String(b));
      }
    }
    catch (IOException ex)
      {
      }
      }
    }
    catch (IOException ex)
    { System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
      ex.printStackTrace();
      return false;
    }
    this.firstDocument = false;
    
    try {
      while (this.indexerErrors.available() > 0)
      { char c = (char) this.indexerErrors.read();
        System.out.println(c);
      }
      while (this.indexerFeedback.available() > 0)
      { byte b[] = new byte[this.indexerFeedback.available()];
    System.out.println("Feedback of " + this.indexerFeedback.available());
    this.indexerFeedback.read(b);
      }
    }
    catch (IOException ex)
    {
    }
    return true;
  }

  /**
   *  Initialise the pass: open required files, check status
   */
  public boolean startPass(int passNumber)
  { this.pass = passNumber;
    this.firstDocument = true;
    
    try {               
      switch (this.pass) {
        case 0:
      mgpp_passes = Runtime.getRuntime().exec("mgpp_passes -f " + this.outputStem + " -T1");
    break;
      
        case 1:
      mgpp_passes = Runtime.getRuntime().exec("mgpp_passes -f " + this.outputStem +" -T2");
    break;

        case 2:
      mgpp_passes = Runtime.getRuntime().exec("mgpp_passes -f " + this.outputStem +" -I1");
    break;

        case 3:
      Process p = Runtime.getRuntime().exec("mgpp_perf_hash_build -f " + this.outputStem);
      p.waitFor();

      mgpp_passes = Runtime.getRuntime().exec("mgpp_passes -f " + this.outputStem +" -I2");
    break;
      }
        
      this.indexerFeedback = mgpp_passes.getInputStream();
      this.indexerErrors   = mgpp_passes.getErrorStream();
      this.indexerTextfeed = mgpp_passes.getOutputStream();
    }
    catch (IOException ex)
    { System.out.println(ex);
      ex.printStackTrace();
      return false;
    }   
    catch (InterruptedException ex)
    { System.out.println(ex);
      ex.printStackTrace();
      return false;
    }
    System.out.println("Pass " + this.pass);
    return true;
  }

  /**
   *  Complete a pass - reset file counters, close files, etc.
   */
  public boolean endPass(int passNumber)
  { // TODO: end pass
    Process p;
    
    try {
      this.indexerTextfeed.write((char) 2);
      this.indexerTextfeed.write(4);
      while (this.indexerErrors.available() > 0)
      { char c = (char) this.indexerErrors.read();
    System.out.print(c);
      }
      while (this.indexerFeedback.available() > 0)
      { byte b[] = new byte[this.indexerFeedback.available()];
        System.out.print("Feedback of " + this.indexerFeedback.available());
    this.indexerFeedback.read(b);
      }

      this.indexerTextfeed.close();
      Thread.sleep(1000);
      this.mgpp_passes.waitFor();
    }
    catch (IOException ex)
    { System.out.println(ex);
    }
    catch (InterruptedException ex)
    { System.out.println(ex);
    }
    System.out.println("Completed with " + this.mgpp_passes.exitValue());
    
    try {
      switch (this.pass)
      {                 
        case 0:
      System.out.println("Compressing dictionary");
      p = Runtime.getRuntime().exec("mgpp_compression_dict -f " + this.outputStem + " -S -H -2 -k 5120");
      p.waitFor();
      System.out.println(p.exitValue());
    break;

        case 3:
      p = Runtime.getRuntime().exec("mgpp_weights_build -f " + this.outputStem);
      p.waitFor();
      System.out.println(p.exitValue());

      p = Runtime.getRuntime().exec("mgpp_invf_dict -f " + this.outputStem);
      p.waitFor();
      System.out.println(p.exitValue());
      
      p = Runtime.getRuntime().exec("mgpp_stem_idx -b 4096 -s1 -f " + this.outputStem + " -d " + this.outputDirectory);
      p.waitFor();
      System.out.println(p.exitValue());
      p = Runtime.getRuntime().exec("mgpp_stem_idx -b 4096 -s2 -f " + this.outputStem + " -d " + this.outputDirectory);
      p.waitFor();
      System.out.println(p.exitValue());
      p = Runtime.getRuntime().exec("mgpp_stem_idx -b 4096 -s3 -f " + this.outputStem + " -d " + this.outputDirectory);
      p.waitFor();
      System.out.println(p.exitValue());
      break;
      }
    }
    catch (IOException ex)
    { System.out.println(ex);
      ex.printStackTrace();
      return false;
    }
    catch (InterruptedException ex)
    { System.out.println(ex);
      ex.printStackTrace();
      return false;
    }
    return true;
  }
    
  /**
   *  Do any tidying up
   */
  public void tidyup()
  {
  }
    
  /**
   *  Return the number of passes required for this index.
   */
  public int getNumberOfPasses()
  { return 4;
  }
}