Changeset 7449

Timestamp:

2004-05-26T16:03:44+12:00 (20 years ago)

Author:

kjdon

Message:

now uses jni wrapper to mg passes instead of piping via stdin/out. also changed the path args to the auxiliary programs so that they all look for files in the same place on windows

File:

• trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java (modified) (26 diffs)

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java

@@ -9 +9 @@
 import java.io.OutputStream;
 import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
 
 import org.w3c.dom.*;
+
+import org.greenstone.mg.*;
 
 import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
@@ -18 +22 @@
 import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
 import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.Misc;
 
 public class MGIndexer extends AbstractIndexer
@@ -26 +31 @@
   boolean      firstDocument;
   String       outputDirectory;
-  InputStream  indexerFeedback;
-  InputStream  indexerErrors;
-  OutputStream indexerTextfeed;
-  Process      mg_passes;
+//   InputStream  indexerFeedback;
+//   InputStream  indexerErrors;
+    //OutputStream indexerTextfeed;
+    StringBuffer indexBuffer;
+    //Process      mg_passes;
   File         textDirectory;
   File         indexDirectory;
@@ -41 +47 @@
     String       currentIndexField;
 
-
+    MGPassesWrapper mgPasses;
+    
   static final char END_OF_DOCUMENT = (char) 2;
-  static final char END_OF_SECTION  = (char) 3;
+    static final char END_OF_SECTION  = (char) 3; // actually this is end of para for mg
   static final char END_OF_STREAM   = (char) 4;
 
-  public static final String MG_INDEX_TYPE = "mg";
+    public static final String MG_INDEX_TYPE = "mg";
     public static final String INDEX_FILE_STEM = "index";
+    
   class MGIndex
   { String name=null;
     String level=null;
     String field=null;
-      boolean error = false;
+      boolean error = false;// assume built until we get an error
 
     public MGIndex(String name, String level, String field)
@@ -58 +66 @@
       this.level = level;
       this.field = field;
-      //this.error = false; // assume built until we get an error
     }
 
@@ -69 +76 @@
     createIndexName();
       }
-      //this.name = null;
-      //this.error = false;
-    }
+   }
 
     public String getLevel()
@@ -124 +129 @@
   { this.indexes = new ArrayList();
     this.overallName = name;
+    
   }
 
@@ -228 +234 @@
     
     // append an 'end of section' marker
-    textBuffer.append(END_OF_SECTION);
+    //textBuffer.append(END_OF_SECTION);
     this.sectionSeqNo ++;
     
@@ -264 +270 @@
           textBuffer.append(value);
           if (valueIter.hasNext()) {
-          textBuffer.append(END_OF_SECTION);
+          //textBuffer.append(END_OF_SECTION);
           }
         }
@@ -305 +311 @@
     node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
       }
-      /*
-      textBuffer.append(END_OF_SECTION);
+      
+      //textBuffer.append(END_OF_SECTION);
       this.sectionSeqNo ++;
-      */
+      
     }
     return node;
@@ -329 +335 @@
    *  the body text of the document.
    */
-  public boolean indexDocument(DocumentID docID, DocumentInterface document)
-  {
-    if (this.pass == 0) {
-      document.removeAllMetadata("gsdl3", "mgseqno");
-    }
-
-    if (!this.firstDocument)
-    { // Send a 'CTRL-B' before the document itself
-      try {
-    this.indexerTextfeed.write(END_OF_DOCUMENT);
-      }
-      catch (IOException ex)
-      { System.out.println("Bad output on end of document" + ex);
-    ex.printStackTrace();
-    return false;
-      }
-    }
-
-    String docText = null;
-
-    int startSeqNo = this.sectionSeqNo;
-    this.sectionSeqNo ++;
-
-    Document domDocument = document.getDOMDocument();
-    if (domDocument != null) {
-      METSStructure sections = document.getDocumentStructure().getStructure("Section");
-      if (sections != null) {
-    docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
-    //  System.out.println(docText);
-      }
-    }
-    if (docText == null) {
-      if (this.currentIndexField.equals("text")) {
-      docText = Character.toString(END_OF_DOCUMENT) + Character.toString(END_OF_SECTION) + 
-      document.getDocumentText();
-      }
-      else {
-    StringBuffer textBuffer = new StringBuffer();
-    textBuffer.append(END_OF_DOCUMENT);
-    textBuffer.append(END_OF_SECTION);
-    List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
-    if (values != null) {
-      Iterator valueIter = values.iterator();
-      while (valueIter.hasNext()) {
-        String value = valueIter.next().toString();
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+    if (this.pass == 0) {
+        document.removeAllMetadata("gsdl3", "mgseqno");
+    }
+    
+    if (!this.firstDocument) {
+        // Send a 'CTRL-B' before the document itself
+        // try {
+        //this.indexerTextfeed.write(END_OF_DOCUMENT);
+        this.indexBuffer.append(END_OF_DOCUMENT);
+        mgPasses.processDocument(indexBuffer.toString());
+        this.indexBuffer.delete(0, this.indexBuffer.length());
         
-        textBuffer.append(value);
-        if (valueIter.hasNext()) {
-        textBuffer.append(END_OF_SECTION);
-          //          sectionSeqNo ++;
-        }
-      }
-    }
-    else {
-      textBuffer.append("No data");
-    }
-    docText = textBuffer.toString();
-      }
-      sectionSeqNo ++;
-    }
-
-    /*    if (this.pass == 0) {
-      System.err.println(docText);
-    }
-    */
-
-    byte [] bytes = docText.getBytes();
-    int pos = 0, end = bytes.length;
-
-    try {
-      while (pos < end) {
-    this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
+    }
+        //   }
+        //  catch (IOException ex)
+        //  { System.out.println("Bad output on end of document" + ex);
+      //    ex.printStackTrace();
+      //    return false;
+      //  }
+    
+
+    String docText = null;
+    
+    int startSeqNo = this.sectionSeqNo;
+    this.sectionSeqNo ++;
+    
+    Document domDocument = document.getDOMDocument();
+    if (domDocument != null) {
+        System.err.println("dom doc is not null");
+        METSStructure sections = document.getDocumentStructure().getStructure("Section");
+        if (sections != null) {
+        System.err.println("sections are not null");
+        docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+        //  System.out.println(docText);
+        }
+    }
+    if (docText == null) {
+        System.err.println("doc text is null");
+        if (this.currentIndexField.equals("text")) {
+        docText = Character.toString(END_OF_DOCUMENT) /*+ Character.toString(END_OF_SECTION)*/ + document.getDocumentText();
+        System.err.println("prepending EOD to doctext");
+    
+        }
+        else {
+        StringBuffer textBuffer = new StringBuffer();
+        textBuffer.append(END_OF_DOCUMENT);
+        System.err.println("* appending EOD to text");
+    
+        //textBuffer.append(END_OF_SECTION);
+        List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
+        if (values != null) {
+            Iterator valueIter = values.iterator();
+            while (valueIter.hasNext()) {
+            String value = valueIter.next().toString();
+            
+            textBuffer.append(value);
+            if (valueIter.hasNext()) {
+                //textBuffer.append(END_OF_SECTION);
+                //        sectionSeqNo ++;
+            }
+            }
+        }
+        else {
+            textBuffer.append("No data");
+        }
+        docText = textBuffer.toString();
+        }
+        sectionSeqNo ++;
+    }
+    
+    
+    this.indexBuffer.append(docText);
+    //byte [] bytes = docText.getBytes();
+    //int pos = 0, end = bytes.length;
+    
+    /*
+      try {
+      while (pos < end) {
+      //this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
+      this.indexBuffer.append((char [])bytes, pos, (end - pos > 512 ? 512 : end - pos));
     pos = pos + 512;
       
@@ -434 +450 @@
       return false;
     }
-
+    */
     // remember that we're not on the first document, 
     this.firstDocument = false;
@@ -445 +461 @@
     this.documentSeqNo += 1;
 
-    try {
-      while (this.indexerErrors.available() > 0)
-      { char c = (char) this.indexerErrors.read();
-        System.out.println(c);
-      }
-      while (this.indexerFeedback.available() > 0)
-      { byte b[] = new byte[this.indexerFeedback.available()];
-        System.out.println("Feedback of " + this.indexerFeedback.available());
-    this.indexerFeedback.read(b);
-      }
-    }
-    catch (IOException ex)
-    {
-    }
+    //    try {
+//       while (this.indexerErrors.available() > 0)
+//       { char c = (char) this.indexerErrors.read();
+//         System.out.println(c);
+//       }
+//       while (this.indexerFeedback.available() > 0)
+//       { byte b[] = new byte[this.indexerFeedback.available()];
+//         System.out.println("Feedback of " + this.indexerFeedback.available());
+//  this.indexerFeedback.read(b);
+//       }
+//    }
+//     catch (IOException ex)
+//     {
+//     }
     return true;
   }
@@ -467 +483 @@
     public boolean startPass(int passNumber)
     {   
+    
       this.pass = passNumber;
       this.firstDocument = true;
@@ -472 +489 @@
       this.sectionSeqNo  = 1;
 
+      this.mgPasses = new MGPassesWrapper();
+      this.indexBuffer = new StringBuffer();
       int indexNo = (this.pass - 2) / 2;
       MGIndex index = null;
@@ -517 +536 @@
       
       // get the parameters for this execution of mg_passes
-      String pathParams = "-f index -d " + (this.pass < 2 ? this.textDirectory.toString() : this.indexDirectory.toString());
-
+      //String pathParams = "-f index -d " + (this.pass < 2 ? this.textDirectory.toString() : this.indexDirectory.toString());
+      mgPasses.setFileName((this.pass < 2 ? this.textDirectory.toString() : this.indexDirectory.toString())+File.separator+ "index");
+      if (!Misc.isWindows()) {
+      mgPasses.setBasePath("/");
+      }
       int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
       
-      try {
+      // try {
+      // TODO add the other options to mg passes
     switch (mgPass) {
-      case 0:
-        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -T1");
+    case 0:
+          //mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -T1");
+        mgPasses.addPass('T', '1');
+        
+        
         break;
 
       case 1:
-        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -T2");
+          //mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -T2");
+          mgPasses.addPass('T', '2');
         break;
         
       case 2:
-        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -2 -m 32 -s 0 -G -t 10 -N1");
+          //mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams + " -b 100000 -2 -m 32 -s 0 -G -t 10 -N1");
+        mgPasses.addPass('N', '1');
+        mgPasses.setInvfLevel('2');
+        mgPasses.setStemOptions("0",0);
         break;
         
       case 3:
-        mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -2 -c 3 -G -t 10 -N2");
+          //mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -2 -c 3 -G -t 10 -N2");
+        mgPasses.addPass('N', '2');
+        mgPasses.setInvfLevel('2');
         break;
     }
         
-    this.indexerFeedback = mg_passes.getInputStream();
-    this.indexerErrors   = mg_passes.getErrorStream();
-    this.indexerTextfeed = mg_passes.getOutputStream();
-      }
-      catch (IOException ex)
-      { System.out.println(ex);
-        ex.printStackTrace();
-    index.setError(true);
-    return false;
-      } 
+    mgPasses.init();
+    //  this.indexerFeedback = mg_passes.getInputStream();
+    //this.indexerErrors   = mg_passes.getErrorStream();
+    //this.indexerTextfeed = mg_passes.getOutputStream();
+    // }
+    //      catch (IOException ex)
+    // { System.out.println(ex);
+    // ex.printStackTrace();
+    //  index.setError(true);
+    //  return false;
+    // }    
       //        catch (InterruptedException ex)
 //        { System.out.println(ex);
@@ -560 +593 @@
     }
   
-
+    public void printProcessOutput(Process p) 
+    throws IOException {
+    BufferedReader error_stream = new BufferedReader(new InputStreamReader( p.getErrorStream(), "UTF-8" ));
+    BufferedReader output_stream = new BufferedReader(new InputStreamReader( p.getInputStream(), "UTF-8" ));
+    while (output_stream.ready()) {
+        System.err.println("out> "+output_stream.readLine());
+    }
+    while (error_stream.ready()) {
+        System.err.println("err> "+error_stream.readLine());
+    }
+    
+    }
     /**
      *  Complete a pass - reset file counters, close files, etc.
@@ -573 +617 @@
     }
     try {
-    this.indexerTextfeed.write(END_OF_DOCUMENT);
-    this.indexerTextfeed.write(END_OF_STREAM);
-    while (this.indexerErrors.available() > 0)
-    { char c = (char) this.indexerErrors.read();
-      System.out.print(c);
-    }
-    while (this.indexerFeedback.available() > 0)
-    { byte b[] = new byte[this.indexerFeedback.available()];
-      System.out.print("Feedback of " + this.indexerFeedback.available());
-      this.indexerFeedback.read(b);
-    }
-
-    this.indexerTextfeed.close();
+    //this.indexerTextfeed.write(END_OF_DOCUMENT);
+    //this.indexerTextfeed.write(END_OF_STREAM);
+    this.indexBuffer.append(END_OF_DOCUMENT);
+    mgPasses.processDocument(indexBuffer.toString());
+    this.indexBuffer.delete(0, this.indexBuffer.length());
+//  while (this.indexerErrors.available() > 0)
+//  { char c = (char) this.indexerErrors.read();
+//    System.out.print(c);
+//  }
+//  while (this.indexerFeedback.available() > 0)
+//  { byte b[] = new byte[this.indexerFeedback.available()];
+//    System.out.print("Feedback of " + this.indexerFeedback.available());
+//    this.indexerFeedback.read(b);
+//  }
+
+    //this.indexerTextfeed.close();
     Thread.sleep(1000);
-    this.mg_passes.waitFor();
-    }
-    catch (IOException ex)
-      { System.out.println(ex);
-      }
+    //this.mg_passes.waitFor();
+    }
+//     catch (IOException ex)
+//       { System.out.println(ex);
+//       }
       catch (InterruptedException ex)
       { System.out.println(ex);
       }
-    int exitValue = this.mg_passes.exitValue();
+    //    int exitValue = this.mg_passes.exitValue();
+    mgPasses.finish();
+    try {
+    Thread.sleep(1000);
+    } catch (Exception e) {}
+    int exitValue = 0;
     System.out.println("Pass " + this.pass + " completed with " + exitValue);
       if (exitValue !=0) {
@@ -605 +657 @@
       }
       int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+      String osextra = "";
+      if (!Misc.isWindows()) {
+      osextra = " -d / ";
+      }
       try {
     switch (mgPass)
@@ -610 +666 @@
       case 0:
         System.out.println("Compressing dictionary");
-        p = Runtime.getRuntime().exec("mg_compression_dict -f index -d " + this.textDirectory.toString() + " -S -H -2 -k 5120");
+        p = Runtime.getRuntime().exec("mg_compression_dict -f " + this.textDirectory.toString()+File.separator+"index" + osextra + " -S -H -2 -k 5120");
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() != 0) {
           System.out.println("Error from mg_compression_dict: " + p.exitValue());
+          index.setError(true);
           
           return false;
@@ -624 +682 @@
     case 2:
         System.out.println("Creating perfect hash");
-        p = Runtime.getRuntime().exec("mg_perf_hash_build -f index -d " + this.indexDirectory.toString());
+        p = Runtime.getRuntime().exec("mg_perf_hash_build -f " + this.indexDirectory.toString()+File.separator+ "index"+osextra);
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() == 0) {
           System.out.println("Perfect hashes completed");
@@ -637 +696 @@
       case 3:
         System.out.println("Writing weights file");
-        p = Runtime.getRuntime().exec("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
+        p = Runtime.getRuntime().exec("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + osextra);
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() == 0) {
           System.out.println("Weights file successfully written");
         }
         else {
-          System.out.println("Unable to create weights file " + "mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
+          System.out.println("Unable to create weights file");
           index.setError(true);
           return false;
@@ -649 +709 @@
         }
 
-        p = Runtime.getRuntime().exec("mg_invf_dict -f index -d " + this.indexDirectory.toString());
+        p = Runtime.getRuntime().exec("mg_invf_dict -f " + this.indexDirectory.toString()+File.separator+"index" + osextra);
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() == 0) {
           System.out.println("Inverted dictionary file successfully written");
@@ -661 +722 @@
         }
         
-        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s1 -f index -d " + this.indexDirectory.toString());
+        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s1 -f " + this.indexDirectory.toString()+File.separator+"index"+osextra);
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() == 0) {
           System.out.println("Stemmed index 1 successfully written");
@@ -673 +735 @@
         }
 
-        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s2 -f index -d " + this.indexDirectory.toString());
+        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s2 -f " + this.indexDirectory.toString()+File.separator+"index"+osextra);
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() == 0) {
           System.out.println("Stemmed index 2 successfully written");
@@ -684 +747 @@
         }
 
-        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s3 -f index -d " + this.indexDirectory.toString());
+        p = Runtime.getRuntime().exec("mg_stem_idx -b 4096 -s3 -f " + this.indexDirectory.toString()+File.separator+"index"+osextra);
         p.waitFor();
+        printProcessOutput(p);
         if (p.exitValue() == 0) {
           System.out.println("Stemmed index 3 successfully written");