Changeset 6898

Timestamp:

2004-02-26T09:54:54+13:00 (20 years ago)

Author:

kjdon

Message:

added in a lot more checks for failed build commands, now uses teh index name as the directory name, remembers which indexes were successfully built, and creates some service descriptions for the buildconfig file

File:

• trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java (modified) (31 diffs)

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java

@@ -17 +17 @@
 import org.greenstone.gsdl3.gs3build.metadata.*;
 import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.util.GSXML;
 
 public class MGIndexer extends AbstractIndexer
@@ -44 +45 @@
 
   public static final String MG_INDEX_TYPE = "mg";
-
+    public static final String INDEX_FILE_STEM = "index";
   class MGIndex
-  { String name;
-    String level;
-    String field;
+  { String name=null;
+    String level=null;
+    String field=null;
+      boolean error = false;
 
     public MGIndex(String name, String level, String field)
@@ -54 +56 @@
       this.level = level;
       this.field = field;
+      //this.error = false; // assume built until we get an error
     }
 
@@ -62 +65 @@
       { this.field = indexLabel.substring(colonAt+1);
         this.level = indexLabel.substring(0, colonAt);
-      }
-      this.name = null;
+    createIndexName();
+      }
+      //this.name = null;
+      //this.error = false;
     }
 
@@ -75 +80 @@
 
     public String getName()
-    { return this.name;
-    }
+    { 
+    if (this.name==null || this.name.equals("")) {
+        createIndexName();
+    }
+    return this.name;
+    }
+
+      public boolean hasError() {
+      return this.error;
+      }
+      public void setError(boolean b) {
+      this.error = b;
+      }
+
+      private void createIndexName() {
+      StringBuffer new_name = new StringBuffer();
+      new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+
+      int c, w;
+      w = 0;
+      c = 0;
+      while (c < this.field.length() && w < 2) {
+          char ch = this.field.charAt(c);
+
+          ch = Character.toLowerCase(ch);
+          if (Character.isLetter(ch)) {
+          if (ch != 'a' && ch != 'e' && ch != 'i' &&
+              ch != 'o' && ch != 'u') {
+              new_name.append(ch);
+              w++;
+          }
+          }
+          c ++;
+      }
+      this.name = new_name.toString();
+
+      }
   }
 
@@ -92 +132 @@
   }
 
-  private String getIndexDirectory(String level, String field)
-  { StringBuffer directory = new StringBuffer();
-    directory.append(Character.toLowerCase((char) level.charAt(0)));
-
-    int c, w;
-    w = 0;
-    c = 0;
-    while (c < field.length() && w < 2) {
-      char ch = field.charAt(c);
-
-      ch = Character.toLowerCase(ch);
-      if (Character.isLetter(ch)) {
-    if (ch != 'a' && ch != 'e' && ch != 'i' &&
-        ch != 'o' && ch != 'u') {
-      directory.append(ch);
-      w++;
-    }
-      }
-      c ++;
-    }
-    return directory.toString();
-  }
+//    private String getIndexDirectory(String level, String field)
+//    { StringBuffer directory = new StringBuffer();
+//      directory.append(Character.toLowerCase((char) level.charAt(0)));
+
+//      int c, w;
+//      w = 0;
+//      c = 0;
+//      while (c < field.length() && w < 2) {
+//        char ch = field.charAt(c);
+
+//        ch = Character.toLowerCase(ch);
+//        if (Character.isLetter(ch)) {
+//      if (ch != 'a' && ch != 'e' && ch != 'i' &&
+//          ch != 'o' && ch != 'u') {
+//        directory.append(ch);
+//        w++;
+//      }
+//        }
+//        c ++;
+//      }
+//      return directory.toString();
+//    }
 
   /**
@@ -126 +166 @@
     if (label.equals(IndexerManager.outputDir)) {
       this.outputDirectory = value;
-      this.textStem   = value + "/text/index";
       this.pass = 0;
     
@@ -139 +178 @@
     return false;
       }
+      this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
 
       // Sign to the user which mg directory is being used...
@@ -159 +199 @@
   private Node recurseDOM(DocumentInterface metsDoc, Node node, 
               AbstractStructure structure, StringBuffer textBuffer,
-              StringBuffer extraBuffer, String indexName,
-              String namespace, String field)
+              StringBuffer extraBuffer, String namespace) 
+              //String indexName, String namespace, String field)
   {
     // send out the ctrl-c...if this is
     if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
-      if ((indexName != null) && indexName.startsWith("s")) {
+    if ((this.indexName != null) && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //indexName.startsWith("s")) {
     METSDivision division = (METSDivision) structure;
 
@@ -179 +219 @@
     }
 
-    descriptive.addMetadata("gsdl3", "mgseqno", indexName + "." + Integer.toString(this.sectionSeqNo));
+    descriptive.addMetadata("gsdl3", "mgseqno", this.indexName + "." + Integer.toString(this.sectionSeqNo));
     metsDoc.setModified(true);
     //  System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
@@ -212 +252 @@
     descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
     if (descriptive != null) {
-      List values = descriptive.getMetadata(namespace, field);
+      List values = descriptive.getMetadata(namespace, this.field);
       
       if (values != null) { 
@@ -240 +280 @@
       // while this node isn't the child's start node, produce the HTML node text, if
       // in text field mode...
-      if (field.equals("text")) {
+      if (this.field.equals("text")) {
     while (node != startNode) {
       XPointer.printNode(node, textBuffer, false);
 
       // print buffer to node
-      node = XPointer.getNextNode(node, (field.equals("text") ? textBuffer : null));
+      node = XPointer.getNextNode(node, (this.field.equals("text") ? textBuffer : null));
     }
       }
       
       // recurse to child
-      node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, indexName, namespace, field);
+      node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // indexName, namespace, field);
     }
 
@@ -257 +297 @@
     if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
       while (node != null) {
-    if (field.equals("text")) {
+    if (this.field.equals("text")) {
       XPointer.printNode(node, textBuffer, false);
     }
-    node = XPointer.getNextNode(node, (field.equals("text") ? textBuffer : null));
+    node = XPointer.getNextNode(node, (this.field.equals("text") ? textBuffer : null));
       }
       /*
@@ -270 +310 @@
   }
 
-  private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, 
-                String indexName, String namespace, String field)
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+    //  String indexName, String namespace, String field)
   { StringBuffer extraBuffer = new StringBuffer();
     Node node = document.getDocumentElement();
     StringBuffer textBuffer = new StringBuffer();
 
-    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, indexName, namespace, field);
+    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //indexName, namespace, field);
     textBuffer.append(extraBuffer.toString());
     return textBuffer.toString();
@@ -313 +353 @@
       METSStructure sections = document.getDocumentStructure().getStructure("Section");
       if (sections != null) {
-    docText = this.prepareDOM(document, domDocument, sections, this.indexName, "gsdl3", this.field);
+    docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.indexName, "gsdl3", this.field);
     //  System.out.println(docText);
       }
@@ -429 +469 @@
 
       int indexNo = (this.pass - 2) / 2;
+      MGIndex index = null;
       if (this.pass >= 2) {
-    MGIndex index = (MGIndex) this.indexes.get(indexNo);
-      
-    // attempt to ensure that the text subdirectory exists
-    this.indexDirectory = new File(outputDirectory, this.getIndexDirectory(index.getLevel(), index.getField()));
-    if (!indexDirectory.exists()) {
-      if (!indexDirectory.mkdir()) {
-        return false;
-      }
-    }
-    else if (!indexDirectory.isDirectory()) {
-      return false;
-    }
+      index = (MGIndex) this.indexes.get(indexNo);
+      if (index.hasError()) {
+          // an error has already occurred for this index, don't continue
+          System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+          return false;
+      }
+      // attempt to ensure that the text subdirectory exists
+      //this.indexDirectory = new File(outputDirectory, this.getIndexDirectory(index.getLevel(), index.getField()));
+      this.indexDirectory = new File(outputDirectory, index.getName());
+      if (!indexDirectory.exists()) {
+          if (!indexDirectory.mkdir()) {
+          return false;
+          }
+      }
+      else if (!indexDirectory.isDirectory()) {
+          return false;
+      }
 
     this.level = index.getLevel();
     this.field = index.getField();
     this.indexName = index.getName();
-    if (this.indexName == null || this.indexName.length() == 0) {
-      this.indexName = this.getIndexDirectory(index.getLevel(), index.getField());
-    }
-    this.indexStem = this.outputDirectory + File.separatorChar + 
-      this.indexName + File.separatorChar + "index"; // TODO: modify for index
+    if (this.level == null || this.field == null ) {
+        System.out.println("invalid index - level or field was null");
+        return false;
+    }
+    //if (this.indexName == null || this.indexName.length() == 0) {
+        //this.indexName = this.getIndexDirectory(index.getLevel(), index.getField());
+    //}
+    this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
     if (this.pass % 2 == 1) {
-      this.indexName = null;
+        this.indexName = null; // why???
     }
       }
@@ -468 +517 @@
 
       int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
-
+      
       try {
     switch (mgPass) {
@@ -484 +533 @@
         
       case 3:
-        Process p = Runtime.getRuntime().exec("mg_perf_hash_build -f index -d " + this.indexDirectory.toString());
-        p.waitFor();
-        if (p.exitValue() == 0) {
-          System.out.println("Perfect hashes completed");
-        }
-        
         mg_passes = Runtime.getRuntime().exec("mg_passes " + pathParams +" -b 100000 -2 -c 3 -G -t 10 -N2");
         break;
@@ -501 +544 @@
       { System.out.println(ex);
         ex.printStackTrace();
+    index.setError(true);
     return false;
       } 
-      catch (InterruptedException ex)
-      { System.out.println(ex);
-        ex.printStackTrace();
-    return false;
-      }
+//        catch (InterruptedException ex)
+//        { System.out.println(ex);
+//          ex.printStackTrace();
+//      index.setError(true);
+//      return false;
+//        }
       System.out.println("Pass " + this.pass);
       return true;
@@ -517 +562 @@
     public boolean endPass(int passNumber)
     { Process p;
-        
-      try {
+    
+    int indexNo = (passNumber - 2) / 2;
+    MGIndex index = null;
+    if (passNumber >= 2) {
+    index = (MGIndex) this.indexes.get(indexNo);
+    }
+    try {
     this.indexerTextfeed.write(END_OF_DOCUMENT);
     this.indexerTextfeed.write(END_OF_STREAM);
@@ -541 +591 @@
       { System.out.println(ex);
       }
-      System.out.println("Pass " + this.pass + " completed with " + this.mg_passes.exitValue());
-
+    int exitValue = this.mg_passes.exitValue();
+      System.out.println("Pass " + this.pass + " completed with " + exitValue);
+      if (exitValue !=0) {
+      //assume something has gone wrong, don't continue
+      if (index != null) {
+          index.setError(true);
+          return false;
+      }
+      }
       int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
-
       try {
     switch (mgPass)
@@ -554 +610 @@
         if (p.exitValue() != 0) {
           System.out.println("Error from mg_compression_dict: " + p.exitValue());
+          
+          return false;
         }
         else {
@@ -559 +617 @@
         }
       break;
+
+    case 2:
+        System.out.println("Creating perfect hash");
+        p = Runtime.getRuntime().exec("mg_perf_hash_build -f index -d " + this.indexDirectory.toString());
+        p.waitFor();
+        if (p.exitValue() == 0) {
+          System.out.println("Perfect hashes completed");
+        } else {
+        System.out.println("Unable to build the perfect hash");
+        index.setError(true);
+        return false;
+        }
+        break;
 
       case 3:
@@ -569 +640 @@
         else {
           System.out.println("Unable to create weights file " + "mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
+          index.setError(true);
+          return false;
+
         }
 
@@ -578 +652 @@
         else {
           System.out.println("Unable to create inverted dictionary file");
+          index.setError(true);
+          return false;
+
         }
         
@@ -583 +660 @@
         p.waitFor();
         if (p.exitValue() == 0) {
-          System.out.println("Stemmed index successfully written");
+          System.out.println("Stemmed index 1 successfully written");
         }
         else {
-          System.out.println("Unable to create stemmed index");
+          System.out.println("Unable to create stemmed index 1");
+          index.setError(true);
+          return false;
+
         }
 
@@ -592 +672 @@
         p.waitFor();
         if (p.exitValue() == 0) {
-          System.out.println("Stemmed index successfully written");
+          System.out.println("Stemmed index 2 successfully written");
         }
         else {
-          System.out.println("Unable to create stemmed index");
+          System.out.println("Unable to create stemmed index 2");
+          index.setError(true);
+          return false;
         }
 
@@ -601 +683 @@
         p.waitFor();
         if (p.exitValue() == 0) {
-          System.out.println("Stemmed index successfully written");
+          System.out.println("Stemmed index 3 successfully written");
         }
         else {
-          System.out.println("Unable to create stemmed index");
+          System.out.println("Unable to create stemmed index 3");
+          index.setError(true);
+          return false;
         }
       break;
@@ -612 +696 @@
       { System.out.println(ex);
         ex.printStackTrace();
+    index.setError(true);
     return false;
       }
@@ -617 +702 @@
       { System.out.println(ex);
         ex.printStackTrace();
+    index.setError(true);
     return false;
       }
@@ -635 +721 @@
     { return 2 + this.indexes.size() * 2;
     }
+
+    public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list) {
+    System.out.println("adding service description, MGIndexer");
+    Document doc = service_rack_list.getOwnerDocument();
+
+    // generate the list of indexes
+    Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+    boolean found_index = false;
+    String def_index = ""; // the default index will just be the first one created for now.
+    for (int i=0; i<this.indexes.size(); i++) {
+        MGIndex index = (MGIndex)this.indexes.get(i);
+        if (!index.hasError()) {
+        Element e = doc.createElement(GSXML.INDEX_ELEM);
+        e.setAttribute(GSXML.NAME_ATT, index.getName());
+        index_list.appendChild(e);
+        if (found_index == false) {
+            // this is the first index
+            found_index = true;
+            def_index = index.getName();
+        }
+        }
+    }
+    
+    if (!found_index) {
+        // no indexes were able to be created, so we can't use them or the text
+        return false;
+    }
+    Element default_index = doc.createElement("defaultIndex");
+    default_index.setAttribute(GSXML.NAME_ATT, def_index);
+
+    Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    service_rack_list.appendChild(search_service_elem);
+    service_rack_list.appendChild(retrieve_service_elem);
+
+    search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGSearch");
+    
+    search_service_elem.appendChild(index_list);
+    search_service_elem.appendChild(default_index);
+    retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGRetrieve");
+    retrieve_service_elem.appendChild(default_index.cloneNode(true));
+    return true;
+    }
+
 }