Changeset 9975

Timestamp:

2005-05-26T16:18:44+12:00 (19 years ago)

Author:

kjdon

Message:

tidied up the MGIndexer. no longer based on AbstractIndexer - I'm going to get rid of that eventually

File:

• trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java (modified) (27 diffs)

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java

@@ -22 +22 @@
 import org.greenstone.gsdl3.gs3build.metadata.*;
 import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
 import org.greenstone.gsdl3.util.GSXML;
 import org.greenstone.gsdl3.util.Misc;
 import org.greenstone.gsdl3.util.Processing;
 
-public class MGIndexer extends AbstractIndexer
+// for debug
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGIndexer //extends AbstractIndexer
+    implements IndexerInterface
 { 
     int          pass;
@@ -33 +38 @@
     boolean      firstDocument;
     String       outputDirectory;
-    //   InputStream  indexerFeedback;
-    //   InputStream  indexerErrors;
-    //OutputStream indexerTextfeed;
     StringBuffer indexBuffer;
-    //Process      mg_passes;
     File         textDirectory;
     File         indexDirectory;
@@ -45 +46 @@
     String       overallName; 
     
-    String       currentIndexName;
-    String       currentIndexLevel;
-    String       currentIndexField;
-
+    MGIndex current_index = null;
     MGPassesWrapper mgPasses;
     
     static final char END_OF_DOCUMENT = (char) 2;
-    static final char END_OF_SECTION  = (char) 3; // actually this is end of para for mg
+    //static final char END_OF_SECTION  = (char) 3; // actually this is end of para for mg - don't use this yet
     static final char END_OF_STREAM   = (char) 4;
 
@@ -62 +60 @@
     String name=null;
     String level=null;
-    String field=null;
+    List fields=null;
     boolean error = false;// assume built until we get an error
-
-    public MGIndex(String name, String level, String field)
+    
+    public MGIndex(Element index_element) {
+        
+        this.fields = new ArrayList();
+        this.name = index_element.getAttribute(GSXML.NAME_ATT);
+        if (this.name.equals("")) {
+        // TODO make this dynamic
+        this.name = "xx";
+        }
+        NodeList children = index_element.getChildNodes();
+        for (int c = 0; c < children.getLength(); c ++) {
+        Node child = children.item(c);
+        
+        if (child.getNodeType() == Node.ELEMENT_NODE) {
+            String name = child.getNodeName();
+      
+            if (name.equals(GSXML.LEVEL_ELEM)) {
+            this.level = DOMUtils.getNodeChildText(children.item(c));
+            }
+            else if (name.equals(GSXML.FIELD_ELEM)) {
+            String fieldName = DOMUtils.getNodeChildText(children.item(c));
+            this.fields.add(fieldName);
+            }
+        }
+        }
+    }
+
+    public MGIndex(String name, String level, List fields)
     { 
         this.name  = name;
         this.level = level;
-        this.field = field;
-    }
-    
+        this.fields = fields;
+    }
+    
+    // old gs2 style config - can we get rid of this??
     public MGIndex(String indexLabel)
     { 
         int colonAt = indexLabel.indexOf(':');
         
-        if (colonAt >= 0) {
-        this.field = indexLabel.substring(colonAt+1);
-        this.level = indexLabel.substring(0, colonAt);
-        createIndexName();
-        }
-    }
-
-    public String getLevel()
-    { 
-        return this.level;
-    }
-
-    public String getField()
-    { 
-        return this.field;
-    }
-
-    public String getName()
-    { 
-        if (this.name==null || this.name.equals("")) {
-        createIndexName();
-        }
-        return this.name;
-    }
-
-    public boolean hasError() {
-        return this.error;
-    }
-    public void setError(boolean b) {
-        this.error = b;
-    }
-
-    private void createIndexName() {
+        if (colonAt < 0) {
+        System.err.println("MGIndex(): invalid index specification: "+indexLabel);
+        return;
+        }
+        String field_string = indexLabel.substring(colonAt+1);
+        String [] field_list = field_string.split(",");
+        this.fields = new ArrayList();
+        for (int i=0; i<field_list.length; i++) {
+        this.fields.add(field_list[i]);
+        }
+        this.level = indexLabel.substring(0, colonAt);
+        //createIndexName
         StringBuffer new_name = new StringBuffer();
         new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
-
         int c, w;
         w = 0;
         c = 0;
-        while (c < this.field.length() && w < 2) {
-        char ch = this.field.charAt(c);
-
+        while (c < field_string.length() && w < 2) {
+        char ch = field_string.charAt(c);
+        
         ch = Character.toLowerCase(ch);
         if (Character.isLetter(ch)) {
@@ -130 +133 @@
         this.name = new_name.toString();
     }
+
+    
+    public String getLevel()
+    { 
+        return this.level;
+    }
+
+    public List getFields()
+    { 
+        return this.fields;
+    }
+
+    public String getName()
+    { 
+//      if (this.name==null || this.name.equals("")) {
+//      createIndexName();
+//      }
+        return this.name;
+    }
+
+    public boolean hasError() {
+        return this.error;
+    }
+    public void setError(boolean b) {
+        this.error = b;
+    }
+
+//  private void createIndexName() {
+//      StringBuffer new_name = new StringBuffer();
+//      new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+
+//      int c, w;
+//      w = 0;
+//      c = 0;
+//      String [] fields_concat = this.fields.toArray
+//      while (c < this.field.length() && w < 2) {
+//      char ch = this.field.charAt(c);
+
+//      ch = Character.toLowerCase(ch);
+//      if (Character.isLetter(ch)) {
+//          if (ch != 'a' && ch != 'e' && ch != 'i' &&
+//          ch != 'o' && ch != 'u') {
+//          new_name.append(ch);
+//          w++;
+//          }
+//      }
+//      c ++;
+//      }
+//      this.name = new_name.toString();
+//  }
+ 
+
     } // MGIndex
 
@@ -147 +202 @@
     return this.overallName; 
     }
-
-    //    private String getIndexDirectory(String level, String field)
-    //    { StringBuffer directory = new StringBuffer();
-    //      directory.append(Character.toLowerCase((char) level.charAt(0)));
-
-    //      int c, w;
-    //      w = 0;
-    //      c = 0;
-    //      while (c < field.length() && w < 2) {
-    //        char ch = field.charAt(c);
-
-    //        ch = Character.toLowerCase(ch);
-    //        if (Character.isLetter(ch)) {
-    //      if (ch != 'a' && ch != 'e' && ch != 'i' &&
-    //          ch != 'o' && ch != 'u') {
-    //        directory.append(ch);
-    //        w++;
-    //      }
-    //        }
-    //        c ++;
-    //      }
-    //      return directory.toString();
-    //    }
+    
+    public boolean configure(Node search_node)
+    {
+    NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+    
+    // add a text 'index' - we should be able to turn this off in the config file? actually mg needs a text index
+    ArrayList list = new ArrayList();
+    list.add("text");
+    MGIndex index = new MGIndex("text", "section", list);
+    indexes.add(index);
+    for (int i = 0; i < index_children.getLength(); i ++) {
+        Element index_elem = (Element)index_children.item(i);
+        index = new MGIndex(index_elem);
+        if (index.getName() != null && index.getLevel() != null && index.getFields()!= null) {
+        
+        indexes.add(index);
+        } else {
+        System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+        }
+    }
+    // TODO make sure all index names are unique
+    return true;
+    }
 
     /**
@@ -206 +261 @@
     }
 
-    public boolean addIndex(String name, String level, String field)
-    {
-    MGIndex index = new MGIndex(name, level, field);
-    this.indexes.add(index);
-    return true;
-    }
 
     private Node recurseDOM(DocumentInterface metsDoc, Node node, 
@@ -218 +267 @@
     //String name, String namespace, String field)
     {
+    List fields = current_index.getFields();
     // send out the ctrl-c...if this is
     if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
         // try doing this for all index types
-        if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+        // actually we should only need to do this once ????
+        if (this.pass == 0) {
+        //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
         METSDivision division = (METSDivision) structure;
 
@@ -251 +303 @@
         // by an end of document character.  This ensures that all indexes use the
         // same document numbering...
-        if (this.currentIndexLevel == null ||
-        this.currentIndexLevel.equals(IndexerInterface.DOCUMENT_LEVEL)) {
+        if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
         extraBuffer.append(END_OF_DOCUMENT);
         }
@@ -259 +310 @@
         this.documentSeqNo ++;
         }
-    
+        
         // produce the body here for metadata output of divisions - in the case of
         // text output, that will happen below...
-        if (!this.currentIndexField.equals("text")) {
+        
+        if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+        // if there is only text, don't do this
         METSDescriptive descriptive;
     
@@ -268 +321 @@
 
         String metadataId = division.getDefaultMetadataReference();
-        
+        // are there other metadata refs to get??
         descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
         if (descriptive != null) {
-            List values = descriptive.getMetadata(namespace, this.currentIndexField);
-      
-            if (values != null) {   
-            Iterator valueIter = values.iterator();
-            while (valueIter.hasNext()) {
-                String value = valueIter.next().toString();
-          
-                textBuffer.append(value);
-                if (valueIter.hasNext()) {
-                //textBuffer.append(END_OF_SECTION);
+            for (int i=0; i<fields.size(); i++) {
+            String field = (String)fields.get(i);
+            if (field.equals("text")) {
+                continue;
+            }
+            List values = descriptive.getMetadata(namespace, field);
+            if (values != null) {   
+                Iterator valueIter = values.iterator();
+                while (valueIter.hasNext()) {
+                String value = valueIter.next().toString();
+                textBuffer.append(value);
+                textBuffer.append(" ");
                 }
             }
@@ -287 +342 @@
         }
     }
-
+    
     // go through our children as required...
     Iterator children = structure.getChildIterator();
     Node startNode;
+    boolean index_text = fields.contains("text");
     while (children.hasNext()) {
         AbstractStructure child = (AbstractStructure) children.next();
@@ -301 +357 @@
         startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
         }
-        //Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
-      
-        // while this node isn't the child's start node, produce the HTML node text, if
-        // in text field mode...
-        if (this.currentIndexField.equals("text")) {
+        
+        // while this node isn't the child's start node, produce the 
+        // HTML node text, if in text field mode...
+        if (index_text) {
         while (node != startNode) {
             XPointer.printNode(node, textBuffer, false);
-
-            // print buffer to node
-            node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
-        }
-        }
-      
+            node = XPointer.getNextNode(node);
+        }
+        }
+        
         // recurse to child
         node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
@@ -320 +373 @@
     // close a document - the actual closing \B will be done by the main
     // loop, so only a required \C is printed here...
+    // why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
     if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
-        while (node != null) {
-        if (this.currentIndexField.equals("text")) {
+        if (index_text) {
+        while (node != null) {
             XPointer.printNode(node, textBuffer, false);
-        }
-        node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+            node = XPointer.getNextNode(node);
+        }
         }
       
@@ -354 +408 @@
     public boolean indexDocument(DocumentID docID, DocumentInterface document)
     {
-    if (this.pass == 0) {
-        document.removeAllMetadata("gsdl3", "mgseqno");
-    }
     
     if (!this.firstDocument) {
@@ -366 +417 @@
 
     String docText = null;
-    
-    int startSeqNo = this.sectionSeqNo;
+    // set the mgseqno if first pass
+    if (this.pass == 0) {
+        document.removeAllMetadata("gsdl3", "mgseqno");
+        document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+    }
+
     this.sectionSeqNo ++;
     
+    //long start = System.currentTimeMillis();
     Document domDocument = document.getDOMDocument();
     if (domDocument != null) {
@@ -380 +436 @@
         }
     }
+    //long finish = System.currentTimeMillis();
+    //System.err.println("dom doc = "+ Long.toString(finish-start));
+    //start = System.currentTimeMillis();
     if (docText == null) {
         System.err.println("dom doc or sections was null - asking for doc text");
-        if (this.currentIndexField.equals("text")) {
-        //docText = Character.toString(END_OF_DOCUMENT) + document.getDocumentText(); 
-        docText = document.getDocumentText();
-        }
-        else {
-        StringBuffer textBuffer = new StringBuffer();
-        //textBuffer.append(END_OF_DOCUMENT);
-        List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
-        if (values != null) {
-            Iterator valueIter = values.iterator();
-            while (valueIter.hasNext()) {
-            String value = valueIter.next().toString();
-            
-            textBuffer.append(value);
-            if (valueIter.hasNext()) {
-                //textBuffer.append(END_OF_SECTION);
-                //        sectionSeqNo ++;
+        StringBuffer doc_text_buffer = new StringBuffer();
+        List fields = this.current_index.getFields();
+        for (int i=0; i<fields.size(); i++) {
+        String field = (String)fields.get(i);
+        if (field.equals("text")) {
+            doc_text_buffer.append(document.getDocumentText());
+        }  else {
+            // its a metadata - do namespace properly!!
+            List values = document.getDocumentMetadataItem("gsdl3", field);
+            if (values != null) {
+            Iterator valueIter = values.iterator();
+            while (valueIter.hasNext()) {
+                String value = valueIter.next().toString();
+                doc_text_buffer.append(value);
             }
             }
         }
-        else {
-            textBuffer.append("No data");
-        }
-        docText = textBuffer.toString();
-        }
+        } // for each field
+        docText = doc_text_buffer.toString();
         sectionSeqNo ++;
     }
-    
+    //finish = System.currentTimeMillis();
+    //System.err.println("whole doc = "+ Long.toString(finish-start));
     
     this.indexBuffer.append(docText);
     // remember that we're not on the first document, 
     this.firstDocument = false;
-    // assign the sequence number on the first pass only, and increment the sequence number.
-    if (this.pass == 0) {
-        document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(startSeqNo));
-    }
-    this.documentSeqNo += 1;
+    this.documentSeqNo ++;
 
     return true;
@@ -436 +486 @@
     this.mgPasses = new MGPassesWrapper();
     this.indexBuffer = new StringBuffer();
-    int indexNo = (this.pass - 2) / 2;
-    MGIndex index = null;
-    if (this.pass >= 2) {
-        index = (MGIndex) this.indexes.get(indexNo);
-        if (index.hasError()) {
-        // an error has already occurred for this index, don't continue
-        System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
-        return false;
-        }
-        // attempt to ensure that the text subdirectory exists
-        this.indexDirectory = new File(outputDirectory, index.getName());
-        if (!indexDirectory.exists()) {
-        if (!indexDirectory.mkdir()) {
-            return false;
-        }
-        }
-        else if (!indexDirectory.isDirectory()) {
-        return false;
-        }
-        
-        this.currentIndexLevel = index.getLevel();
-        this.currentIndexField = index.getField();
-        this.currentIndexName = index.getName();
-        
-        if (this.currentIndexLevel == null || this.currentIndexField == null ) {
-        System.out.println("invalid index - level or field was null");
-        return false;
-        }
-        this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
-        if (this.pass % 2 == 1) {
-        this.currentIndexName = null; // why???
-        }
-    }
-    else {
-        
-        this.currentIndexField = "text";
-        this.currentIndexLevel = "section";
-        this.currentIndexName = null;
-    }
-    
-    // get the parameters for this execution of mg_passes
-    mgPasses.setFileName((this.pass < 2 ? this.textDirectory.toString() : this.indexDirectory.toString())+File.separator+ "index");
+    int indexNo = this.pass/2;
+    this.current_index = null;
+    
+    this.current_index = (MGIndex) this.indexes.get(indexNo);
+    if (this.current_index.hasError()) {
+        // an error has already occurred for this index, don't continue
+        System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+        return false;
+    }
+    // attempt to ensure that the text/index subdirectory exists
+    this.indexDirectory = new File(outputDirectory, current_index.getName());
+    if (!indexDirectory.exists()) {
+        if (!indexDirectory.mkdir()) {
+        return false;
+        }
+    }
+    else if (!indexDirectory.isDirectory()) {
+        return false;
+    }
+
+    this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+    if (this.pass == 0) {
+        // first pass, also set up the textStem
+        this.textDirectory = this.indexDirectory;
+        this.textStem = this.indexStem;
+    }
+    mgPasses.setFileName(this.indexStem);
     if (!Misc.isWindows()) {
         mgPasses.setBasePath("/");
@@ -516 +551 @@
     
     mgPasses.init();
-    System.out.println("Pass " + this.pass);
+    System.out.println("Starting Pass " + this.pass);
     return true;
     }
@@ -524 +559 @@
      */
     public boolean endPass(int passNumber) {
-    Process p;
-    
-    int indexNo = (passNumber - 2) / 2;
-    MGIndex index = null;
-    if (passNumber >= 2) {
-        index = (MGIndex) this.indexes.get(indexNo);
-    }
     try {
         this.indexBuffer.append(END_OF_DOCUMENT);
@@ -549 +577 @@
     if (exit_value !=0) {
         //assume something has gone wrong, don't continue
-        if (index != null) {
-        index.setError(true);
-        return false;
-        }
+        current_index.setError(true);
+        return false;
     }
     int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
@@ -559 +585 @@
         osextra = " -d / ";
     }
-
+    
     switch (mgPass) {
         
     case 0:
         System.out.println("Compressing dictionary");
-        exit_value = Processing.runProcess("mg_compression_dict -f " + this.textDirectory.toString()+File.separator+"index" + osextra + " -S -H -2 -k 5120");
+        exit_value = Processing.runProcess("mg_compression_dict -f " + this.textStem + osextra + " -S -H -2 -k 5120");
         if (exit_value == 0) {
         System.out.println("Compressed dictionary successfully written");
         } else {
         System.err.println("Error from mg_compression_dict: " + exit_value);
-        index.setError(true);
+        current_index.setError(true);
         
         return false;
@@ -577 +603 @@
     case 2:
         System.out.println("Creating perfect hash");
-        exit_value = Processing.runProcess("mg_perf_hash_build -f " + this.indexDirectory.toString()+File.separator+ "index"+osextra);
+        exit_value = Processing.runProcess("mg_perf_hash_build -f " + this.indexStem+osextra);
         if (exit_value ==0) {
         System.out.println("Perfect hashes completed");
         } else {
         System.err.println("Unable to build the perfect hash");
-        index.setError(true);
+        current_index.setError(true);
         return false;
         }
@@ -594 +620 @@
         } else {
         System.err.println("Unable to create weights file");
-        index.setError(true);
+        current_index.setError(true);
         return false;
         }
         
         System.out.println("Creating inverted dictionary");
-        exit_value = Processing.runProcess("mg_invf_dict -f " + this.indexDirectory.toString()+File.separator+"index" + osextra);
+        exit_value = Processing.runProcess("mg_invf_dict -f " + this.indexStem + osextra);
         if (exit_value ==0) {
         System.out.println("Inverted dictionary file successfully written");
         } else {
         System.out.println("Unable to create inverted dictionary file");
-        index.setError(true);
+        current_index.setError(true);
         return false;
         }
         
         System.out.println("Creating Stem indexes");
-        exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s1 -f " + this.indexDirectory.toString()+File.separator+"index"+osextra);
+        exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s1 -f " + this.indexStem+osextra);
         if (exit_value == 0) {
         System.out.println("Stemmed index 1 successfully written");
         } else {
         System.out.println("Unable to create stemmed index 1");
-        index.setError(true);
-        return false;
-        }
-        
-        exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s2 -f " + this.indexDirectory.toString()+File.separator+"index"+osextra);
+        current_index.setError(true);
+        return false;
+        }
+        
+        exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s2 -f " + this.indexStem+osextra);
         if (exit_value == 0) {
         System.out.println("Stemmed index 2 successfully written");
         } else {
         System.out.println("Unable to create stemmed index 2");
-        index.setError(true);
-        return false;
-        }
-        exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s3 -f " + this.indexDirectory.toString()+File.separator+"index"+osextra);
+        current_index.setError(true);
+        return false;
+        }
+        exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s3 -f " + this.indexStem+osextra);
         if (exit_value == 0) {
         System.out.println("Stemmed index 3 successfully written");
         } else {
         System.out.println("Unable to create stemmed index 3");
-        index.setError(true);
+        current_index.setError(true);
         return false;
         }
@@ -654 +680 @@
     public int getNumberOfPasses()
     { 
-    return 2 + this.indexes.size() * 2;
+    //return 2 + this.indexes.size() * 2;
+    return this.indexes.size()*2;
     }
 
@@ -665 +692 @@
     boolean found_index = false;
     String def_index = ""; // the default index will just be the first one created for now.
-    for (int i=0; i<this.indexes.size(); i++) {
+    for (int i=1; i<this.indexes.size(); i++) {
+        // start at 1: 0 will be the text index
         MGIndex index = (MGIndex)this.indexes.get(i);
         if (!index.hasError()) {
@@ -688 +716 @@
     base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
     Element index_stem = doc.createElement("indexStem");
-    index_stem.setAttribute(GSXML.NAME_ATT, "index");
+    index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
     
     Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);