Changeset 6376

Timestamp:

2004-01-09T12:51:47+13:00 (20 years ago)

Author:

cs025

Message:

Fixed some problems in MG indexing; also changed use of mgseqno to a
by-index basis.

Location:

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers

Files:

• IndexerManager.java (modified) (1 diff)
• MGIndexer.java (modified) (15 diffs)

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java

@@ -53 +53 @@
       if (document.isIndexed()) {
         if (!this.indexers[i].indexDocument(document.getID(), document)) {
-          System.out.println("Ending document");
+          System.out.println("Ending document " + document.getID());
         }
         
         // note any changes made to this document...
-        if (document.isModified() || true) {
+        if (document.isModified()) {
           this.documents.modifiedDocument(document);
+          //          System.out.println("Writing document "+document.getID());
         }
+        /**
+        if (p == 0) {
+          System.out.println("Writing document "+document.getID());
+        }
+        */
       }
     }

trunk/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java

@@ -34 +34 @@
   String       textStem;
   List         indexes;
+  String       indexName;
   String       level;
   String       field;
+
+  static final char END_OF_DOCUMENT = (char) 2;
+  static final char END_OF_SECTION  = (char) 3;
+  static final char END_OF_STREAM   = (char) 4;
 
   class MGIndex
@@ -134 +139 @@
 
   private Node recurseDOM(DocumentInterface metsDoc, Node node, 
-              AbstractStructure structure, StringBuffer buffer,
+              AbstractStructure structure, StringBuffer textBuffer,
+              StringBuffer extraBuffer, String indexName,
               String namespace, String field)
   {
     // send out the ctrl-c...if this is
     if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
-      if (this.pass == 0) {
+      if ((indexName != null) && indexName.startsWith("s")) {
     METSDivision division = (METSDivision) structure;
 
@@ -154 +160 @@
     }
 
-    descriptive.addMetadata("gsdl3", "mgseqno", Integer.toString(this.sectionSeqNo));
-      }
-
-      buffer.append((char) 3);
-      if (this.level != null && 
-      this.level.equals(IndexerInterface.SECTION_LEVEL)) {
-    buffer.append((char) 2);
+    descriptive.addMetadata("gsdl3", "mgseqno", indexName + "." + Integer.toString(this.documentSeqNo));
+    metsDoc.setModified(true);
+    //  System.out.println("Assigning " + this.documentSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+      }
+      
+      // append an 'end of section' marker
+      textBuffer.append(END_OF_SECTION);
+
+      // for document-level indexes, always append an 'end of document' tag at the
+      // end of the document for each section.  Otherwise, each section is followed
+      // by an end of document character.  This ensures that all indexes use the
+      // same document numbering...
+      if (this.level == null ||
+      this.level.equals(IndexerInterface.DOCUMENT_LEVEL)) {
+    // extraBuffer.append(END_OF_DOCUMENT);
+      }
+      else {
+    textBuffer.append(END_OF_DOCUMENT);
+    this.documentSeqNo ++;
       }
       this.sectionSeqNo ++;
-    }
-
-    // go through our children if required...
+
+      // produce the body here for metadata output of divisions - in the case of
+      // text output, that will happen below...
+      if (!this.field.equals("text")) 
+      { METSDescriptive descriptive;
+    
+    METSDivision division = (METSDivision) structure;
+
+    String metadataId = division.getDefaultMetadataReference();
+        
+    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+    if (descriptive != null) {
+      List values = descriptive.getMetadata(namespace, field);
+      
+      if (values != null) { 
+        Iterator valueIter = values.iterator();
+        while (valueIter.hasNext()) {
+          String value = valueIter.next().toString();
+          
+          textBuffer.append(value);
+          if (valueIter.hasNext()) {
+        textBuffer.append(END_OF_SECTION);
+          }
+        }
+      }
+    }
+      }
+    }
+
+    // go through our children as required...
     Iterator children = structure.getChildIterator();
     while (children.hasNext()) {
@@ -174 +219 @@
       Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
       
-      // while this node isn't the child's start node, produce the node text
+      // while this node isn't the child's start node, produce the HTML node text, if
+      // in text field mode...
       if (field.equals("text")) {
     while (node != startNode) {
-      XPointer.printNode(node, buffer, false);
+      XPointer.printNode(node, textBuffer, false);
 
       // print buffer to node
-      node = XPointer.getNextNode(node, (field.equals("text") ? buffer : null));
+      node = XPointer.getNextNode(node, (field.equals("text") ? textBuffer : null));
     }
       }
       
       // recurse to child
-      this.recurseDOM(metsDoc, node, child, buffer, namespace, field);
+      this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, indexName, namespace, field);
     }
 
@@ -193 +239 @@
       while (node != null) {
     if (field.equals("text")) {
-      XPointer.printNode(node, buffer, false);
-    }
-    else {
-      METSDescriptive descriptive;
-
-      if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
-        METSDivision division = (METSDivision) structure;
-
-        String metadataId = division.getDefaultMetadataReference();
-        
-        descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
-        if (descriptive != null) {
-          List values = descriptive.getMetadata(namespace, field);
-          
-          Iterator valueIter = values.iterator();
-          while (valueIter.hasNext()) {
-        String value = valueIter.next().toString();
-        
-        buffer.append(value);
-        if (valueIter.hasNext()) {
-          buffer.append((char) 3);
-        }
-          }
-        }
-      }
-    }
-    node = XPointer.getNextNode(node, (field.equals("text") ? buffer : null));
-      }
-      buffer.append((char) 3);
+      XPointer.printNode(node, textBuffer, false);
+    }
+    node = XPointer.getNextNode(node, (field.equals("text") ? textBuffer : null));
+      }
+      /*
+      textBuffer.append(END_OF_SECTION);
       this.sectionSeqNo ++;
+      */
     }
     return node;
   }
 
-  private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace, String field)
-  { Node node = document.getDocumentElement();
+  private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, 
+                String indexName, String namespace, String field)
+  { StringBuffer extraBuffer = new StringBuffer();
+    Node node = document.getDocumentElement();
     StringBuffer textBuffer = new StringBuffer();
-    
-    this.recurseDOM(metsDoc, node, structure, textBuffer, namespace, field);
+
+    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, indexName, namespace, field);
+    textBuffer.append(extraBuffer.toString());
     return textBuffer.toString();
   }
@@ -255 +282 @@
     String docText = null;
 
+    int startSeqNo = this.sectionSeqNo;
+
     Document domDocument = document.getDOMDocument();
-    if (domDocument != null) {    
+    if (domDocument != null) {
       METSStructure sections = document.getDocumentStructure().getStructure("Section");
       if (sections != null) {
-    docText = this.prepareDOM(document, domDocument, sections, "gsdl3", this.field);
+    docText = this.prepareDOM(document, domDocument, sections, this.indexName, "gsdl3", this.field);
     //  System.out.println(docText);
       }
     }
     if (docText == null) {
-      docText = document.getDocumentText();
+      if (this.field.equals("text")) {
+    docText = Character.toString(END_OF_DOCUMENT) + Character.toString(END_OF_SECTION) + 
+      document.getDocumentText();
+      }
+      else {
+    StringBuffer textBuffer = new StringBuffer();
+    textBuffer.append(END_OF_DOCUMENT);
+    textBuffer.append(END_OF_SECTION);
+    List values = document.getDocumentMetadataItem("gsdl3", this.field);
+    if (values != null) {
+      Iterator valueIter = values.iterator();
+      while (valueIter.hasNext()) {
+        String value = valueIter.next().toString();
+        
+        textBuffer.append(value);
+        if (valueIter.hasNext()) {
+          textBuffer.append(END_OF_SECTION);
+          sectionSeqNo ++;
+        }
+      }
+    }
+    else {
+      textBuffer.append("No data");
+    }
+    docText = textBuffer.toString();
+      }
+      sectionSeqNo ++;
     }
 
@@ -284 +339 @@
     }
     catch (IOException ex)
-    {
+        { System.out.println(ex);
     }
 
@@ -297 +352 @@
     }
     catch (IOException ex)
-    {
+        { System.out.println(ex);
     }
       }
@@ -311 +366 @@
     this.firstDocument = false;
     if (this.pass == 0) {
-      document.addDocumentMetadata("gsdl3", "mgseqno", Integer.toString(this.documentSeqNo));
+      document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(this.documentSeqNo));
+      //      System.out.println("Assigning " + startSeqNo + " to " + document.getID());
     }
     this.documentSeqNo += 1;
-      
+
     try {
       while (this.indexerErrors.available() > 0)
@@ -343 +399 @@
 
       int indexNo = (this.pass - 2) / 2;
-      if (indexNo >= 0) {
+      if (this.pass >= 2) {
     MGIndex index = (MGIndex) this.indexes.get(indexNo);
       
@@ -357 +413 @@
     }
 
-    this.indexStem = this.outputDirectory + File.separatorChar + 
-      this.getIndexDirectory(index.getLevel(), index.getField()) +
-      File.separatorChar + "index"; // TODO: modify for index
     this.level = index.getLevel();
     this.field = index.getField();
+    this.indexName = this.getIndexDirectory(index.getLevel(), index.getField());
+    this.indexStem = this.outputDirectory + File.separatorChar + 
+      this.indexName + File.separatorChar + "index"; // TODO: modify for index
+    if (this.pass % 2 == 1) {
+      this.indexName = null;
+    }
       }
       else {
     this.field = "text";
-      }
+    this.level = "document";
+    this.indexName = null;
+      }
+      System.out.println("level is " + this.level);
+      System.out.println("field is " + this.field);
+      System.out.println("index name is " + this.indexName);
       
       // get the parameters for this execution of mg_passes
@@ -422 +486 @@
         
       try {
-    this.indexerTextfeed.write((char) 2);
-    this.indexerTextfeed.write(4);
+    this.indexerTextfeed.write(END_OF_DOCUMENT);
+    this.indexerTextfeed.write(END_OF_STREAM);
     while (this.indexerErrors.available() > 0)
     { char c = (char) this.indexerErrors.read();
@@ -444 +508 @@
       { System.out.println(ex);
       }
-      System.out.println("Completed with " + this.mg_passes.exitValue());
+      System.out.println("Pass " + this.pass + " completed with " + this.mg_passes.exitValue());
 
       int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
@@ -458 +522 @@
           System.out.println("Error from mg_compression_dict: " + p.exitValue());
         }
+        else {
+          System.out.println("Compressed dictionary successfully written");
+        }
       break;
 
@@ -468 +535 @@
         }
         else {
-          System.out.println("Unable to create weights file");
+          System.out.println("Unable to create weights file " + "mg_weights_build -f " + this.indexStem + " -t " + this.textStem + " -d /");
         }