Changeset 18444 for indexers

Timestamp:

2009-02-01T14:41:33+13:00 (15 years ago)

Author:

davidb

Message:

Modifications for incremental building to support files that need to be deleted

File:

• indexers/trunk/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneIndexer.java (modified) (10 diffs)

indexers/trunk/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneIndexer.java

@@ -59 +59 @@
 public class GS2LuceneIndexer {
 
+    protected static boolean debug = false;
+
+    protected static void debug(String message)
+    {
+    if (debug) {
+        System.err.println(message);
+    }
+    }
+
+
     public static void main (String args[]) throws Exception 
     { 
@@ -82 +92 @@
             if (i<argc) {
             verbosity = Integer.parseInt(args[i]);
-            }
+            if (verbosity>=5) {
+                debug = true;
+            }
+            }
+        }
+        else if (args[i].equals("-debug")) {
+            debug = true;
         }
         else {
@@ -128 +144 @@
         while ((line = brin.readLine()) != null) {
         xml_text.append(line);
-        if (line.startsWith("</Doc>")) {
+
+        debug("Got line " + line);
+
+        if (line.endsWith("</Delete>")) {
+
+            indexer.delete(xml_text.toString());
+            xml_text = new StringBuffer(1024);          
+        }
+        else if (line.startsWith("</Doc>")) {
             indexer.index(xml_text.toString());
             xml_text = new StringBuffer(1024);
@@ -162 +186 @@
     String current_contents_ = "";
 
+    String mode_ = "";
     protected String file_id_ = null;
 
     static private String[] stop_words = GS2Analyzer.STOP_WORDS;
+
 
     /** pass in true if want to create a new index, false if want to use the existing one */
@@ -198 +224 @@
     public void index (String file_id, File file) 
     {
+        mode_ = "index";
         file_id_ = file_id;
         path_ = "";
@@ -215 +242 @@
     public void index (String xml_text) 
     {
+        mode_ = "index";
         file_id_ = "<xml doc on stdin>";
+        path_ = "";
+
+        try {
+        sax_parser_.parse(new InputSource(new StringReader(xml_text)), this);
+        }
+        catch (Exception e) {
+        println("parse error:");
+        e.printStackTrace();
+        }
+    }
+
+    /** delete one document, based on doc_id in <Delete>doc_id</Delete> */
+    public void delete(String xml_text) 
+    {
+        mode_ = "delete";
+        file_id_ = "<delete doc>";
         path_ = "";
 
@@ -250 +294 @@
     public void startDocument() throws SAXException 
     {
-        println("Starting to index " + file_id_);
-        print("[");
+        if (mode_.equals("index")) {
+        println("Starting to index " + file_id_);
+        print("[");
+        }
     }
 
     public void endDocument() throws SAXException 
     {
-        println("]");
-        println("... indexing finished.");
+        if (mode_.equals("index")) {
+        println("]");
+        println("... indexing finished.");
+        }
     }
 
@@ -263 +311 @@
         throws SAXException 
     {
-        path_ = appendPathLink(path_, qName, atts);
-
-        if (qName.equals(doc_tag_level_)) {
-        pushOnStack(); // start new doc
-        current_node_ = qName;
-
-        String node_id = atts.getValue("gs2:id");
-        print(" " + qName + ": " + node_id );
-        current_doc_.add(new Field("nodeID", node_id, Field.Store.YES, Field.Index.UN_TOKENIZED));
-
-        current_doc_oid_ = atts.getValue("gs2:docOID");
-        current_doc_.add(new Field("docOID", current_doc_oid_, Field.Store.YES, Field.Index.UN_TOKENIZED));
-        }
-
-        if (isIndexable(atts)) {
-        indexable_current_node_ = qName;    
-        }
-        else {
-        indexable_current_node_ = "";
+        if (mode_.equals("index")) {
+        path_ = appendPathLink(path_, qName, atts);
+        
+        if (qName.equals(doc_tag_level_)) {
+            pushOnStack(); // start new doc
+            current_node_ = qName;
+            
+            String node_id = atts.getValue("gs2:id");
+            print(" " + qName + ": " + node_id );
+            current_doc_.add(new Field("nodeID", node_id, Field.Store.YES, Field.Index.UN_TOKENIZED));
+            
+            current_doc_oid_ = atts.getValue("gs2:docOID");
+            current_doc_.add(new Field("docOID", current_doc_oid_, Field.Store.YES, Field.Index.UN_TOKENIZED));
+        }
+        
+        if (isIndexable(atts)) {
+            indexable_current_node_ = qName;    
+        }
+        else {
+            indexable_current_node_ = "";
+        }
         }
     }
@@ -300 +350 @@
     public void endElement(String uri, String localName, String qName) throws SAXException 
     {
-        if (qName.equals(indexable_current_node_))
-        {
-            current_doc_.add(new Field(qName, current_contents_, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES));
-            // The byXX fields are used for sorting search results
-            // We don't want to do that for Text or AllFields fields
-            // They need to be untokenised for sorting
-            if (!qName.equals("TX") && !qName.equals("ZZ"))
-            {
-                current_doc_.add(new Field("by" + qName, current_contents_, Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
-            }
-
-            current_contents_ = "";
-        }
-
-        if (qName.equals(doc_tag_level_)) {
-        try {
-            writer_.updateDocument(new Term("docOID", current_doc_oid_), current_doc_, analyzer_);
-        } 
-        catch (java.io.IOException e) {
-            e.printStackTrace();
-        }
-        popOffStack(); // end document
-        }
-    
-        path_ = removePathLink(path_);
+        if (mode_.equals("index")) {
+        if (qName.equals(indexable_current_node_))
+            {
+            current_doc_.add(new Field(qName, current_contents_, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES));
+            // The byXX fields are used for sorting search results
+            // We don't want to do that for Text or AllFields fields
+            // They need to be untokenised for sorting
+            if (!qName.equals("TX") && !qName.equals("ZZ"))
+                {
+                current_doc_.add(new Field("by" + qName, current_contents_, Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+                }
+            
+            current_contents_ = "";
+            }
+        
+        if (qName.equals(doc_tag_level_)) {
+            try {
+            writer_.updateDocument(new Term("docOID", current_doc_oid_), current_doc_, analyzer_);
+            } 
+            catch (java.io.IOException e) {
+            e.printStackTrace();
+            }
+            popOffStack(); // end document
+        }
+        
+        path_ = removePathLink(path_);
+        }
+        else if (mode_.equals("delete")) {
+        if (qName.equals("Delete")) {
+            try {
+            deleteDocument(current_contents_);
+            current_contents_ = "";
+            }
+            catch (java.io.IOException e) {
+            e.printStackTrace();
+            }
+        }
+        }
     }
 
@@ -404 +467 @@
         }
     }
+
+
+    protected void deleteDocument(String doc_id)
+        throws IOException
+    {
+        debug("GS2LuceneDelete.deleteDocument(" + doc_id + ")");
+        debug("- Initial number of documents in index: " + writer_.docCount());
+        writer_.deleteDocuments(new Term("docOID", doc_id));
+        debug("- Final number of documents in index: " + writer_.docCount());
+    }
+
+
     }
 }