Changeset 16432 for indexers/trunk/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneIndexer.java

Timestamp:

2008-07-16T16:55:20+12:00 (16 years ago)

Author:

mdewsnip

Message:

Merged the Indexer class into GS2LuceneIndexer to keep all the indexing stuff together -- not sure why they were ever separate?

File:

• indexers/trunk/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneIndexer.java (modified) (2 diffs)

indexers/trunk/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneIndexer.java

@@ -30 +30 @@
 import java.io.*;
 import java.util.Vector;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+
+import java.util.Stack;
+import java.io.FileInputStream;
+import java.io.File;
+import java.io.StringReader;
+import java.net.URL;
+
 
 /**
@@ -123 +144 @@
     indexer.finish();
     }
+
+
+    static public class Indexer extends DefaultHandler 
+    {
+    IndexWriter writer_   = null;
+    SAXParser sax_parser_ = null;
+    String doc_tag_level_ = null;
+
+    Stack stack_ = null;
+    String path_ = "";
+
+    Document current_doc_ = null;
+    String current_node_ = "";
+    String indexable_current_node_ = "";
+    String current_contents_ = "";
+
+    protected String file_id_ = null;
+
+    /** pass in true if want to create a new index, false if want to use the existing one */
+    public Indexer (String doc_tag_level, File index_dir, boolean create) 
+    {
+        doc_tag_level_ = doc_tag_level;
+
+        try {
+        stack_ = new Stack();
+        SAXParserFactory sax_factory = SAXParserFactory.newInstance();
+        sax_parser_ = sax_factory.newSAXParser();
+
+        XMLReader reader = sax_parser_.getXMLReader(); 
+        reader.setFeature("http://xml.org/sax/features/validation", false);
+
+        writer_ = new IndexWriter(index_dir.getPath(), new StandardAnalyzer(), create);
+        // by default, will only index 10,000 words per document
+        // Can throw out_of_memory errors
+        writer_.setMaxFieldLength(Integer.MAX_VALUE);
+        if (create) {
+            writer_.optimize();
+        }
+
+        } catch (Exception e) {
+        // do nothing!
+        }
+    }
+
+    /** index one document */
+    public void index (String file_id, File file) 
+    {
+        file_id_ = file_id;
+        path_ = "";
+        String base_path = file.getPath();
+        base_path = base_path.substring(0, base_path.lastIndexOf(File.separatorChar));
+
+        try {       
+        sax_parser_.parse(new InputSource(new FileInputStream(file)), this);
+        }
+        catch (Exception e) {
+        println("parse error:");
+        e.printStackTrace();
+        }
+    }
+
+    /** index one document stored as string*/
+    public void index (String xml_text) 
+    {
+        file_id_ = "<xml doc on stdin>";
+        path_ = "";
+
+        try {
+        sax_parser_.parse(new InputSource(new StringReader(xml_text)), this);
+        }
+        catch (Exception e) {
+        println("parse error:");
+        e.printStackTrace();
+        }
+    }
+    
+    public void finish() 
+    {
+        /** optimise the index */
+        try {
+        writer_.optimize();
+        writer_.close();
+        } 
+        catch (Exception e) {
+        }
+    }
+
+    protected void print(String s) 
+    { 
+        System.out.print(s); 
+    }
+
+    protected void println(String s) 
+    { 
+        System.out.println(s); 
+    }
+
+    public void startDocument() throws SAXException 
+    {
+        println("Starting to index " + file_id_);
+        print("[");
+    }
+
+    public void endDocument() throws SAXException 
+    {
+        println("]");
+        println("... indexing finished.");
+    }
+
+    public void startElement(String uri, String localName, String qName, Attributes atts) 
+        throws SAXException 
+    {
+        path_ = appendPathLink(path_, qName, atts);
+
+        if (qName.equals(doc_tag_level_)) {
+        pushOnStack(); // start new doc
+        current_node_ = qName;
+        String node_id = atts.getValue("gs2:id");
+
+        print(" " + qName + ": " + node_id );
+        current_doc_.add(new Field("nodeID", node_id, Field.Store.YES, Field.Index.UN_TOKENIZED));
+        }
+
+        if (XMLTagInfo.isIndexable(atts)) {
+        indexable_current_node_ = qName;    
+        }
+        else {
+        indexable_current_node_ = "";
+        }
+
+    }
+
+    public void endElement(String uri, String localName, String qName) throws SAXException 
+    {
+        if (qName.equals(indexable_current_node_))
+        {
+            current_doc_.add(new Field(qName, current_contents_, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES));
+            // We only need the term vector for the TX field
+            if (!qName.equals("TX"))
+            {
+                current_doc_.add(new Field("by" + qName, current_contents_, Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.NO));
+            }
+
+            current_contents_ = "";
+        }
+
+        if (qName.equals(doc_tag_level_)) {
+        try {
+            writer_.addDocument(current_doc_);
+        } 
+        catch (java.io.IOException e) {
+            e.printStackTrace();
+        }
+        popOffStack(); // end document
+        }
+    
+        path_ = removePathLink(path_);
+    }
+
+    public void characters(char ch[], int start, int length) throws SAXException 
+    {
+        String data = new String(ch, start, length).trim();
+        if (data.length() > 0 ) {
+        current_contents_ += data;
+        }
+    }
+    
+    protected String appendPathLink(String path, String qName, Attributes atts) 
+    {
+
+        path = path + "/"+qName;
+        if (atts.getLength()>0) {
+        String id = atts.getValue("gs2:id");
+        if (id != null) {
+            path +=  "[@gs2:id='"+id+"']";
+        }
+        else {
+            id = atts.getValue("gs3:id");
+            if (id != null) {
+            path +=  "[@gs3:id='"+id+"']";
+            }
+        }
+        }
+        return path;
+    }
+
+    protected String removePathLink(String path) 
+    {
+
+        int i=path.lastIndexOf('/');
+        if (i==-1) {
+        path="";
+        } else {
+        path = path.substring(0, i);
+        }
+        return path;
+    }
+
+
+    /** these are what we save on the stack */
+    private class MyDocument 
+    {
+        public Document doc = null;
+        public String contents = null;
+        public String tagname = "";
+    
+    }
+
+
+    protected void pushOnStack() 
+    {
+        if (current_doc_ != null) {
+        MyDocument save = new MyDocument();
+        save.doc = current_doc_;
+        save.contents = current_contents_;
+        save.tagname = current_node_;
+        stack_.push(save);
+        }
+        current_doc_ = new Document();
+        current_contents_ = "";
+        current_node_ = "";
+    }
+
+    protected void popOffStack() 
+    {
+        if (!stack_.empty()) {
+        MyDocument saved = (MyDocument)stack_.pop();
+        current_doc_ = saved.doc;
+        current_contents_ = saved.contents;
+        current_node_ = saved.tagname;
+        } else {
+        current_doc_ = new Document();
+        current_contents_ = "";
+        current_node_ = "";
+        }
+    }
+    }
 }