Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2Analyzer.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2Analyzer.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2Analyzer.java	(revision 29146)
@@ -0,0 +1,86 @@
+/**********************************************************************
+ *
+ * GS2Analyzer.java 
+ *
+ * Copyright 2004 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+package org.greenstone.LuceneWrapper3;
+
+
+import java.io.*;
+import java.util.Set;
+
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.standard.*;
+
+import org.apache.lucene.analysis.ASCIIFoldingFilter;
+
+import org.apache.lucene.util.Version;
+
+
+public class GS2Analyzer extends GS2StandardAnalyzer 
+{
+    
+    static Version matchVersion = Version.LUCENE_24;
+
+
+    public GS2Analyzer() 
+    {
+	super(matchVersion);
+    }
+    
+
+    public GS2Analyzer(Set stopWords) 
+    {
+	super(matchVersion,stopWords);
+    }
+
+
+    public GS2Analyzer(String [] stopwords) 
+    {
+	super(matchVersion,StopFilter.makeStopSet(stopwords));
+    }
+
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
+    src.setMaxTokenLength(maxTokenLength);
+    src.setReplaceInvalidAcronym(replaceInvalidAcronym);
+    TokenStream tok = new StandardFilter(matchVersion, src);
+    tok = new LowerCaseFilter(matchVersion, tok);
+    tok = new StopFilter(matchVersion, tok, stopwords);
+
+    // top it up with accent folding
+    tok = new ASCIIFoldingFilter(tok);
+
+    return new TokenStreamComponents(src, tok) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        src.setMaxTokenLength(GS2Analyzer.this.maxTokenLength);
+        return super.reset(reader);
+      }
+    };
+  }
+
+}
+
+
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2IndexModifier.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2IndexModifier.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2IndexModifier.java	(revision 29146)
@@ -0,0 +1,132 @@
+/** @file GS2IndexModifier.java
+ *
+ *  Extends the standard IndexModifier so that a use can retrieve documents and
+ *  the underlying IndexReader and IndexWriter. 
+ *
+ *  Addresses the problem that the Lucene standard IndexModifier can't actually
+ *  return a specified Document object, nor does it expose the IndexReader and
+ *  IndexWriter so the user can access their methods instead.
+ *
+ *  A component of the Greenstone digital library software from the New Zealand 
+ *  Digital Library Project at the University of Waikato, New Zealand.
+ *
+ *  This program is free software; you can redistribute it and/or modify it 
+ *  under the terms of the GNU General Public License as published by the Free 
+ *  Software Foundation; either version 2 of the License, or (at your option) 
+ *  any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for 
+ *  more details.
+ *
+ *  You should have received a copy of the GNU General Public License along 
+ *  with this program; if not, write to the Free Software Foundation, Inc., 675
+ *  Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *  Copyright (c) 2006 DL Consulting Ltd., New Zealand
+ */
+
+package org.greenstone.LuceneWrapper3;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+
+/**
+ */
+public class GS2IndexModifier
+    extends IndexModifier
+{
+    /** **/
+    private boolean debug = true;
+
+    /**
+     */
+    public GS2IndexModifier (String dir_name, Analyzer analyzer)
+        throws IOException
+    {
+        super(dir_name, analyzer, false);
+    }
+    /** GS2IndexModifier() **/
+
+    /**
+     */
+    protected void debug(String message)
+    {
+        if(debug)
+            {
+                System.err.println(message);
+            }
+    }
+    /** debug() **/
+
+    /**
+     */
+    public Document document(int n)
+        throws IOException
+    {
+        // Ensure the reader exists
+        createIndexReader();
+        return indexReader.document(n);
+    }
+    /** document() **/
+
+    /**
+     */
+    public int getDocNumByNodeID(int node_id)
+        throws IOException
+    {
+        debug("GS2IndexModifier.getDocument(" + node_id + ")");
+        int doc_num = -1;
+        // Create a new term to encapsulate this node id
+	// was nodeID, now using docOID --kjdon
+        Term term = new Term("docOID", String.valueOf(node_id));
+        debug("Searching using term: " + term.toString());
+        // Ensure the indexReader exists
+        createIndexReader();
+        // We can return an enumeration of docNums which contains this term
+        TermPositions term_positions = indexReader.termPositions(term);
+        // If the size of term_positions is not 1 then somethings gone wrong in
+        // that the original terms did not uniquely identify a document.
+        if (term_positions.next())
+            {
+                doc_num = term_positions.doc();
+            }
+        else
+            {
+                debug("Term doesn't exists in index");
+            }
+        term_positions.close();
+        // To the garbage collection with thee
+        term_positions = null;
+        term = null;
+        // Done
+        return doc_num;
+    }
+    /** getDocumentByNodeID() **/
+
+    /**
+     */
+    public IndexReader getIndexReader()
+    {
+        return indexReader;
+    }
+    /** getIndexReader() **/
+
+    /**
+     */
+    public IndexWriter getIndexWriter()
+    {
+        return indexWriter;
+    }
+    /** getIndexWriter() **/
+}
+/** GS2IndexModifier **/
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneDelete.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneDelete.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneDelete.java	(revision 29146)
@@ -0,0 +1,175 @@
+/** @file GS2LuceneDelete.java
+ *
+ *  Provides a wrapper to the document deleting features of Lucene. 
+ *
+ *  This java application makes use of the existing Lucene class IndexModifier 
+ *  to access and make changes to the information stored about documents in a
+ *  Lucene database. This is an essential component of the IncrementalBuilder
+ *  PERL module, and endevours to make editing the text and metadata of 
+ *  documents without having to rebuild the entire collection a reality (in
+ *  other words, true incremental/dynamic building).
+ *
+ *  A component of the Greenstone digital library software from the New Zealand 
+ *  Digital Library Project at the University of Waikato, New Zealand.
+ *
+ *  This program is free software; you can redistribute it and/or modify it 
+ *  under the terms of the GNU General Public License as published by the Free 
+ *  Software Foundation; either version 2 of the License, or (at your option) 
+ *  any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for 
+ *  more details.
+ *
+ *  You should have received a copy of the GNU General Public License along 
+ *  with this program; if not, write to the Free Software Foundation, Inc., 675
+ *  Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *  Copyright (c) 2006 DL Consulting Ltd., New Zealand
+ */
+
+package org.greenstone.LuceneWrapper3;
+
+import java.io.IOException;
+import java.io.File;
+//import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+
+
+/** Contains methods for deleting a document that has previously been indexed
+ *  into a Lucene database.
+ *  @author John Thompson, DL Consulting Ltd. (unless stated otherwise)
+ */
+public class GS2LuceneDelete
+{
+    /** This is the main entry point to the deletor and is responsible for 
+     *  parsing the arguments and creating an instance of the deletor class.
+     *
+     *  @param  args The arguments passed into the application as a string
+     *               array
+     *  @return An integer describing the exit state of the application
+     *  @throws Exception on any fatal error state
+     */
+    static public void main (String args[])
+        throws Exception
+    {
+        // Parse arguments
+        String index_path = "";
+        int node_id = -1;
+
+        for (int i = 0; i < args.length; i += 2)
+        {
+	    if (args[i].equals("--index"))
+	    {
+		index_path = args[i + 1];
+	    }
+	    else if (args[i].equals("--nodeid"))
+	    {
+		node_id = Integer.parseInt(args[i + 1]);
+	    }
+	    else 
+	    {
+		System.out.println("Error! Unknown argument: " + args[i]);
+		GS2LuceneDelete.printUsage();
+		System.exit(0);
+	    }
+	}
+
+        // Check arguments
+        if (index_path.equals(""))
+        {
+	    System.out.println("Error! Missing index path");
+	    GS2LuceneDelete.printUsage();
+	    System.exit(0);
+	}
+        if (node_id == -1)
+        {
+	    System.out.println("Error! Missing or invalid Node ID");
+	    GS2LuceneDelete.printUsage();
+	    System.exit(0);
+	}
+
+        // Instantiate deletor, and perform the delete
+        GS2LuceneDelete deletor = new GS2LuceneDelete(index_path);
+        deletor.deleteDocument(node_id);
+        deletor.destroy();
+        deletor = null;
+    }
+
+
+    /** Display program usage message.
+     */
+    static public void printUsage()
+    {
+        System.out.println("usage: GS2LuceneDelete --index <path> --nodeid <int>");
+        System.out.println("");
+        System.out.println("where:");
+        System.out.println("  index    - is the full path to the directory containing the directory");
+        System.out.println("             to edit, including the level (ie didx, sidx)");
+        System.out.println("  nodeid   - the unique identifier of the document to delete. This is the");
+        System.out.println("             same as the docnum in the GDBM");
+        System.out.println("");
+    }
+
+
+    /** **/
+    private boolean debug = true;
+
+    /** **/
+    private IndexWriter index_writer = null;
+
+
+    /** Constructor which takes the path to the Lucene index to be edited.
+     *
+     *  @param  index_path The full path to the index directory as a String
+     */
+    public GS2LuceneDelete(String index_path)
+        throws IOException
+    {
+	SimpleFSDirectory index_path_dir = new SimpleFSDirectory(new File(index_path));
+	index_writer = new IndexWriter(index_path_dir, new GS2Analyzer(), 
+				       MaxFieldLength.UNLIMITED);
+    }
+
+
+    /** When called prints a debug message but only if debugging is enabled.
+     */
+    public void debug(String message)
+    {
+        if (debug)
+        {
+	    System.err.println(message);
+	}
+    }
+
+
+    /** Destructor which unallocates connection to Lucene.
+     */
+    public void destroy()
+        throws IOException
+    {
+	index_writer.close();
+	index_writer = null;
+    }
+
+
+    /** Delete the indicated document from the Lucene index. This process is
+     *  very similar to the initial step of index editing.
+     *
+     *  @param  node_id   The unique identifier of a Lucene document as an
+     *                    integer
+     */
+    public void deleteDocument(int node_id)
+        throws IOException
+    {
+        debug("GS2LuceneDelete.deleteDocument(" + node_id + ")");
+        debug("- Initial number of documents in index: " + index_writer.numDocs());
+	index_writer.deleteDocuments(new Term("nodeid", "" + node_id));
+        debug("- Final number of documents in index: " + index_writer.numDocs());
+    }
+}
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneEditor.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneEditor.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneEditor.java	(revision 29146)
@@ -0,0 +1,363 @@
+/** @file GS2LuceneEditor.java
+ *
+ *  Provides a wrapper to the index/document editing features of Lucene. 
+ *
+ *  This java application makes use of the existing Lucene class IndexModifier 
+ *  to access and make changes to the information stored about documents in a
+ *  Lucene database. This is an essential component of the IncrementalBuilder
+ *  PERL module, and endevours to make editing the text and metadata of 
+ *  documents without having to rebuild the entire collection a reality (in
+ *  other words, true incremental/dynamic building).
+ *
+ *  A component of the Greenstone digital library software from the New Zealand 
+ *  Digital Library Project at the University of Waikato, New Zealand.
+ *
+ *  This program is free software; you can redistribute it and/or modify it 
+ *  under the terms of the GNU General Public License as published by the Free 
+ *  Software Foundation; either version 2 of the License, or (at your option) 
+ *  any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for 
+ *  more details.
+ *
+ *  You should have received a copy of the GNU General Public License along 
+ *  with this program; if not, write to the Free Software Foundation, Inc., 675
+ *  Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *  Copyright (c) 2006 DL Consulting Ltd., New Zealand
+ */
+
+package org.greenstone.LuceneWrapper3;
+
+import java.io.IOException;
+import java.io.File;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.Vector;
+
+import org.apache.lucene.analysis.Analyzer;
+//import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+
+
+/** Contains methods for modifying a document that has previously been indexed
+ *  into a Lucene database.
+ *  @author John Thompson, DL Consulting Ltd.
+ */
+public class GS2LuceneEditor
+{
+    /** This is the main entry point to the editor and is responsible for 
+     *  parsing the arguments and creating an instance of the editor class.
+     *
+     *  @param  args The arguments passed into the application as a string
+     *               array
+     *  @return An integer describing the exit state of the application
+     *  @throws Exception on any fatal error state
+     *
+     *  @author John Thompson, DL Consulting Ltd.
+     */
+    static public void main (String args[])
+        throws Exception
+    {
+        // Parse arguments
+        int node_id = -1;
+        String field = "";
+        String index_path = "";
+        String new_value = "";
+        String old_value = "";
+
+        for (int i = 0; i < args.length; i += 2)
+            {
+                if (args[i].equals("--index"))
+                    {
+                        index_path = args[i + 1];
+                    }
+                else if (args[i].equals("--nodeid"))
+                    {
+                        String temp = args[i + 1];
+                        node_id = Integer.parseInt(temp);
+                        temp = null; // Off to the gc with you!
+                    }
+                else if (args[i].equals("--field"))
+                    {
+                        field = args[i + 1];
+                    }
+                else if (args[i].equals("--oldvalue"))
+                    {
+                        old_value = args[i + 1];
+                    }
+                else if (args[i].equals("--newvalue"))
+                    {
+                        new_value = args[i + 1];
+                    }
+                else 
+                    {
+                        System.out.println("Error! Unknown argument: " + args[i]);
+                        GS2LuceneEditor.printUsage();
+                    }
+            }
+
+        // Check arguments
+        if(index_path.equals(""))
+            {
+                System.out.println("Error! Missing index path");
+                GS2LuceneEditor.printUsage();
+            }
+        if(field.equals(""))
+            {
+                System.out.println("Error! Missing field");
+                GS2LuceneEditor.printUsage();
+            }
+        if(node_id == -1)
+            {
+                System.out.println("Error! Missing or invalid Node ID");
+                GS2LuceneEditor.printUsage();
+            }
+        if(old_value.equals("") && new_value.equals(""))
+            {
+                System.out.println("Error! No modification requested");
+                GS2LuceneEditor.printUsage(); 
+            }
+
+
+        // Instantiate editor, and perform the edit
+        GS2LuceneEditor editor = new GS2LuceneEditor(index_path);
+        editor.editIndex(node_id, field, old_value, new_value);
+        editor.destroy();
+        editor = null;
+    }
+    /** main() **/
+
+    /** **/
+    private boolean debug = true;
+
+    /** **/
+    private GS2IndexModifier index_modifier;
+
+    /** Constructor which takes the path to the Lucene index to be edited.
+     *
+     *  @param  index_path The full path to the index directory as a String
+     *
+     *  @author John Thompson, DL Consulting Ltd.
+     */
+    public GS2LuceneEditor(String index_path)
+        throws IOException
+    {
+        Analyzer analyzer = new GS2Analyzer();
+        // create an index in /tmp/index, overwriting an existing one:
+        index_modifier = new GS2IndexModifier(index_path, analyzer);
+    }
+    /** GS2LuceneEditor **/
+
+    /**
+     */
+    public void debug(String message)
+    {
+        if(debug)
+            {
+                System.err.println(message);
+            }
+    }
+    /** debug() **/
+
+    /** Destructor which unallocates connection to Lucene.
+     */
+    public void destroy()
+        throws IOException
+    {
+        index_modifier.close();
+        index_modifier = null;
+    }
+
+    /** Make an edit to a Lucene index.
+     *
+     *  @param  oid       The unique identifier of a Lucene document as an
+     *                    integer
+     *  @param  field     The field to be modified as a String
+     *  @param  old_value The existing value to be changed or removed as a 
+     *                    String
+     *  @param  old_value The replacement value to be changed or added as a 
+     *                    String
+     *
+     *  @author John Thompson, DL Consulting Ltd.
+     */
+    public void editIndex(int node_id, String field, String old_value, String new_value)
+        throws IOException
+    {
+        debug("GS2LuceneEditor.editIndex(" + node_id + ",'" + field + "','" + old_value + "','" + new_value + "')");
+        debug("- Initial number of documents in index: " + index_modifier.numDocs());
+        // Retrieve the document requested
+        int doc_num = index_modifier.getDocNumByNodeID(node_id);
+        if (doc_num != -1)
+            {
+                debug("* Found document #" + doc_num);
+                // Retrieve the actual document
+                Document document = index_modifier.document(doc_num);
+                // Remove the document from the index before modifying
+                index_modifier.deleteDocument(doc_num);
+                debug("* Removed document from index prior to editing");
+                // Retrieve the requested fields values, and turn it into a 
+                // vector
+                debug("* Modifying the value of the field: " + field);
+                doEdit(document, field, old_value, new_value);
+
+                // We have to do a similar modification to the ZZ field
+                // too
+                debug("* Modifying the value of the field: ZZ");
+                doEdit(document, "ZZ", old_value, new_value);
+
+                // Re-index document
+                index_modifier.addDocument(document);
+                debug("* Reindexing modified document");
+            }
+        else
+            {
+                debug("- No such document!");
+                Document document = new Document();
+
+                // Retrieve the requested fields values, and turn it into a 
+                // vector
+                debug("* Adding the value to the field: " + field);
+                doEdit(document, field, old_value, new_value);
+
+                // We have to do a similar modification to the ZZ field
+                // too
+                debug("* Adding the value to the field: ZZ");
+                doEdit(document, "ZZ", old_value, new_value);
+
+                // We also have to initialize the nodeId value
+		// changed to use docOID --kjdon
+                document.add(new Field("docOID", String.valueOf(node_id), Field.Store.YES, Field.Index.ANALYZED));
+
+                // Re-index document
+                index_modifier.addDocument(document);
+                debug("* Indexing new document");
+            }
+
+
+    }
+    /** editIndex() **/
+
+    /**
+     */
+    protected void doEdit(Document document, String field, String old_value, String new_value)
+    {
+        if (debug)
+            {
+                debug("GS2LuceneEditor.doEdit(Document, \"" + field + "\", \"" + old_value + "\", \"" + new_value + "\")");
+            }
+
+        String values_raw[] = document.getValues(field);
+        if(values_raw != null)
+            {
+                Vector values = new Vector(Arrays.asList(values_raw));
+                // Remove all the values for this field (no other safe way to
+                // do this
+                document.removeFields(field);
+                // DEBUG
+                if (debug)
+                    {
+                        debug("- Before modification:");
+                        for(int i = 0; i < values.size(); i++)
+                            {
+                                debug("\t" + field + "[" + i + "]: " + values.get(i));
+                            }
+                    }
+                // If old_value is set, remove it from the values array
+                if(!old_value.equals(""))
+                    {
+                        // Remove all occurances of this metadata - this means
+                        // it becomes a bit dangerous to have multiple pieces
+                        // of metadata with exactly the same metadata - but
+                        // this is only for indexing purposes so its not so
+                        // bad.
+                        while(values.contains(old_value))
+                            {
+                                values.remove(old_value);
+                            }
+                    }                
+                // If new_value is set, add it to the values array
+                if(!new_value.equals("") && !values.contains(new_value))
+                    {
+                        values.add(new_value);
+                    }
+                // DEBUG
+                if(debug)
+                    {
+                        debug("- After modification:");
+                        for(int i = 0; i < values.size(); i++)
+                            {
+                                debug("\t" + field + "[" + i + "]: " + values.get(i));
+                            }
+                    }
+                // Add all the values for this field
+                for(int i = 0; i < values.size(); i++)
+                    {
+                        document.add(new Field(field, (String)values.get(i), Field.Store.YES, Field.Index.ANALYZED));
+                    }
+                values.clear();
+                values = null;
+            }
+        // We may be adding a value to a field that current has no values
+        else if (!new_value.equals(""))
+            {
+                Vector values = new Vector();
+                values.add(new_value);
+                // DEBUG
+                if(debug)
+                    {
+                        debug("- Brand spanking new values:");
+                        for(int i = 0; i < values.size(); i++)
+                            {
+                                debug("\t" + field + "[" + i + "]: " + values.get(i));
+                            }
+                    }
+                // Add all the values for this field
+                for(int i = 0; i < values.size(); i++)
+                    {
+                        document.add(new Field(field, (String)values.get(i), Field.Store.YES, Field.Index.ANALYZED));
+                    }
+                values.clear();
+                values = null;
+            }
+        // Can't do a removal unless something exists
+        else
+            {
+                debug("- No such field for this document: " + field);
+            }
+        values_raw = null;
+    }
+    /** doEdit() **/
+
+    /**
+     */
+    static public void printUsage()
+    {
+        System.out.println("usage: GS2LuceneEditor --index <path> --nodeid <int> --field <string>");
+        System.out.println("                      [--oldvalue <string>] [--newvalue <string>]");
+        System.out.println("");
+        System.out.println("where:");
+        System.out.println("  index    - is the full path to the directory containing the directory");
+        System.out.println("             to edit, including the level (ie didx, sidx)");
+        System.out.println("  nodeid   - the unique identifier of the document to change. This is the");
+        System.out.println("             same as the docnum in the GDBM");
+        System.out.println("  field    - the two letter code of the metadata field to edit. These can");
+        System.out.println("             found in the build.cfg file. ZZ is not a valid field as it");
+        System.out.println("             is handled as a special case");
+        System.out.println("  oldvalue - the current value of the metadata field if it is to be");
+        System.out.println("             replaced or removed");
+        System.out.println("  newvalue - the new value for the metadata field if it is to be replaced");
+        System.out.println("             or added");
+        System.out.println("");
+        System.exit(0);
+    }
+    /** printUsage() **/
+
+}
+/** class GS2LuceneEditor **/
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneIndexer.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneIndexer.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneIndexer.java	(revision 29146)
@@ -0,0 +1,503 @@
+/**********************************************************************
+ *
+ * GS2LuceneIndexer.java 
+ *
+ * Copyright 2004 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+package org.greenstone.LuceneWrapper3;
+
+
+import java.io.*;
+import java.util.Vector;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.analysis.Analyzer;
+
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+
+import java.util.Stack;
+import java.io.FileInputStream;
+import java.io.File;
+import java.io.StringReader;
+import java.net.URL;
+
+
+/**
+ * class for indexing XML generated by lucenebuildproc.pm
+  */
+
+public class GS2LuceneIndexer {
+
+    protected static boolean debug = false;
+
+    protected static void debug(String message)
+    {
+	if (debug) {
+	    System.err.println(message);
+	}
+    }
+
+
+    public static void main (String args[]) throws Exception 
+    { 
+	int verbosity = 1;
+	// Default is to edit the existing index
+	boolean create_new_index = false;
+
+	Vector filtered_args = new Vector();
+
+	int argc = args.length;
+	int i = 0;
+	while (i<argc) {
+	    if (args[i].startsWith("-")) {
+
+		// -removeold causes the existing index to be overwritten
+		if (args[i].equals("-removeold")) {
+		    create_new_index = true;
+		}
+
+		// -verbosity [num]
+		else if (args[i].equals("-verbosity")) {
+		    i++;
+		    if (i<argc) {
+			verbosity = Integer.parseInt(args[i]);
+			if (verbosity>=5) {
+			    debug = true;
+			}
+		    }
+		}
+		else if (args[i].equals("-debug")) {
+		    debug = true;
+		}
+		else {
+		    System.err.println("Unrecognised option: " + args[i]);
+		}
+	    }
+	    else {
+		filtered_args.add((Object)args[i]);
+	    }
+	    i++;
+	}
+
+	if (filtered_args.size() != 3) {
+	    System.err.println("Usage: java GS2LuceneIndexer [-removeold|-verbosity [num]] doc-tag-level building_dir index");
+	    return;
+	}
+ 
+	String doc_tag_level     = (String)filtered_args.get(0);
+	String building_dirname  = (String)filtered_args.get(1);
+	String index_dirname     = (String)filtered_args.get(2);
+
+	String import_dirname = building_dirname + File.separator + "text";
+
+	File import_dir   = new File(import_dirname);
+	File building_dir = new File(building_dirname);
+
+	if (!import_dir.exists()) {
+	    System.err.println("Couldn't find import directory: "+import_dirname);
+	    return;
+	}
+
+	File idx_dir = new File(building_dir.getPath()+File.separator+index_dirname+File.separator);
+	idx_dir.mkdir();
+	
+	// Set up indexer
+	Indexer indexer = new Indexer(doc_tag_level, idx_dir, create_new_index);
+
+	// Read from stdin the files to process
+	try {
+	    InputStreamReader isr = new InputStreamReader(System.in, "UTF-8");
+	    BufferedReader brin = new BufferedReader(isr);
+
+	    StringBuffer xml_text = new StringBuffer(1024);
+	    String line = null;
+	    while ((line = brin.readLine()) != null) {
+		xml_text.append(line);
+		xml_text.append(" ");
+
+		debug("Got line " + line);
+
+		if (line.endsWith("</Delete>")) {
+
+		    indexer.delete(xml_text.toString());
+		    xml_text = new StringBuffer(1024);		    
+		}
+		else if (line.startsWith("</Doc>")) {
+		    indexer.index(xml_text.toString());
+		    xml_text = new StringBuffer(1024);
+		}
+	    }
+
+	    brin.close();
+	    isr.close();
+
+	} catch (IOException e) {
+	    System.err.println("Error: unable to read from stdin");
+	    e.printStackTrace();
+	}
+
+	indexer.finish();
+    }
+
+
+    static public class Indexer extends DefaultHandler 
+    {
+	IndexWriter writer_   = null;
+	Analyzer analyzer_    = null;
+	SAXParser sax_parser_ = null;
+	String doc_tag_level_ = null;
+
+	Stack stack_ = null;
+	String path_ = "";
+
+	Document current_doc_ = null;
+	String current_node_ = "";
+	String current_doc_oid_ = "";
+	String indexable_current_node_ = "";
+      boolean tokenize = true;
+	String current_contents_ = "";
+
+	String mode_ = "";
+	protected String file_id_ = null;
+
+	/** pass in true if want to create a new index, false if want to use the existing one */
+	public Indexer (String doc_tag_level, File index_dir, boolean create) 
+	{
+	    doc_tag_level_ = doc_tag_level;
+
+	    try {
+		stack_ = new Stack();
+		SAXParserFactory sax_factory = SAXParserFactory.newInstance();
+		sax_parser_ = sax_factory.newSAXParser();
+
+		XMLReader reader = sax_parser_.getXMLReader(); 
+		reader.setFeature("http://xml.org/sax/features/validation", false);
+
+		SimpleFSDirectory index_dir_dir = new SimpleFSDirectory(new File(index_dir.getPath()));
+
+		analyzer_ = new GS2Analyzer(); // uses build in stop_word_set
+
+		writer_ = new IndexWriter(index_dir_dir, analyzer_, create, MaxFieldLength.UNLIMITED);
+				       
+		// by default, will only index 10,000 words per document
+		// Can throw out_of_memory errors
+		writer_.setMaxFieldLength(Integer.MAX_VALUE);
+		if (create) {
+		    writer_.optimize();
+		}
+	    }
+	    catch (Exception e) {
+		// We need to know if creating/opening the index fails
+		e.printStackTrace();
+	    }
+	}
+
+	/** index one document */
+	public void index (String file_id, File file) 
+	{
+	    mode_ = "add";
+	    file_id_ = file_id;
+	    path_ = "";
+	    String base_path = file.getPath();
+	    base_path = base_path.substring(0, base_path.lastIndexOf(File.separatorChar));
+
+	    try {	    
+		sax_parser_.parse(new InputSource(new FileInputStream(file)), this);
+	    }
+	    catch (Exception e) {
+		println("parse error:");
+		e.printStackTrace();
+	    }
+	}
+
+	/** index one document stored as string*/
+	public void index (String xml_text) 
+	{
+	    mode_ = "add";
+	    file_id_ = "<xml doc on stdin>";
+	    path_ = "";
+
+	    try {
+		sax_parser_.parse(new InputSource(new StringReader(xml_text)), this);
+	    }
+	    catch (Exception e) {
+		println("parse error:");
+		e.printStackTrace();
+	    }
+	}
+
+	/** delete one document, based on doc_id in <Delete>doc_id</Delete> */
+	public void delete(String xml_text) 
+	{
+	    mode_ = "delete";
+	    file_id_ = "<delete doc>";
+	    path_ = "";
+
+	    try {
+		sax_parser_.parse(new InputSource(new StringReader(xml_text)), this);
+	    }
+	    catch (Exception e) {
+		println("parse error:");
+		e.printStackTrace();
+	    }
+	}
+    
+	public void finish() 
+	{
+	    /** optimise the index */
+	    try {
+		writer_.optimize();
+		writer_.close();
+	    } 
+	    catch (Exception e) {
+	    }
+	}
+
+	protected void print(String s) 
+	{ 
+	    System.err.print(s); 
+	}
+
+	protected void println(String s) 
+	{ 
+	    System.err.println(s); 
+	}
+
+	public void startDocument() throws SAXException 
+	{
+	    println("Starting to process " + file_id_);
+	    print("[");
+	}
+
+	public void endDocument() throws SAXException 
+	{
+	    println("]");
+	    println("... processing finished.");
+	}
+
+	public void startElement(String uri, String localName, String qName, Attributes atts) 
+	    throws SAXException 
+	{
+	    path_ = appendPathLink(path_, qName, atts);
+	    
+	    if (qName.equals(doc_tag_level_)) {
+		mode_ = atts.getValue("gs2:mode");
+
+		pushOnStack(); // start new doc
+		current_node_ = qName;
+		
+		//String node_id = atts.getValue("gs2:id");
+		//print(" " + qName + ": " + node_id + " (" + mode_ + ")" );
+		//current_doc_.add(new Field("nodeID", node_id, Field.Store.YES, Field.Index.NOT_ANALYZED));
+		
+		current_doc_oid_ = atts.getValue("gs2:docOID");
+		print(" " + qName + ": " + current_doc_oid_ + " (" + mode_ + ")" );
+		current_doc_.add(new Field("docOID", current_doc_oid_, Field.Store.YES, Field.Index.NOT_ANALYZED));
+	    }
+	    
+	    if (isIndexable(atts)) {
+		indexable_current_node_ = qName;	
+		if (isTokenized(atts)) {
+		  tokenize = true;
+		} else {
+		  tokenize = false;
+		}
+	    }
+	    else {
+		indexable_current_node_ = "";
+	    }
+	}
+
+      public static boolean isTokenized(Attributes atts) {
+	boolean tokenize = true;
+	String tok = atts.getValue("tokenize");
+	if (tok!=null && tok.equals("0")) {
+	  tokenize = false;
+	}
+	return tokenize;
+      }
+
+	public static boolean isIndexable(Attributes atts) 
+	{
+	    boolean is_indexable = false;
+
+	    String index = atts.getValue("index");
+	    if (index!=null) {
+		if (index.equals("1")) {
+		    is_indexable = true;
+		}
+	    }
+	    return is_indexable;
+	}
+
+	public void endElement(String uri, String localName, String qName) throws SAXException 
+	{
+	    if (mode_.equals("delete")) {
+		try {
+		    deleteDocument(current_doc_oid_);
+		}
+		catch (java.io.IOException e) {
+		    e.printStackTrace();
+		}
+	    }
+	    else if (mode_.equals("add") || mode_.equals("update")) {
+		if (qName.equals(indexable_current_node_))
+		    {
+		      if (tokenize) {
+			current_doc_.add(new Field(qName, current_contents_, Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
+		      } else {
+			current_doc_.add(new Field(qName, current_contents_, Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO));
+		      }
+			// // The byXX fields are used for sorting search results
+			// // We don't want to do that for Text or AllFields fields
+			// // They need to be untokenised for sorting
+			// if (!qName.equals("TX") && !qName.equals("ZZ"))
+			//     {
+			// 	current_doc_.add(new Field("by" + qName, current_contents_, Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO));
+			//     }
+			
+			current_contents_ = "";
+		    }
+		
+		if (qName.equals(doc_tag_level_)) {
+		    try {
+			// perhaps this is more efficient if addDocument()
+			// used for "add" and updateDocument() for "update"
+			writer_.updateDocument(new Term("docOID", current_doc_oid_), current_doc_, analyzer_);
+		    } 
+		    catch (java.io.IOException e) {
+			e.printStackTrace();
+		    }
+		    popOffStack(); // end document
+		}
+		
+		path_ = removePathLink(path_);
+	    }
+	}
+
+	public void characters(char ch[], int start, int length) throws SAXException 
+	{
+	    String data = new String(ch, start, length).trim();
+	    if (data.length() > 0 ) {
+		current_contents_ += data;
+	    }
+	}
+    
+	protected String appendPathLink(String path, String qName, Attributes atts) 
+	{
+
+	    path = path + "/"+qName;
+	    if (atts.getLength()>0) {
+	      // was gs2:id, changed to gs2:docOID --kjdon
+		String id = atts.getValue("gs2:docOID");
+		if (id != null) {
+		    path +=  "[@gs2:docOID='"+id+"']";
+		}
+		else {
+		  // is this ever used? not in perl currently
+		    id = atts.getValue("gs3:id");
+		    if (id != null) {
+			path +=  "[@gs3:id='"+id+"']";
+		    }
+		}
+	    }
+	    return path;
+	}
+
+	protected String removePathLink(String path) 
+	{
+
+	    int i=path.lastIndexOf('/');
+	    if (i==-1) {
+		path="";
+	    } else {
+		path = path.substring(0, i);
+	    }
+	    return path;
+	}
+
+
+	/** these are what we save on the stack */
+	private class MyDocument 
+	{
+	    public Document doc = null;
+	    public String contents = null;
+	    public String tagname = "";
+	
+	}
+
+
+	protected void pushOnStack() 
+	{
+	    if (current_doc_ != null) {
+		MyDocument save = new MyDocument();
+		save.doc = current_doc_;
+		save.contents = current_contents_;
+		save.tagname = current_node_;
+		stack_.push(save);
+	    }
+	    current_doc_ = new Document();
+	    current_contents_ = "";
+	    current_node_ = "";
+	}
+
+	protected void popOffStack() 
+	{
+	    if (!stack_.empty()) {
+		MyDocument saved = (MyDocument)stack_.pop();
+		current_doc_ = saved.doc;
+		current_contents_ = saved.contents;
+		current_node_ = saved.tagname;
+	    } else {
+		current_doc_ = new Document();
+		current_contents_ = "";
+		current_node_ = "";
+	    }
+	}
+
+
+	protected void deleteDocument(String doc_id)
+	    throws IOException
+	{
+	    debug("GS2LuceneDelete.deleteDocument(" + doc_id + ")");
+	    debug("- Initial number of documents in index: " + writer_.numDocs());
+	    writer_.deleteDocuments(new Term("docOID", doc_id));
+	    debug("- Final number of documents in index: " + writer_.numDocs());
+	}
+
+
+    }
+}
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneQuery.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneQuery.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneQuery.java	(revision 29146)
@@ -0,0 +1,708 @@
+/**********************************************************************
+ *
+ * GS2LuceneQuery.java 
+ *
+ * Copyright 2004 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+package org.greenstone.LuceneWrapper3;
+
+
+import java.io.*;
+import java.util.*;
+import java.util.regex.*;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.BooleanQuery; // for the TooManyClauses exception
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.MultiTermQuery.ConstantScoreAutoRewrite;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermRangeFilter;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+
+public class GS2LuceneQuery extends SharedSoleneQuery
+{
+  public static String SORT_RANK = "rank";
+  public static String SORT_NATURAL = "natural";
+
+    protected String full_indexdir="";
+
+  protected int sort_type = SortField.SCORE;
+  protected boolean reverse_sort = false;
+    protected Sort sorter=new Sort();
+    protected Filter filter = null;
+
+    protected static Version matchVersion = Version.LUCENE_24;
+
+    protected QueryParser query_parser = null;
+    protected QueryParser query_parser_no_stop_words = null;
+    protected Searcher searcher = null;
+    protected IndexReader reader = null;
+
+    public GS2LuceneQuery() {
+	super();
+
+	// Create one query parser with the standard set of stop words, and one with none
+
+	query_parser = new QueryParser(matchVersion, TEXTFIELD, new GS2Analyzer()); // uses built-in stop_words_set
+    	query_parser_no_stop_words = new QueryParser(matchVersion, TEXTFIELD, new GS2Analyzer(new String[] { }));
+    }
+    
+    
+    public boolean initialise() {
+
+	if (!super.initialise()) {
+	    return false;
+	}
+
+
+    	if (full_indexdir==null || full_indexdir.length()==-1){
+	    utf8out.println("Index directory is not indicated ");
+	    utf8out.flush();
+	    return false;
+    	}
+
+        try {
+	    Directory full_indexdir_dir = FSDirectory.open(new File(full_indexdir));
+    	    searcher = new IndexSearcher(full_indexdir_dir,true);
+	    // need to set this to get it to compute ranks when sorting by fields
+	    ((IndexSearcher)searcher).setDefaultFieldSortScoring(true,true);
+    	    reader = ((IndexSearcher) searcher).getIndexReader();
+	    
+	    
+	    this.sorter = new Sort(new SortField(this.sort_field, this.sort_type, this.reverse_sort));
+	}
+	catch (IOException exception) {
+    	    exception.printStackTrace();
+	    return false;
+    	}
+	return true;
+
+    }
+
+    public void setIndexDir(String full_indexdir) {
+	this.full_indexdir = full_indexdir;
+    }
+
+    public void setSortField(String sort_field) {
+      if (sort_field.equals(SORT_RANK)) {
+	this.sort_field = null;
+	this.sort_type = SortField.SCORE;
+      } else if (sort_field.equals(SORT_NATURAL)) {
+	this.sort_field = null;
+	this.sort_type = SortField.DOC;
+      } else {
+	this.sort_field  = sort_field;
+	this.sort_type = SortField.STRING; // for now. numeric??
+      }
+    }
+  public void setReverseSort(boolean reverse) {
+    this.reverse_sort = reverse;
+  }
+  public boolean getReverseSort() {
+    return this.reverse_sort;
+  }
+
+    public void setFilterString(String filter_string) {
+	super.setFilterString(filter_string);
+	this.filter = parseFilterString(filter_string);
+    }
+
+    public Filter getFilter() {
+	return this.filter;
+    }
+
+    
+    public LuceneQueryResult runQuery(String query_string) {
+	
+	if (query_string == null || query_string.equals("")) {
+	    utf8out.println("The query word is not indicated ");
+	    utf8out.flush();
+	    return null;
+	}
+
+ 	LuceneQueryResult lucene_query_result=new LuceneQueryResult();
+	lucene_query_result.clear();
+	    
+	try {    	        
+	    Query query_including_stop_words = query_parser_no_stop_words.parse(query_string);
+	    query_including_stop_words = query_including_stop_words.rewrite(reader);
+		
+	    // System.err.println("********* query_string " + query_string + "****");
+
+	    Query query = parseQuery(reader, query_parser, query_string, fuzziness);
+
+	    // GS2's LuceneWrapper uses lucene-2.3.2. GS3's LuceneWrapper3 works with lucene-3.3.0. 
+	    // This change in lucene core library for GS3 (present since after version 2.4.1) had the
+	    // side-effect that searching on "econom*" didn't display what terms it was searching for, 
+	    // whereas it had done so in GS2. 
+
+	    // The details of this problem and its current solution are explained in the ticket 
+	    // http://trac.greenstone.org/ticket/845
+
+	    // We need to change the settings for the rewriteMethod in order to get searches on wildcards
+	    // to produce search terms again when the query gets rewritten.
+
+	    // We try, in order:
+	    // 1. RewriteMethod set to BooleanQuery, to get it working as in GS2 which uses lucene-2.3.2
+	    // it will expand wildcard searches to its terms when searching at both section AND doc level.
+	    // If that throws a TooManyClauses exception (like when searching for "a*" over lucene demo collection)
+	    // 2. Then try a custom rewriteMethod which sets termCountCutoff=350 and docCountPercent cutoff=0.1%
+	    // If that throws a TooManyClauses exception (could perhaps happen if the collection has a huge number of docs
+	    // 3. Then try the default apache rewriteMethod with its optimum defaults of 
+	    // termCountCutoff=350 and docCountPercent cutoff=0.1%
+	    // 	See http://lucene.apache.org/core/3_6_1/api/core/org/apache/lucene/search/MultiTermQuery.html
+
+	    if(query instanceof MultiTermQuery) {
+		MultiTermQuery multiTermQuery = (MultiTermQuery)query;
+		multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
+		     // less CPU intensive than MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE)
+	    }
+
+	    try {
+		query = query.rewrite(reader);
+	    } 
+	    catch(BooleanQuery.TooManyClauses clauseException) {
+		// Example test case: try searching the lucene demo collection for "a*" 
+		// and you'll hit this exception
+
+		lucene_query_result.setError(LuceneQueryResult.TOO_MANY_CLAUSES_ERROR);
+
+		if(query instanceof MultiTermQuery) {
+
+		    // CustomRewriteMethod: setting the docCountPercent cutoff to a custom 100%. 
+		    // This will at least expand the query to its terms when searching with wildcards at section-level 
+		    // (though it doesn't seem to work for doc-level searches, no matter what the cutoffs are set to).
+
+		    MultiTermQuery.ConstantScoreAutoRewrite customRewriteMethod = new MultiTermQuery.ConstantScoreAutoRewrite();
+		    customRewriteMethod.setDocCountPercent(100.0);
+		    customRewriteMethod.setTermCountCutoff(350); // same as default
+		    
+		    MultiTermQuery multiTermQuery = (MultiTermQuery)query;
+		    multiTermQuery.setRewriteMethod(customRewriteMethod);
+		    try {
+			query = query.rewrite(reader);
+		    } 
+		    catch(BooleanQuery.TooManyClauses clauseExceptionAgain) {
+
+			// do what the code originally did: use the default rewriteMethod which
+			// uses a default docCountPercent=0.1 (%) and termCountCutoff=350
+
+			multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
+			query = query.rewrite(reader);
+		    }
+		}
+	    }
+
+	    // Get the list of expanded query terms and their frequencies 
+	    // num docs matching, and total frequency	    
+	    HashSet terms = new HashSet();
+	    query.extractTerms(terms);
+
+	    HashMap doc_term_freq_map = new HashMap();
+	    
+	    Iterator iter = terms.iterator();
+	    while (iter.hasNext()) {
+		    
+		Term term = (Term) iter.next();
+		
+		// Get the term frequency over all the documents
+		TermDocs term_docs = reader.termDocs(term);
+		int term_freq = 0;
+		int match_docs = 0;
+		while (term_docs.next())
+		{
+		    if (term_docs.freq() != 0)
+		    {
+			term_freq += term_docs.freq();
+			match_docs++;
+
+			// Calculate the document-level term frequency as well
+			Integer lucene_doc_num_obj = new Integer(term_docs.doc());
+			int doc_term_freq = 0;
+                        if (doc_term_freq_map.containsKey(lucene_doc_num_obj))
+			{
+			    doc_term_freq = ((Integer) doc_term_freq_map.get(lucene_doc_num_obj)).intValue();
+			}
+			doc_term_freq += term_docs.freq();
+
+			doc_term_freq_map.put(lucene_doc_num_obj, new Integer(doc_term_freq));
+		    }
+		}
+
+		// Create a term 
+		lucene_query_result.addTerm(term.text(), term.field(), match_docs, term_freq);
+	    }
+	
+	    // Get the list of stop words removed from the query
+	    HashSet terms_including_stop_words = new HashSet();
+	    query_including_stop_words.extractTerms(terms_including_stop_words);
+	    Iterator terms_including_stop_words_iter = terms_including_stop_words.iterator();
+	    while (terms_including_stop_words_iter.hasNext()) {
+		Term term = (Term) terms_including_stop_words_iter.next();
+		if (!terms.contains(term)) {
+		    lucene_query_result.addStopWord(term.text());
+		}
+	    }
+	    
+	    // do the query
+	    // Simple case for getting all the matching documents
+	    if (end_results == Integer.MAX_VALUE) {
+		// Perform the query (filter and sorter may be null)
+		TopFieldDocs hits = searcher.search(query, filter, end_results, sorter);
+		lucene_query_result.setTotalDocs(hits.totalHits);
+
+		// Output the matching documents
+		lucene_query_result.setStartResults(start_results);
+		lucene_query_result.setEndResults(hits.totalHits);
+
+		for (int i = start_results; i <= hits.totalHits; i++) {
+		    int lucene_doc_num = hits.scoreDocs[i - 1].doc;
+		    Document doc = reader.document(lucene_doc_num);
+		    int doc_term_freq = 0;
+		    Integer doc_term_freq_object = (Integer) doc_term_freq_map.get(new Integer(lucene_doc_num));
+		    if (doc_term_freq_object != null)
+		    {
+			doc_term_freq = doc_term_freq_object.intValue();
+		    }
+		    lucene_query_result.addDoc(doc.get("docOID").trim(), hits.scoreDocs[i-1].score, doc_term_freq);
+		}
+	    }
+
+	    // Slightly more complicated case for returning a subset of the matching documents
+	    else {
+		// Perform the query (filter may be null)
+		TopFieldDocs hits = searcher.search(query, filter, end_results, sorter);
+		lucene_query_result.setTotalDocs(hits.totalHits);
+		
+		lucene_query_result.setStartResults(start_results);
+		lucene_query_result.setEndResults(end_results < hits.scoreDocs.length ? end_results: hits.scoreDocs.length);
+
+		// Output the matching documents
+		for (int i = start_results; (i <= hits.scoreDocs.length && i <= end_results); i++) {
+		    int lucene_doc_num = hits.scoreDocs[i - 1].doc;
+		    Document doc = reader.document(lucene_doc_num);
+		    int doc_term_freq = 0;
+		    Integer doc_term_freq_object = (Integer) doc_term_freq_map.get(new Integer(lucene_doc_num));
+		    if (doc_term_freq_object != null)
+		    {
+			doc_term_freq = doc_term_freq_object.intValue();
+		    }
+		    lucene_query_result.addDoc(doc.get("docOID").trim(), hits.scoreDocs[i-1].score, doc_term_freq);
+		}
+	    }
+	}
+	
+	catch (ParseException parse_exception) {
+	    lucene_query_result.setError(LuceneQueryResult.PARSE_ERROR);
+	}
+	catch (BooleanQuery.TooManyClauses too_many_clauses_exception) {
+	    lucene_query_result.setError(LuceneQueryResult.TOO_MANY_CLAUSES_ERROR);
+	}
+	catch (IOException exception) {
+	    lucene_query_result.setError(LuceneQueryResult.IO_ERROR);
+	    exception.printStackTrace();
+	}
+	catch (Exception exception) {
+	    lucene_query_result.setError(LuceneQueryResult.OTHER_ERROR);
+	    exception.printStackTrace();
+	}
+	return lucene_query_result;
+    }
+
+    public void setDefaultConjunctionOperator(String default_conjunction_operator) {
+	super.setDefaultConjunctionOperator(default_conjunction_operator);
+
+	if (default_conjunction_operator.equals("AND")) {
+	    query_parser.setDefaultOperator(query_parser.AND_OPERATOR);
+	    query_parser_no_stop_words.setDefaultOperator(query_parser.AND_OPERATOR);
+	} else { // default is OR
+	    query_parser.setDefaultOperator(query_parser.OR_OPERATOR);
+	    query_parser_no_stop_words.setDefaultOperator(query_parser.OR_OPERATOR);
+	}
+    }
+     
+       
+    public void cleanUp() {
+	super.cleanUp();
+	try {
+	    if (searcher != null) {
+		searcher.close();
+	    }
+	} catch (IOException exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    protected Query parseQuery(IndexReader reader, QueryParser query_parser, String query_string, String fuzziness)
+	throws java.io.IOException, org.apache.lucene.queryParser.ParseException
+    {
+	// Split query string into the search terms and the filter terms
+	// * The first +(...) term contains the search terms so count
+	//   up '(' and stop when we finish matching ')'
+	int offset = 0;
+	int paren_count = 0;
+	boolean seen_paren = false;
+	while (offset < query_string.length() && (!seen_paren || paren_count > 0)) {
+	    if (query_string.charAt(offset) == '(') {
+		paren_count++;
+		seen_paren = true;
+	    }
+	    if (query_string.charAt(offset) == ')') {
+		paren_count--;
+	    }
+	    offset++;
+	}
+	String query_prefix = query_string.substring(0, offset);
+	String query_suffix = query_string.substring(offset);
+	
+	///ystem.err.println("Prefix: " + query_prefix);
+	///ystem.err.println("Suffix: " + query_suffix);
+	
+	Query query = query_parser.parse(query_prefix);
+	query = query.rewrite(reader);
+	
+	// If this is a fuzzy search, then we need to add the fuzzy
+	// flag to each of the query terms
+	if (fuzziness != null && query.toString().length() > 0) {
+	    
+	    // Revert the query to a string
+	    System.err.println("Rewritten query: " + query.toString());
+	    // Search through the string for TX:<term> query terms
+	    // and append the ~ operator. Note that this search will
+	    // not change phrase searches (TX:"<term> <term>") as
+	    // fuzzy searching is not possible for these entries.
+	    // Yahoo! Time for a state machine!
+	    StringBuffer mutable_query_string = new StringBuffer(query.toString());
+	    int o = 0; // Offset
+	    // 0 = BASE, 1 = SEEN_T, 2 = SEEN_TX, 3 = SEEN_TX:
+	    int s = 0; // State
+	    while(o < mutable_query_string.length()) {
+		char c = mutable_query_string.charAt(o);
+		if (s == 0 && c == TEXTFIELD.charAt(0)) {
+		    ///ystem.err.println("Found T!");
+		    s = 1;
+		}
+		else if (s == 1) {
+		    if (c == TEXTFIELD.charAt(1)) {
+			///ystem.err.println("Found X!");
+			s = 2;
+		    }
+		    else {
+			s = 0; // Reset
+		    }
+		}
+		else if (s == 2) {
+		    if (c == ':') {
+			///ystem.err.println("Found TX:!");
+			s = 3;
+		    }
+		    else {
+			s = 0; // Reset
+		    }
+		}
+		else if (s == 3) {
+		    // Don't process phrases
+		    if (c == '"') {
+			///ystem.err.println("Stupid phrase...");
+			s = 0; // Reset
+		    }
+		    // Found the end of the term... add the
+		    // fuzzy search indicator
+		    // Nor outside the scope of parentheses
+		    else if (Character.isWhitespace(c) || c == ')') {
+			///ystem.err.println("Yahoo! Found fuzzy term.");
+			mutable_query_string.insert(o, '~' + fuzziness);
+			o++;
+			s = 0; // Reset
+		    }
+		}
+		o++;
+	    }
+	    // If we were in the state of looking for the end of a
+	    // term - then we just found it!
+	    if (s == 3) {
+		    
+		mutable_query_string.append('~' + fuzziness);
+	    }
+	    // Reparse the query
+	    ///ystem.err.println("Fuzzy query: " + mutable_query_string.toString() + query_suffix);
+	    query = query_parser.parse(mutable_query_string.toString() + query_suffix);
+	}
+	else {
+	    query = query_parser.parse(query_prefix + query_suffix);
+	}
+	
+	return query;
+    }
+
+    protected Filter parseFilterString(String filter_string)
+    {
+	Filter result = null;
+	Pattern pattern = Pattern.compile("\\s*\\+(\\w+)\\:([\\{\\[])(\\d+)\\s+TO\\s+(\\d+)([\\}\\]])\\s*");
+	Matcher matcher = pattern.matcher(filter_string);
+	if (matcher.matches()) {
+	    String field_name = matcher.group(1);
+	    boolean include_lower = matcher.group(2).equals("[");
+	    String lower_term = matcher.group(3);
+	    String upper_term = matcher.group(4);
+	    boolean include_upper = matcher.group(5).equals("]");
+	    result = new TermRangeFilter(field_name, lower_term, upper_term, include_lower, include_upper);
+	}
+	else {
+	    System.err.println("Error: Could not understand filter string \"" + filter_string + "\"");
+	}
+	return result;
+    }
+    
+
+    /** command line program and auxiliary methods */
+
+    // Fairly self-explanatory I should hope
+    static protected boolean query_result_caching_enabled = false;
+
+
+    static public void main (String args[])
+    {
+	if (args.length == 0) {
+	    System.out.println("Usage: GS2LuceneQuery <index directory> [-fuzziness value] [-filter filter_string] [-sort sort_field] [-reverse_sort][-dco AND|OR] [-startresults number -endresults number] [query]");
+	    return;
+	}
+
+	try {
+	    String index_directory = args[0];
+	    
+	    GS2LuceneQuery queryer = new GS2LuceneQuery();
+	    queryer.setIndexDir(index_directory);
+
+	    // Prepare the index cache directory, if query result caching is enabled
+	    if (query_result_caching_enabled) {
+		// Make the index cache directory if it doesn't already exist
+		File index_cache_directory = new File(index_directory, "cache");
+		if (!index_cache_directory.exists()) {
+		    index_cache_directory.mkdir();
+		}
+
+		// Disable caching if the index cache directory isn't available
+		if (!index_cache_directory.exists() || !index_cache_directory.isDirectory()) {
+		    query_result_caching_enabled = false;
+		}
+	    }
+
+	    String query_string = null;
+
+	    // Parse the command-line arguments
+            for (int i = 1; i < args.length; i++) {
+		if (args[i].equals("-sort")) {
+		    i++;
+		    queryer.setSortField(args[i]);
+		}
+		else if (args[i].equals("-reverse_sort")) {
+		  queryer.setReverseSort(true);
+		}
+		else if (args[i].equals("-filter")) {
+		    i++;
+		    queryer.setFilterString(args[i]);
+		}
+		else if (args[i].equals("-dco")) {
+		    i++;
+		    queryer.setDefaultConjunctionOperator(args[i]);
+		}
+		else if (args[i].equals("-fuzziness")) {
+		    i++;
+		    queryer.setFuzziness(args[i]);
+		}
+		else if (args[i].equals("-startresults")) {
+		    i++;
+		    if (args[i].matches("\\d+")) {
+			queryer.setStartResults(Integer.parseInt(args[i]));
+		    }
+		}
+		else if (args[i].equals("-endresults")) {
+		    i++;
+		    if (args[i].matches("\\d+")) {
+			queryer.setEndResults(Integer.parseInt(args[i]));
+		    }
+		}
+		else {
+		    query_string = args[i];
+		}
+	    }
+	    
+	    if (!queryer.initialise()) {
+		return;
+	    }
+	    
+	    // The query string has been specified as a command-line argument
+	    if (query_string != null) {
+		runQueryCaching(index_directory, queryer, query_string);
+	    }
+
+	    // Read queries from STDIN
+	    else {
+		BufferedReader in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
+		while (true) {
+		    // Read the query from STDIN
+		    query_string = in.readLine();
+		    if (query_string == null || query_string.length() == -1) {
+			break;
+		    }
+
+		    runQueryCaching(index_directory, queryer, query_string);
+		    
+		}
+	    }
+	    queryer.cleanUp();
+	}
+ 	catch (IOException exception) {
+ 	    exception.printStackTrace();
+ 	}
+    }
+
+    protected static void runQueryCaching(String index_directory, GS2LuceneQuery queryer, String query_string) 
+	throws IOException
+    {
+	StringBuffer query_results_xml = new StringBuffer();
+
+	// Check if this query result has been cached from a previous search (if it's enabled)
+	File query_result_cache_file = null;
+	if (query_result_caching_enabled) {
+	    // Generate the cache file name from the query options
+	    String query_result_cache_file_name = query_string + "-";
+	    String fuzziness = queryer.getFuzziness();
+	    query_result_cache_file_name += ((fuzziness != null) ? fuzziness : "") + "-";
+	    String filter_string = queryer.getFilterString();
+	    query_result_cache_file_name += ((filter_string != null) ? filter_string : "") + "-";
+	    String sort_string = queryer.getSortField();
+	    query_result_cache_file_name += ((sort_string != null) ? sort_string : "") + "-";
+	    String reverse_sort_string = (queryer.getReverseSort() ? "1" : "0");
+	    query_result_cache_file_name += reverse_sort_string + "-";
+	    String default_conjunction_operator = queryer.getDefaultConjunctionOperator();
+	    query_result_cache_file_name += default_conjunction_operator + "-";
+	    int start_results = queryer.getStartResults();
+	    int end_results = queryer.getEndResults();
+	    query_result_cache_file_name += start_results + "-" + end_results;
+	    query_result_cache_file_name = fileSafe(query_result_cache_file_name);
+
+	    // If the query result cache file exists, just return its contents and we're done
+	    File index_cache_directory = new File(index_directory, "cache");
+	    query_result_cache_file = new File(index_cache_directory, query_result_cache_file_name);
+	    if (query_result_cache_file.exists() && query_result_cache_file.isFile()) {
+		FileInputStream fis = new FileInputStream(query_result_cache_file);
+		InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
+		BufferedReader buffered_reader = new BufferedReader(isr);
+		String line = "";
+		while ((line = buffered_reader.readLine()) != null) {
+		    query_results_xml.append(line + "\n");
+		}
+		String query_results_xml_string = query_results_xml.toString();
+		query_results_xml_string = query_results_xml_string.replaceFirst("cached=\"false\"", "cached=\"true\"");
+
+		utf8out.print(query_results_xml_string);
+		utf8out.flush();
+
+		return;
+	    }
+	}
+	
+	// not cached
+	query_results_xml.append("<ResultSet cached=\"false\">\n");
+	query_results_xml.append("<QueryString>" + LuceneQueryResult.xmlSafe(query_string) + "</QueryString>\n");
+	Filter filter = queryer.getFilter();
+	if (filter != null) {
+	    query_results_xml.append("<FilterString>" + filter.toString() + "</FilterString>\n");
+	}
+	
+	LuceneQueryResult query_result = queryer.runQuery(query_string);
+	if (query_result == null) {
+	    System.err.println("Couldn't run the query");
+	    return;
+	}
+	
+	if (query_result.getError() != LuceneQueryResult.NO_ERROR) {
+	    query_results_xml.append("<Error type=\""+query_result.getErrorString()+"\" />\n");
+	} else {
+	    query_results_xml.append(query_result.getXMLString());
+	}
+	query_results_xml.append("</ResultSet>\n");
+
+	utf8out.print(query_results_xml);
+	utf8out.flush();
+
+	// Cache this query result, if desired
+	if (query_result_caching_enabled) {
+	    // Catch any exceptions thrown trying to write the query result cache file and warn about them, but don't
+	    //   bother with the full stack trace. It won't affect the functionality if we can't write some cache
+	    //   files, it will just affect the speed of subsequent requests.
+	    // Example exceptions are "permission denied" errors, or "filename too long" errors (the filter string
+	    //   can get very long in some collections)
+	    try
+	    {
+		FileWriter query_result_cache_file_writer = new FileWriter(query_result_cache_file);
+		query_result_cache_file_writer.write(query_results_xml.toString());
+		query_result_cache_file_writer.close();
+	    }
+	    catch (Exception exception)
+	    {
+		System.err.println("Warning: Exception occurred trying to write query result cache file (" + exception + ")");
+	    }
+	}
+    }
+    
+    protected static String fileSafe(String text)
+    {
+	StringBuffer file_safe_text = new StringBuffer();
+	for (int i = 0; i < text.length(); i++) {
+	    char character = text.charAt(i);
+	    if ((character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') || character == '-') {
+		file_safe_text.append(character);
+	    }
+	    else {
+		file_safe_text.append('%');
+		file_safe_text.append((int) character);
+	    }
+	}
+	return file_safe_text.toString();
+    }
+
+    
+}
+
+
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2StandardAnalyzer.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2StandardAnalyzer.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2StandardAnalyzer.java	(revision 29146)
@@ -0,0 +1,145 @@
+package org.greenstone.LuceneWrapper3;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Modified so the class is *not* final
+//
+// This is an interim measure.  
+//
+// We should rewrite GS2Analyzer so it inherits directly from
+// StopwordAnalyzeBase and uses ASCIIFoldingFilter rather than
+// ISOLatin1AccentFilter as the latter is deprecated and the former is
+// a superset
+
+
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.util.Version;
+
+import org.apache.lucene.analysis.standard.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Set;
+
+/**
+ * Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
+ * LowerCaseFilter} and {@link StopFilter}, using a list of
+ * English stop words.
+ *
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version}
+ * compatibility when creating StandardAnalyzer:
+ * <ul>
+ *   <li> As of 3.1, StandardTokenizer implements Unicode text segmentation,
+ *        and StopFilter correctly handles Unicode 4.0 supplementary characters
+ *        in stopwords.  {@link ClassicTokenizer} and {@link ClassicAnalyzer} 
+ *        are the pre-3.1 implementations of StandardTokenizer and
+ *        StandardAnalyzer.
+ *   <li> As of 2.9, StopFilter preserves position increments
+ *   <li> As of 2.4, Tokens incorrectly identified as acronyms
+ *        are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1068</a>)
+ * </ul>
+ */
+public class GS2StandardAnalyzer extends StopwordAnalyzerBase {
+
+  /** Default maximum allowed token length */
+  public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
+
+  protected int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
+
+  /**
+   * Specifies whether deprecated acronyms should be replaced with HOST type.
+   * See {@linkplain "https://issues.apache.org/jira/browse/LUCENE-1068"}
+   */
+  protected final boolean replaceInvalidAcronym;
+
+  /** An unmodifiable set containing some common English words that are usually not
+  useful for searching. */
+  public static final Set<?> STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET; 
+
+  /** Builds an analyzer with the given stop words.
+   * @param matchVersion Lucene version to match See {@link
+   * <a href="#version">above</a>}
+   * @param stopWords stop words */
+  public GS2StandardAnalyzer(Version matchVersion, Set<?> stopWords) {
+    super(matchVersion, stopWords);
+    replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24);
+  }
+
+  /** Builds an analyzer with the default stop words ({@link
+   * #STOP_WORDS_SET}).
+   * @param matchVersion Lucene version to match See {@link
+   * <a href="#version">above</a>}
+   */
+  public GS2StandardAnalyzer(Version matchVersion) {
+    this(matchVersion, STOP_WORDS_SET);
+  }
+
+  /** Builds an analyzer with the stop words from the given file.
+   * @see WordlistLoader#getWordSet(File)
+   * @param matchVersion Lucene version to match See {@link
+   * <a href="#version">above</a>}
+   * @param stopwords File to read stop words from */
+  public GS2StandardAnalyzer(Version matchVersion, File stopwords) throws IOException {
+    this(matchVersion, WordlistLoader.getWordSet(stopwords));
+  }
+
+  /** Builds an analyzer with the stop words from the given reader.
+   * @see WordlistLoader#getWordSet(Reader)
+   * @param matchVersion Lucene version to match See {@link
+   * <a href="#version">above</a>}
+   * @param stopwords Reader to read stop words from */
+  public GS2StandardAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
+    this(matchVersion, WordlistLoader.getWordSet(stopwords));
+  }
+
+  /**
+   * Set maximum allowed token length.  If a token is seen
+   * that exceeds this length then it is discarded.  This
+   * setting only takes effect the next time tokenStream or
+   * reusableTokenStream is called.
+   */
+  public void setMaxTokenLength(int length) {
+    maxTokenLength = length;
+  }
+    
+  /**
+   * @see #setMaxTokenLength
+   */
+  public int getMaxTokenLength() {
+    return maxTokenLength;
+  }
+
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
+    src.setMaxTokenLength(maxTokenLength);
+    src.setReplaceInvalidAcronym(replaceInvalidAcronym);
+    TokenStream tok = new StandardFilter(matchVersion, src);
+    tok = new LowerCaseFilter(matchVersion, tok);
+    tok = new StopFilter(matchVersion, tok, stopwords);
+    return new TokenStreamComponents(src, tok) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        src.setMaxTokenLength(GS2StandardAnalyzer.this.maxTokenLength);
+        return super.reset(reader);
+      }
+    };
+  }
+}
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/IndexModifier.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/IndexModifier.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/IndexModifier.java	(revision 29146)
@@ -0,0 +1,625 @@
+package org.greenstone.LuceneWrapper3;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.LockObtainFailedException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.StaleReaderException;
+import org.apache.lucene.index.SerialMergeScheduler;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * <p>[Note that as of <b>2.1</b>, all but one of the
+ * methods in this class are available via {@link
+ * IndexWriter}.  The one method that is not available is
+ * {@link #deleteDocument(int)}.]</p>
+ *
+ * A class to modify an index, i.e. to delete and add documents. This
+ * class hides {@link IndexReader} and {@link IndexWriter} so that you
+ * do not need to care about implementation details such as that adding
+ * documents is done via IndexWriter and deletion is done via IndexReader.
+ * 
+ * <p>Note that you cannot create more than one <code>IndexModifier</code> object
+ * on the same directory at the same time.
+ * 
+ * <p>Example usage:
+ * 
+<!-- ======================================================== -->
+<!-- = Java Sourcecode to HTML automatically converted code = -->
+<!-- =   Java2Html Converter V4.1 2004 by Markus Gebhard  markus@jave.de   = -->
+<!-- =     Further information: http://www.java2html.de     = -->
+<div align="left" class="java">
+<table border="0" cellpadding="3" cellspacing="0" bgcolor="#ffffff">
+   <tr>
+  <!-- start source code -->
+   <td nowrap="nowrap" valign="top" align="left">
+    <code>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">Analyzer&nbsp;analyzer&nbsp;=&nbsp;</font><font color="#7f0055"><b>new&nbsp;</b></font><font color="#000000">StandardAnalyzer</font><font color="#000000">()</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#3f7f5f">//&nbsp;create&nbsp;an&nbsp;index&nbsp;in&nbsp;/tmp/index,&nbsp;overwriting&nbsp;an&nbsp;existing&nbsp;one:</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">IndexModifier&nbsp;indexModifier&nbsp;=&nbsp;</font><font color="#7f0055"><b>new&nbsp;</b></font><font color="#000000">IndexModifier</font><font color="#000000">(</font><font color="#2a00ff">&#34;/tmp/index&#34;</font><font color="#000000">,&nbsp;analyzer,&nbsp;</font><font color="#7f0055"><b>true</b></font><font color="#000000">)</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">Document&nbsp;doc&nbsp;=&nbsp;</font><font color="#7f0055"><b>new&nbsp;</b></font><font color="#000000">Document</font><font color="#000000">()</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">doc.add</font><font color="#000000">(</font><font color="#7f0055"><b>new&nbsp;</b></font><font color="#000000">Field</font><font color="#000000">(</font><font color="#2a00ff">&#34;id&#34;</font><font color="#000000">,&nbsp;</font><font color="#2a00ff">&#34;1&#34;</font><font color="#000000">,&nbsp;Field.Store.YES,&nbsp;Field.Index.NOT_ANALYZED</font><font color="#000000">))</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">doc.add</font><font color="#000000">(</font><font color="#7f0055"><b>new&nbsp;</b></font><font color="#000000">Field</font><font color="#000000">(</font><font color="#2a00ff">&#34;body&#34;</font><font color="#000000">,&nbsp;</font><font color="#2a00ff">&#34;a&nbsp;simple&nbsp;test&#34;</font><font color="#000000">,&nbsp;Field.Store.YES,&nbsp;Field.Index.ANALYZED</font><font color="#000000">))</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">indexModifier.addDocument</font><font color="#000000">(</font><font color="#000000">doc</font><font color="#000000">)</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#7f0055"><b>int&nbsp;</b></font><font color="#000000">deleted&nbsp;=&nbsp;indexModifier.delete</font><font color="#000000">(</font><font color="#7f0055"><b>new&nbsp;</b></font><font color="#000000">Term</font><font color="#000000">(</font><font color="#2a00ff">&#34;id&#34;</font><font color="#000000">,&nbsp;</font><font color="#2a00ff">&#34;1&#34;</font><font color="#000000">))</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">System.out.println</font><font color="#000000">(</font><font color="#2a00ff">&#34;Deleted&nbsp;&#34;&nbsp;</font><font color="#000000">+&nbsp;deleted&nbsp;+&nbsp;</font><font color="#2a00ff">&#34;&nbsp;document&#34;</font><font color="#000000">)</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">indexModifier.flush</font><font color="#000000">()</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">System.out.println</font><font color="#000000">(</font><font color="#000000">indexModifier.docCount</font><font color="#000000">()&nbsp;</font><font color="#000000">+&nbsp;</font><font color="#2a00ff">&#34;&nbsp;docs&nbsp;in&nbsp;index&#34;</font><font color="#000000">)</font><font color="#000000">;</font><br/>
+<font color="#ffffff">&nbsp;&nbsp;&nbsp;&nbsp;</font><font color="#000000">indexModifier.close</font><font color="#000000">()</font><font color="#000000">;</font></code>
+
+   </td>
+  <!-- end source code -->
+   </tr>
+</table>
+</div>
+<!-- =       END of automatically generated HTML code       = -->
+<!-- ======================================================== -->
+ *
+ * <p>Not all methods of IndexReader and IndexWriter are offered by this
+ * class. If you need access to additional methods, either use those classes
+ * directly or implement your own class that extends <code>IndexModifier</code>.
+ *
+ * <p>Although an instance of this class can be used from more than one
+ * thread, you will not get the best performance. You might want to use
+ * IndexReader and IndexWriter directly for that (but you will need to
+ * care about synchronization yourself then).
+ *
+ * <p>While you can freely mix calls to add() and delete() using this class,
+ * you should batch you calls for best performance. For example, if you
+ * want to update 20 documents, you should first delete all those documents,
+ * then add all the new documents.
+ *
+ * @deprecated Please use {@link IndexWriter} instead.
+ */
+public class IndexModifier {
+
+  protected IndexWriter indexWriter = null;
+  protected IndexReader indexReader = null;
+
+  protected Directory directory = null;
+  protected Analyzer analyzer = null;
+  protected boolean open = false, closeDir = false;
+
+  // Lucene defaults:
+  protected PrintStream infoStream = null;
+  protected boolean useCompoundFile = true;
+
+  protected int maxBufferedDocs = IndexWriter.DEFAULT_MAX_BUFFERED_DOCS;
+  protected int maxFieldLength = IndexWriter.DEFAULT_MAX_FIELD_LENGTH;
+    protected int mergeFactor = LogMergePolicy.DEFAULT_MERGE_FACTOR;
+
+  /**
+   * Open an index with write access.
+   *
+   * @param directory the index directory
+   * @param analyzer the analyzer to use for adding new documents
+   * @param create <code>true</code> to create the index or overwrite the existing one;
+   * 	<code>false</code> to append to the existing index
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public IndexModifier(Directory directory, Analyzer analyzer, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException {
+    init(directory, analyzer, create);
+  }
+
+  /**
+   * Open an index with write access.
+   *
+   * @param dirName the index directory
+   * @param analyzer the analyzer to use for adding new documents
+   * @param create <code>true</code> to create the index or overwrite the existing one;
+   * 	<code>false</code> to append to the existing index
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public IndexModifier(String dirName, Analyzer analyzer, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException {
+      Directory dir = FSDirectory.open(new File(dirName));
+      this.closeDir = true;
+      init(dir, analyzer, create);
+  }
+
+  /**
+   * Open an index with write access.
+   *
+   * @param file the index directory
+   * @param analyzer the analyzer to use for adding new documents
+   * @param create <code>true</code> to create the index or overwrite the existing one;
+   * 	<code>false</code> to append to the existing index
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public IndexModifier(File file, Analyzer analyzer, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException {
+      Directory dir = FSDirectory.open(file);
+      this.closeDir = true;
+      init(dir, analyzer, create);
+  }
+
+  /**
+   * Initialize an IndexWriter.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  protected void init(Directory directory, Analyzer analyzer, boolean create) throws CorruptIndexException, LockObtainFailedException, IOException {
+    this.directory = directory;
+    synchronized(this.directory) {
+      this.analyzer = analyzer;
+      indexWriter = new IndexWriter(directory, analyzer, create, IndexWriter.MaxFieldLength.LIMITED);
+      open = true;
+    }
+  }
+
+  /**
+   * Throw an IllegalStateException if the index is closed.
+   * @throws IllegalStateException
+   */
+  protected void assureOpen() {
+    if (!open) {
+      throw new IllegalStateException("Index is closed");
+    }
+  }
+
+  /**
+   * Close the IndexReader and open an IndexWriter.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  protected void createIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
+    if (indexWriter == null) {
+      if (indexReader != null) {
+        indexReader.close();
+        indexReader = null;
+      }
+      indexWriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(maxFieldLength));
+      // IndexModifier cannot use ConcurrentMergeScheduler
+      // because it synchronizes on the directory which can
+      // cause deadlock
+      indexWriter.setMergeScheduler(new SerialMergeScheduler());
+      indexWriter.setInfoStream(infoStream);
+      indexWriter.setUseCompoundFile(useCompoundFile);
+      if (maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH)
+        indexWriter.setMaxBufferedDocs(maxBufferedDocs);
+      indexWriter.setMergeFactor(mergeFactor);
+    }
+  }
+
+  /**
+   * Close the IndexWriter and open an IndexReader.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  protected void createIndexReader() throws CorruptIndexException, IOException {
+    if (indexReader == null) {
+      if (indexWriter != null) {
+        indexWriter.close();
+        indexWriter = null;
+      }
+      indexReader = IndexReader.open(directory);
+    }
+  }
+
+  /**
+   * Make sure all changes are written to disk.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public void flush() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        indexWriter.close();
+        indexWriter = null;
+        createIndexWriter();
+      } else {
+        indexReader.close();
+        indexReader = null;
+        createIndexReader();
+      }
+    }
+  }
+
+  /**
+   * Adds a document to this index, using the provided analyzer instead of the
+   * one specific in the constructor.  If the document contains more than
+   * {@link #setMaxFieldLength(int)} terms for a given field, the remainder are
+   * discarded.
+   * @see IndexWriter#addDocument(Document, Analyzer)
+   * @throws IllegalStateException if the index is closed
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public void addDocument(Document doc, Analyzer docAnalyzer) throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      if (docAnalyzer != null)
+        indexWriter.addDocument(doc, docAnalyzer);
+      else
+        indexWriter.addDocument(doc);
+    }
+  }
+
+  /**
+   * Adds a document to this index.  If the document contains more than
+   * {@link #setMaxFieldLength(int)} terms for a given field, the remainder are
+   * discarded.
+   * @see IndexWriter#addDocument(Document)
+   * @throws IllegalStateException if the index is closed
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public void addDocument(Document doc) throws CorruptIndexException, LockObtainFailedException, IOException {
+    addDocument(doc, null);
+  }
+
+  /**
+   * Deletes all documents containing <code>term</code>.
+   * This is useful if one uses a document field to hold a unique ID string for
+   * the document.  Then to delete such a document, one merely constructs a
+   * term with the appropriate field and the unique ID string as its text and
+   * passes it to this method.  Returns the number of documents deleted.
+   * @return the number of documents deleted
+   * @see IndexReader#deleteDocuments(Term)
+   * @throws IllegalStateException if the index is closed
+   * @throws StaleReaderException if the index has changed
+   *  since this reader was opened
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexReader();
+      return indexReader.deleteDocuments(term);
+    }
+  }
+
+  /**
+   * Deletes the document numbered <code>docNum</code>.
+   * @see IndexReader#deleteDocument(int)
+   * @throws StaleReaderException if the index has changed
+   *  since this reader was opened
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IllegalStateException if the index is closed
+   */
+  public void deleteDocument(int docNum) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexReader();
+      indexReader.deleteDocument(docNum);
+    }
+  }
+
+
+  /**
+   * Returns the number of documents currently in this
+   * index.  If the writer is currently open, this returns
+   * {@link IndexWriter#docCount()}, else {@link
+   * IndexReader#numDocs()}.  But, note that {@link
+   * IndexWriter#docCount()} does not take deletions into
+   * account, unlike {@link IndexReader#numDocs}.
+   * @throws IllegalStateException if the index is closed
+   */
+  public int numDocs() throws java.io.IOException {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        return indexWriter.numDocs();
+      } else {
+        return indexReader.numDocs();
+      }
+    }
+  }
+
+  /**
+   * Merges all segments together into a single segment, optimizing an index
+   * for search.
+   * @see IndexWriter#optimize()
+   * @throws IllegalStateException if the index is closed
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public void optimize() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      indexWriter.optimize();
+    }
+  }
+
+  /**
+   * If non-null, information about merges and a message when
+   * {@link #getMaxFieldLength()} is reached will be printed to this.
+   * <p>Example: <tt>index.setInfoStream(System.err);</tt>
+   * @see IndexWriter#setInfoStream(PrintStream)
+   * @throws IllegalStateException if the index is closed
+   */
+  public void setInfoStream(PrintStream infoStream) throws java.io.IOException {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        indexWriter.setInfoStream(infoStream);
+      }
+      this.infoStream = infoStream;
+    }
+  }
+
+  /**
+   * @see IndexModifier#setInfoStream(PrintStream)
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public PrintStream getInfoStream() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      return indexWriter.getInfoStream();
+    }
+  }
+
+  /**
+   * Setting to turn on usage of a compound file. When on, multiple files
+   * for each segment are merged into a single file once the segment creation
+   * is finished. This is done regardless of what directory is in use.
+   * @see IndexWriter#setUseCompoundFile(boolean)
+   * @throws IllegalStateException if the index is closed
+   */
+  public void setUseCompoundFile(boolean useCompoundFile) {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        indexWriter.setUseCompoundFile(useCompoundFile);
+      }
+      this.useCompoundFile = useCompoundFile;
+    }
+  }
+
+  /**
+   * @see IndexModifier#setUseCompoundFile(boolean)
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public boolean getUseCompoundFile() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      return indexWriter.getUseCompoundFile();
+    }
+  }
+
+  /**
+   * The maximum number of terms that will be indexed for a single field in a
+   * document.  This limits the amount of memory required for indexing, so that
+   * collections with very large files will not crash the indexing process by
+   * running out of memory.<p/>
+   * Note that this effectively truncates large documents, excluding from the
+   * index terms that occur further in the document.  If you know your source
+   * documents are large, be sure to set this value high enough to accommodate
+   * the expected size.  If you set it to Integer.MAX_VALUE, then the only limit
+   * is your memory, but you should anticipate an OutOfMemoryError.<p/>
+   * By default, no more than 10,000 terms will be indexed for a field.
+   * @see IndexWriter#setMaxFieldLength(int)
+   * @throws IllegalStateException if the index is closed
+   */
+  public void setMaxFieldLength(int maxFieldLength) {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        indexWriter.setMaxFieldLength(maxFieldLength);
+      }
+      this.maxFieldLength = maxFieldLength;
+    }
+  }
+
+  /**
+   * @see IndexModifier#setMaxFieldLength(int)
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public int getMaxFieldLength() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      return indexWriter.getMaxFieldLength();
+    }
+  }
+
+  /**
+   * Determines the minimal number of documents required before the buffered
+   * in-memory documents are merging and a new Segment is created.
+   * Since Documents are merged in a {@link org.apache.lucene.store.RAMDirectory},
+   * large value gives faster indexing.  At the same time, mergeFactor limits
+   * the number of files open in a FSDirectory.
+   *
+   * <p>The default value is 10.
+   *
+   * @see IndexWriter#setMaxBufferedDocs(int)
+   * @throws IllegalStateException if the index is closed
+   * @throws IllegalArgumentException if maxBufferedDocs is smaller than 2
+   */
+  public void setMaxBufferedDocs(int maxBufferedDocs) {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        indexWriter.setMaxBufferedDocs(maxBufferedDocs);
+      }
+      this.maxBufferedDocs = maxBufferedDocs;
+    }
+  }
+
+  /**
+   * @see IndexModifier#setMaxBufferedDocs(int)
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public int getMaxBufferedDocs() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      return indexWriter.getMaxBufferedDocs();
+    }
+  }
+
+  /**
+   * Determines how often segment indices are merged by addDocument().  With
+   * smaller values, less RAM is used while indexing, and searches on
+   * unoptimized indices are faster, but indexing speed is slower.  With larger
+   * values, more RAM is used during indexing, and while searches on unoptimized
+   * indices are slower, indexing is faster.  Thus larger values (&gt; 10) are best
+   * for batch index creation, and smaller values (&lt; 10) for indices that are
+   * interactively maintained.
+   * <p>This must never be less than 2.  The default value is 10.
+   *
+   * @see IndexWriter#setMergeFactor(int)
+   * @throws IllegalStateException if the index is closed
+   */
+  public void setMergeFactor(int mergeFactor) {
+    synchronized(directory) {
+      assureOpen();
+      if (indexWriter != null) {
+        indexWriter.setMergeFactor(mergeFactor);
+      }
+      this.mergeFactor = mergeFactor;
+    }
+  }
+
+  /**
+   * @see IndexModifier#setMergeFactor(int)
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws LockObtainFailedException if another writer
+   *  has this index open (<code>write.lock</code> could not
+   *  be obtained)
+   * @throws IOException if there is a low-level IO error
+   */
+  public int getMergeFactor() throws CorruptIndexException, LockObtainFailedException, IOException {
+    synchronized(directory) {
+      assureOpen();
+      createIndexWriter();
+      return indexWriter.getMergeFactor();
+    }
+  }
+
+  /**
+   * Close this index, writing all pending changes to disk.
+   *
+   * @throws IllegalStateException if the index has been closed before already
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public void close() throws CorruptIndexException, IOException {
+    synchronized(directory) {
+      if (!open)
+        throw new IllegalStateException("Index is closed already");
+      if (indexWriter != null) {
+        indexWriter.close();
+        indexWriter = null;
+      } else if (indexReader != null) {
+        indexReader.close();
+        indexReader = null;
+      }
+      open = false;
+      if (closeDir) {
+        directory.close();
+      }
+      closeDir = false;
+    }
+  }
+
+  public String toString() {
+    return "Index@" + directory;
+  }
+
+  /*
+  // used as an example in the javadoc:
+  public static void main(String[] args) throws IOException {
+    Analyzer analyzer = new StandardAnalyzer();
+    // create an index in /tmp/index, overwriting an existing one:
+    IndexModifier indexModifier = new IndexModifier("/tmp/index", analyzer, true);
+    Document doc = new Document();
+    doc.add(new Fieldable("id", "1", Fieldable.Store.YES, Fieldable.Index.NOT_ANALYZED));
+    doc.add(new Fieldable("body", "a simple test", Fieldable.Store.YES, Fieldable.Index.ANALYZED));
+    indexModifier.addDocument(doc);
+    int deleted = indexModifier.delete(new Term("id", "1"));
+    System.out.println("Deleted " + deleted + " document");
+    indexModifier.flush();
+    System.out.println(indexModifier.docCount() + " docs in index");
+    indexModifier.close();
+  }*/
+  
+}
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/LuceneQueryResult.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/LuceneQueryResult.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/LuceneQueryResult.java	(revision 29146)
@@ -0,0 +1,41 @@
+/**********************************************************************
+ *
+ * LuceneQueryResult.java 
+ *
+ * Copyright 2007 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+package org.greenstone.LuceneWrapper3;
+
+import java.util.Vector;
+
+/** Opportunity to fine tune QueryResult for lucene search
+ *
+ */
+
+public class LuceneQueryResult extends SharedSoleneQueryResult {
+    
+    // Currently no fine tuning -- rely on underlying shared Solr/Lucene base class
+    LuceneQueryResult() {
+	super();
+    } 
+}
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/SharedSoleneQuery.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/SharedSoleneQuery.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/SharedSoleneQuery.java	(revision 29146)
@@ -0,0 +1,135 @@
+/**********************************************************************
+ *
+ * SharedSoleneQuery.java 
+ *
+ * Copyright 2004 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+package org.greenstone.LuceneWrapper3;
+
+
+import java.io.*;
+import java.util.*;
+import java.util.regex.*;
+
+
+
+abstract public class SharedSoleneQuery
+{
+    static protected String TEXTFIELD = "TX";
+
+    protected String default_conjunction_operator = "OR";
+    protected String fuzziness = null;
+    protected String sort_field = null;
+    protected String filter_string = null;
+    protected int start_results=1;
+    protected int end_results=Integer.MAX_VALUE;
+
+    static protected PrintWriter utf8out = null;
+
+    static
+    {
+	try {
+	    OutputStreamWriter osw = new OutputStreamWriter(System.out, "UTF-8");
+	    utf8out = new PrintWriter(osw, true);
+	}
+        catch (UnsupportedEncodingException e) { 
+	    System.out.println(e); 
+	}
+    }
+
+    
+    public SharedSoleneQuery() {
+	// nothing currently to do in shared base class to Solr/Lucene
+    }
+    
+    
+    public boolean initialise() {
+	return true;
+    }
+    
+    abstract public SharedSoleneQueryResult runQuery(String query_string);
+    
+
+    public void setDefaultConjunctionOperator(String default_conjunction_operator) {
+	this.default_conjunction_operator = default_conjunction_operator.toUpperCase();
+    }
+    
+    public String getDefaultConjunctionOperator() {
+	return this.default_conjunction_operator;
+    }
+    
+    public void setEndResults(int end_results) {
+	this.end_results = end_results;
+    }
+    public int getEndResults() {
+	return this.end_results;
+    }
+        
+    public void setFilterString(String filter_string) {
+	this.filter_string = filter_string;
+    }
+    public String getFilterString() {
+	return this.filter_string ;
+    }
+        
+    public void setFuzziness(String fuzziness) {
+	this.fuzziness = fuzziness;
+    }
+    public String getFuzziness() {
+	return this.fuzziness;
+    }
+    
+    public void setSortField(String sort_field) {
+	this.sort_field = sort_field;
+    }
+    public String getSortField() {
+	return this.sort_field;
+    }
+        
+    public void setStartResults(int start_results) {
+	if (start_results < 1) {
+	    start_results = 1;
+	}
+	this.start_results = start_results;
+    }
+    public int getStartResults() {
+	return this.start_results;
+    }
+        
+    public void cleanUp() {
+	// nothing currently to do in shared Solr/Lucene base class
+    }
+
+    protected void finalize() throws Throwable 
+    {
+	try {
+	    utf8out.flush(); 
+	} finally {
+	    super.finalize();
+	}
+    }
+
+
+    
+}
+
+
Index: /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/SharedSoleneQueryResult.java
===================================================================
--- /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/SharedSoleneQueryResult.java	(revision 29146)
+++ /main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/SharedSoleneQueryResult.java	(revision 29146)
@@ -0,0 +1,259 @@
+/**********************************************************************
+ *
+ * SharedSoleneQueryResult.java 
+ *
+ * Copyright 2007 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+package org.greenstone.LuceneWrapper3;
+
+import java.util.Vector;
+
+/** a QueryResult class for a lucene search
+ *
+ */
+public class SharedSoleneQueryResult {
+    
+    public static final int NO_ERROR = 0;
+    public static final int PARSE_ERROR = 1;
+    public static final int TOO_MANY_CLAUSES_ERROR = 2;
+    public static final int IO_ERROR = 3;
+    public static final int SERVER_ERROR = 4;
+    public static final int OTHER_ERROR = 5;
+    
+    /** the list of DocInfo */
+    protected Vector docs_=null;
+    /** the list of TermInfo */
+    protected Vector terms_=null;
+    /** the list of stopwords found in the query */
+    protected Vector stopwords_ = null;
+    /** the total number of docs found - not necessarily the size of docs_*/
+    protected int total_num_docs_=0;
+    /** the start result number if we are retrieving only a portion of the results */
+    protected int start_results_ = 0;
+    /** the end result number if we are retrieving only a portion of the results */
+    protected int end_results_ = 0;
+    /** whether an error has occurred and what kind it is*/
+    protected int error_ = NO_ERROR;
+
+    public SharedSoleneQueryResult() {
+	docs_ = new Vector();
+	terms_ = new Vector();
+	stopwords_ = new Vector();
+    }
+    
+    /** clear the info from the last query - should be called before setting any new docs/terms */
+    public void clear() {
+	total_num_docs_=0;
+	docs_.clear();
+	terms_.clear();
+	stopwords_.clear();
+	error_ = NO_ERROR;
+    }
+
+    /** returns the result as a String - useful for printing out results */
+    public String toString() {
+	
+	String result = "";
+	result += "docs (ranks): ";
+	for (int i=0; i<docs_.size(); i++) {
+	    result += ((DocInfo)docs_.elementAt(i)).toString()+", ";
+	}
+	result += "\nterms: ";
+	for (int i=0; i<terms_.size(); i++) {
+	    result += ((TermInfo)terms_.elementAt(i)).toString()+", ";
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	
+	return result;
+    }
+    /** a shorter representation - just terms and total docs - not the 
+	individual docnums and ranks */
+    public String toShortString() {
+	String result = "";
+	result += "\nterms: ";
+	for (int i=0; i<terms_.size(); i++) {
+	    result += ((TermInfo)terms_.elementAt(i)).toString()+", ";
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	return result;
+    }
+    
+    public void setTotalDocs(int num) {
+	total_num_docs_=num;
+    }
+    
+    public void setStartResults(int start) {
+	start_results_ = start;
+    }
+
+    public void setEndResults(int end) {
+	end_results_ = end;
+    }
+
+    public void addDoc(String id, float rank, int termfreq)
+    {
+	docs_.add(new DocInfo(id, rank, termfreq));
+    }
+    
+    public void addTerm(String term, String field, int match, int freq) {
+	TermInfo ti = new TermInfo();
+	ti.term_=term;
+	ti.field_=field;
+	ti.match_docs_=match;
+	ti.term_freq_=freq;
+	terms_.add(ti);
+    }
+    public void addStopWord(String stopword) {
+	stopwords_.add(stopword);
+    }
+    public Vector getDocs() {
+	return docs_;
+    }
+    
+    public int getError() {
+	return error_;
+    }
+    
+    public String getErrorString() {
+	if (error_ == PARSE_ERROR) {
+	    return "PARSE_EXCEPTION";
+	}
+	if (error_ == TOO_MANY_CLAUSES_ERROR) {
+	    return "TOO_MANY_CLAUSES";
+	}
+	if (error_ == IO_ERROR) {
+	    return "IO_ERROR";
+	}
+	if (error_ == NO_ERROR) {
+	    return "NO_ERROR";
+	}
+	return "UNKNOWN";
+    }
+
+    public Vector getTerms() {
+	return terms_;
+    }
+    
+    public Vector getStopWords() {
+	return stopwords_;
+    }
+    public int getTotalDocs() {
+	return total_num_docs_;
+    }
+    
+    public void setError(int error) {
+	error_ = error;
+    }
+    
+    public String getXMLString() {
+	StringBuffer buffer = new StringBuffer();
+
+	// terms
+	buffer.append("<QueryTermsInfo num=\"" + terms_.size() + "\"/>\n");
+	for (int i=0; i<terms_.size(); i++) {
+	    buffer.append(((TermInfo)terms_.elementAt(i)).toXMLString()+"\n");
+	}
+
+	// stopwords
+	for (int i=0; i<stopwords_.size(); i++) {
+	    buffer.append("<StopWord value=\"" + (String)stopwords_.elementAt(i)+"\" />\n");
+	}
+	
+	// results
+	buffer.append("<MatchingDocsInfo num=\"" + total_num_docs_ + "\"/>\n");
+	buffer.append("<StartResults num=\"" + start_results_ + "\"/>\n");
+	buffer.append("<EndResults num=\"" + end_results_ + "\"/>\n");
+	
+	for (int i=0; i< docs_.size(); i++) {
+	    buffer.append(((DocInfo)docs_.elementAt(i)).toXMLString()+"\n");
+	}
+
+	return buffer.toString();
+    }
+
+ 
+    public class TermInfo {
+	
+	/** the term itself */
+	public String term_=null;
+	/** the field for which this term was queried */
+	public String field_=null;
+	/** the number of documents containing this term */
+	public int match_docs_=0;
+	/** overall term freq for this term */
+	public int term_freq_=0;
+	
+	public TermInfo() {
+	}
+	
+	/** output the class as a string */
+	public String toString() {
+	    String result="";
+	    result +="<"+field_+">\""+term_+" docs("+match_docs_;
+	    result +=")freq("+term_freq_+")";
+	    return result;
+	}
+
+	/** output as an XML element */
+	public String toXMLString() {
+	    return "<Term value=\"" + xmlSafe(term_) + "\" field=\"" + field_ + "\" freq=\"" + term_freq_ + "\" />";
+	}
+    }
+
+
+    public class DocInfo
+    {
+	public String id_ = "";
+	public float rank_ = 0;
+	public int termfreq_ = 0;
+
+	public DocInfo (String id, float rank, int termfreq)
+	{
+	    id_ = id;
+	    rank_ = rank;
+	    termfreq_ = termfreq;
+	}
+
+	public String toString()
+	{
+	    return "" + id_ + " (" + rank_ + ") (" + termfreq_ + ")";
+	}
+
+	public String toXMLString()
+	{
+	    return "<Match id=\"" + id_ + "\" rank=\"" + rank_ + "\" termfreq=\"" + termfreq_ + "\" />";
+	}
+    }
+
+
+    // where should this go???
+    public static String xmlSafe(String text) {
+	text = text.replaceAll("&","&amp;amp;");
+	text = text.replaceAll("<","&amp;lt;");
+	text = text.replaceAll(">","&amp;gt;");
+	text = text.replaceAll("'","&amp;#039;");
+	text = text.replaceAll("\\\"","&amp;quot;");
+	return text;
+    }
+ 
+}
