Changeset 26157 for main/trunk

Timestamp:

2012-09-06T16:54:10+12:00 (12 years ago)

Author:

ak19

Message:

A much better way to ensure searches with wild cards get expanded to query terms as in GS2 which uses an older version of lucene (2.3.2). It now works as before at both document level and section level searches with wild cards. If a BooleanQuery.TooManyClauses exception is thrown, it will try rewriting the query once more, but using other rewritemethods, so that results are still returned (instead of the exception forcing it to say 0 documents are returned).

File:

• main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneQuery.java (modified) (4 diffs)

main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper3/GS2LuceneQuery.java

@@ -39 +39 @@
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanQuery.TooManyClauses;
+import org.apache.lucene.search.BooleanQuery; // for the TooManyClauses exception
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.MultiTermQuery.ConstantScoreAutoRewrite;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeFilter;
@@ -54 +56 @@
 import org.apache.lucene.util.Version;
 
-import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.MultiTermQuery.ConstantScoreAutoRewrite;
 
 public class GS2LuceneQuery extends SharedSoleneQuery
@@ -152 +152 @@
 
         // GS2's LuceneWrapper uses lucene-2.3.2. GS3's LuceneWrapper3 works with lucene-3.3.0. 
-        // This change in lucene core library for GS3 had the side-effect that searching on 
-        // "econom*" didn't display what terms it was searching for, whereas it had done so in GS2. 
+        // This change in lucene core library for GS3 (present since after version 2.4.1) had the
+        // side-effect that searching on "econom*" didn't display what terms it was searching for, 
+        // whereas it had done so in GS2. 
 
         // The details of this problem and its current solution are explained in the ticket 
         // http://trac.greenstone.org/ticket/845
 
-        // We need to change the settings for rewriteMethod in order to get searches on wildcards to 
-        // produce search terms again when the query is rewritten.
+        // We need to change the settings for the rewriteMethod in order to get searches on wildcards
+        // to produce search terms again when the query gets rewritten.
+
+        // We try, in order:
+        // 1. RewriteMethod set to BooleanQuery, to get it working as in GS2 which uses lucene-2.3.2
+        // it will expand wildcard searches to its terms when searching at both section AND doc level.
+        // If that throws a TooManyClauses exception (like when searching for "a*" over lucene demo collection)
+        // 2. Then try a custom rewriteMethod which sets termCountCutoff=350 and docCountPercent cutoff=0.1%
+        // If that throws a TooManyClauses exception (could perhaps happen if the collection has a huge number of docs
+        // 3. Then try the default apache rewriteMethod with its optimum defaults of 
+        // termCountCutoff=350 and docCountPercent cutoff=0.1%
+        //  See http://lucene.apache.org/core/3_6_1/api/core/org/apache/lucene/search/MultiTermQuery.html
 
         if(query instanceof MultiTermQuery) {
-
-        // default docCountPercent=0.1; default termCountCutoff=350
-
-        // Creating custom cutoff values, taking into account of existing cutoff values
-        MultiTermQuery.ConstantScoreAutoRewrite customRewriteMethod = new MultiTermQuery.ConstantScoreAutoRewrite();
-        customRewriteMethod.setDocCountPercent(100.0);//MultiTermQuery.ConstantScoreAutoRewrite.DEFAULT_DOC_COUNT_PERCENT);
-        customRewriteMethod.setTermCountCutoff(350);
-
         MultiTermQuery multiTermQuery = (MultiTermQuery)query;
-        multiTermQuery.setRewriteMethod(customRewriteMethod);
-
-        // the above works when searching with wildcards over sections, the following also 
-        // works on book searches, but has been discouraged as it can throw an exception if 
-        // the number of terms exceeds BooleanQuery.getMaxClauseCount().
-        // http://lucene.apache.org/core/3_6_1/api/core/org/apache/lucene/search/MultiTermQuery.html
-
-        //multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);//MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); 
-        }
-
-        query = query.rewrite(reader);
-        
+        multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
+             // less CPU intensive than MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE)
+        }
+
+        try {
+        query = query.rewrite(reader);
+        } 
+        catch(BooleanQuery.TooManyClauses clauseException) {
+        // Example test case: try searching the lucene demo collection for "a*" 
+        // and you'll hit this exception
+
+        lucene_query_result.setError(LuceneQueryResult.TOO_MANY_CLAUSES_ERROR);
+
+        if(query instanceof MultiTermQuery) {
+
+            // CustomRewriteMethod: setting the docCountPercent cutoff to a custom 100%. 
+            // This will at least expand the query to its terms when searching with wildcards at section-level 
+            // (though it doesn't seem to work for doc-level searches, no matter what the cutoffs are set to).
+
+            MultiTermQuery.ConstantScoreAutoRewrite customRewriteMethod = new MultiTermQuery.ConstantScoreAutoRewrite();
+            customRewriteMethod.setDocCountPercent(100.0);
+            customRewriteMethod.setTermCountCutoff(350); // same as default
+            
+            MultiTermQuery multiTermQuery = (MultiTermQuery)query;
+            multiTermQuery.setRewriteMethod(customRewriteMethod);
+            try {
+            query = query.rewrite(reader);
+            } 
+            catch(BooleanQuery.TooManyClauses clauseExceptionAgain) {
+
+            // do what the code originally did: use the default rewriteMethod which
+            // uses a default docCountPercent=0.1 (%) and termCountCutoff=350
+
+            multiTermQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT);
+            query = query.rewrite(reader);
+            }
+        }
+        }
+
         // Get the list of expanded query terms and their frequencies 
         // num docs matching, and total frequency       
@@ -285 +315 @@
         lucene_query_result.setError(LuceneQueryResult.PARSE_ERROR);
     }
-    catch (TooManyClauses too_many_clauses_exception) {
+    catch (BooleanQuery.TooManyClauses too_many_clauses_exception) {
         lucene_query_result.setError(LuceneQueryResult.TOO_MANY_CLAUSES_ERROR);
     }