Changeset 12372 for trunk/indexers/lucene-gs

Timestamp:

2006-08-02T12:45:56+12:00 (18 years ago)

Author:

mdewsnip

Message:

Now returns the stop words that have been removed from the query.

File:

• trunk/indexers/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneQuery.java (modified) (4 diffs)

trunk/indexers/lucene-gs/src/org/greenstone/LuceneWrapper/GS2LuceneQuery.java

@@ -38 +38 @@
         try {
         Searcher searcher = new IndexSearcher(args[0]);
+        IndexReader reader = ((IndexSearcher) searcher).getIndexReader();
+
         Sort sorter = new Sort();
+
             // New code to allow the default conjunction operator to be
             // definable
@@ -56 +59 @@
                 }
 
-        Analyzer analyzer = new StandardAnalyzer();
-        IndexReader reader = ((IndexSearcher) searcher).getIndexReader();
+        // Create one query parser with the standard set of stop words, and one with none
+        QueryParser query_parser = new QueryParser("TX", new StandardAnalyzer());
+        QueryParser query_parser_no_stop_words = new QueryParser("TX", new StandardAnalyzer(new String[] { }));
+
+        // Lucene does "OR" queries by default; do an "AND" query if specified
+        if (default_conjuction_operator.equals("AND")) {
+        query_parser.setDefaultOperator(query_parser.AND_OPERATOR);
+        query_parser_no_stop_words.setDefaultOperator(query_parser.AND_OPERATOR);
+        }
 
         BufferedReader in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
@@ -69 +79 @@
 
         // Parse the query and rewrite it into individual terms (eg. for wildcard searches)
-        QueryParser query_parser = new QueryParser("TX", analyzer);
-                // Set the default conjuction operator
-                System.err.println("**** DCO = " + default_conjuction_operator);
-                if (default_conjuction_operator.equals("AND"))
-                    {
-                        query_parser.setDefaultOperator(query_parser.AND_OPERATOR);
-                    }
-                // Otherwise its OR
-
         Query query = query_parser.parse(query_string);
         query = query.rewrite(reader);
+        Query query_including_stop_words = query_parser_no_stop_words.parse(query_string);
+        query_including_stop_words = query_including_stop_words.rewrite(reader);
 
         // Perform the query
@@ -90 +93 @@
         query.extractTerms(terms);
         System.out.println("  <QueryTermsInfo num=\"" + terms.size() + "\"/>");
-        Iterator iter = terms.iterator();
-        while (iter.hasNext()) {
-            Term term = (Term) iter.next();
-                    //System.out.println("  <Term value=\"" + term.text() + "\" freq=\"" + reader.docFreq(term) + "\"/>");
+        Iterator terms_iter = terms.iterator();
+        while (terms_iter.hasNext()) {
+            Term term = (Term) terms_iter.next();
             System.out.println("  <Term value=\"" + term.text() + "\" freq=\"" + reader.docFreq(term) + "\" field=\"" + term.field() + "\"/>");
+        }
+
+        // Return the list of stop words removed from the query
+        HashSet terms_including_stop_words = new HashSet();
+        query_including_stop_words.extractTerms(terms_including_stop_words);
+        System.out.println("  <StopWordsInfo num=\"" + (terms_including_stop_words.size() - terms.size()) + "\"/>");
+        Iterator terms_including_stop_words_iter = terms_including_stop_words.iterator();
+        while (terms_including_stop_words_iter.hasNext()) {
+            Term term = (Term) terms_including_stop_words_iter.next();
+            if (!terms.contains(term)) {
+            System.err.println("  <StopWord value=\"" + term.text() + "\"/>");
+            }
         }