source: main/trunk/greenstone2/common-src/indexers/lucene-gs/src/org/greenstone/LuceneWrapper/lucene-version-3.3/GS2LuceneQuery.java@ 24716

Last change on this file since 24716 was 24716, checked in by davidb, 13 years ago

A version of the LuceneWrapper code that works with Lucene version 3.3
File size: 20.5 KB
Line 
1/**********************************************************************
2 *
3 * GS2LuceneQuery.java
4 *
5 * Copyright 2004 The New Zealand Digital Library Project
6 *
7 * A component of the Greenstone digital library software
8 * from the New Zealand Digital Library Project at the
9 * University of Waikato, New Zealand.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 *
25 *********************************************************************/
26package org.greenstone.LuceneWrapper;
27
28
29import java.io.*;
30import java.util.*;
31import java.util.regex.*;
32
33import org.apache.lucene.analysis.Analyzer;
34import org.apache.lucene.analysis.standard.StandardAnalyzer;
35import org.apache.lucene.document.Document;
36import org.apache.lucene.index.IndexReader;
37import org.apache.lucene.index.Term;
38import org.apache.lucene.index.TermDocs;
39import org.apache.lucene.queryParser.ParseException;
40import org.apache.lucene.queryParser.QueryParser;
41import org.apache.lucene.search.BooleanQuery.TooManyClauses;
42import org.apache.lucene.search.Filter;
43import org.apache.lucene.search.Hit;
44import org.apache.lucene.search.Hits;
45import org.apache.lucene.search.IndexSearcher;
46import org.apache.lucene.search.Query;
47import org.apache.lucene.search.TermRangeFilter;
48import org.apache.lucene.search.Searcher;
49import org.apache.lucene.search.ScoreDoc;
50import org.apache.lucene.search.Sort;
51import org.apache.lucene.search.SortField;
52import org.apache.lucene.search.TopFieldDocs;
53
54import org.apache.lucene.store.Directory;
55import org.apache.lucene.store.FSDirectory;
56import org.apache.lucene.util.Version;
57
58public class GS2LuceneQuery extends SharedSoleneQuery
59{
60 protected String full_indexdir="";
61
62 protected Sort sorter=new Sort();
63 protected Filter filter = null;
64
65 protected static Version matchVersion = Version.LUCENE_24;
66
67 protected QueryParser query_parser = null;
68 protected QueryParser query_parser_no_stop_words = null;
69 protected Searcher searcher = null;
70 protected IndexReader reader = null;
71
72 public GS2LuceneQuery() {
73 super();
74
75 // Create one query parser with the standard set of stop words, and one with none
76
77 query_parser = new QueryParser(matchVersion, TEXTFIELD, new GS2Analyzer()); // uses built-in stop_words_set
78 query_parser_no_stop_words = new QueryParser(matchVersion, TEXTFIELD, new GS2Analyzer(new String[] { }));
79 }
80
81
82 public boolean initialise() {
83
84 if (!super.initialise()) {
85 return false;
86 }
87
88
89 if (full_indexdir==null || full_indexdir.length()==-1){
90 utf8out.println("Index directory is not indicated ");
91 utf8out.flush();
92 return false;
93 }
94
95 try {
96 Directory full_indexdir_dir = FSDirectory.open(new File(full_indexdir));
97 searcher = new IndexSearcher(full_indexdir_dir,true);
98 reader = ((IndexSearcher) searcher).getIndexReader();
99
100 }
101 catch (IOException exception) {
102 exception.printStackTrace();
103 return false;
104 }
105 return true;
106
107 }
108
109 public void setIndexDir(String full_indexdir) {
110 this.full_indexdir = full_indexdir;
111 }
112
113 public void setSortField(String sort_field) {
114 super.setSortField(sort_field);
115
116 if (sort_field == null) {
117 this.sorter = new Sort();
118 } else {
119 this.sorter = new Sort(new SortField(sort_field,SortField.STRING)); // **** can do better than this?!?
120 }
121 }
122
123 public void setFilterString(String filter_string) {
124 super.setFilterString(filter_string);
125 this.filter = parseFilterString(filter_string);
126 }
127
128 public Filter getFilter() {
129 return this.filter;
130 }
131
132
133 public LuceneQueryResult runQuery(String query_string) {
134
135 if (query_string == null || query_string.equals("")) {
136 utf8out.println("The query word is not indicated ");
137 utf8out.flush();
138 return null;
139 }
140
141 LuceneQueryResult lucene_query_result=new LuceneQueryResult();
142 lucene_query_result.clear();
143
144 try {
145 Query query_including_stop_words = query_parser_no_stop_words.parse(query_string);
146 query_including_stop_words = query_including_stop_words.rewrite(reader);
147
148 // System.err.println("********* query_string " + query_string + "****");
149
150 Query query = parseQuery(reader, query_parser, query_string, fuzziness);
151 query = query.rewrite(reader);
152
153 // Get the list of expanded query terms and their frequencies
154 // num docs matching, and total frequency
155 HashSet terms = new HashSet();
156 query.extractTerms(terms);
157
158 HashMap doc_term_freq_map = new HashMap();
159
160 Iterator iter = terms.iterator();
161 while (iter.hasNext()) {
162
163 Term term = (Term) iter.next();
164
165 // Get the term frequency over all the documents
166 TermDocs term_docs = reader.termDocs(term);
167 int term_freq = 0;
168 int match_docs = 0;
169 while (term_docs.next())
170 {
171 if (term_docs.freq() != 0)
172 {
173 term_freq += term_docs.freq();
174 match_docs++;
175
176 // Calculate the document-level term frequency as well
177 Integer lucene_doc_num_obj = new Integer(term_docs.doc());
178 int doc_term_freq = 0;
179 if (doc_term_freq_map.containsKey(lucene_doc_num_obj))
180 {
181 doc_term_freq = ((Integer) doc_term_freq_map.get(lucene_doc_num_obj)).intValue();
182 }
183 doc_term_freq += term_docs.freq();
184
185 doc_term_freq_map.put(lucene_doc_num_obj, new Integer(doc_term_freq));
186 }
187 }
188
189 // Create a term
190 lucene_query_result.addTerm(term.text(), term.field(), match_docs, term_freq);
191 }
192
193 // Get the list of stop words removed from the query
194 HashSet terms_including_stop_words = new HashSet();
195 query_including_stop_words.extractTerms(terms_including_stop_words);
196 Iterator terms_including_stop_words_iter = terms_including_stop_words.iterator();
197 while (terms_including_stop_words_iter.hasNext()) {
198 Term term = (Term) terms_including_stop_words_iter.next();
199 if (!terms.contains(term)) {
200 lucene_query_result.addStopWord(term.text());
201 }
202 }
203
204 // do the query
205 // Simple case for getting all the matching documents
206 if (end_results == Integer.MAX_VALUE) {
207 // Perform the query (filter and sorter may be null)
208 TopFieldDocs hits = searcher.search(query, filter, end_results, sorter);
209 lucene_query_result.setTotalDocs(hits.totalHits);
210
211 // Output the matching documents
212 lucene_query_result.setStartResults(start_results);
213 lucene_query_result.setEndResults(hits.totalHits);
214
215 for (int i = start_results; i <= hits.totalHits; i++) {
216 int lucene_doc_num = hits.scoreDocs[i - 1].doc;
217 Document doc = reader.document(lucene_doc_num);
218 int doc_term_freq = 0;
219 Integer doc_term_freq_object = (Integer) doc_term_freq_map.get(new Integer(lucene_doc_num));
220 if (doc_term_freq_object != null)
221 {
222 doc_term_freq = doc_term_freq_object.intValue();
223 }
224 lucene_query_result.addDoc(doc.get("docOID").trim(), hits.scoreDocs[i-1].score, doc_term_freq);
225 }
226 }
227
228 // Slightly more complicated case for returning a subset of the matching documents
229 else {
230 // Perform the query (filter may be null)
231 TopFieldDocs hits = searcher.search(query, filter, end_results, sorter);
232 lucene_query_result.setTotalDocs(hits.totalHits);
233
234 lucene_query_result.setStartResults(start_results);
235 lucene_query_result.setEndResults(end_results < hits.scoreDocs.length ? end_results: hits.scoreDocs.length);
236
237 // Output the matching documents
238 for (int i = start_results; (i <= hits.scoreDocs.length && i <= end_results); i++) {
239 int lucene_doc_num = hits.scoreDocs[i - 1].doc;
240 Document doc = reader.document(lucene_doc_num);
241 int doc_term_freq = 0;
242 Integer doc_term_freq_object = (Integer) doc_term_freq_map.get(new Integer(lucene_doc_num));
243 if (doc_term_freq_object != null)
244 {
245 doc_term_freq = doc_term_freq_object.intValue();
246 }
247 lucene_query_result.addDoc(doc.get("docOID").trim(), hits.scoreDocs[i-1].score, doc_term_freq);
248 }
249 }
250 }
251
252 catch (ParseException parse_exception) {
253 lucene_query_result.setError(LuceneQueryResult.PARSE_ERROR);
254 }
255 catch (TooManyClauses too_many_clauses_exception) {
256 lucene_query_result.setError(LuceneQueryResult.TOO_MANY_CLAUSES_ERROR);
257 }
258 catch (IOException exception) {
259 lucene_query_result.setError(LuceneQueryResult.IO_ERROR);
260 exception.printStackTrace();
261 }
262 catch (Exception exception) {
263 lucene_query_result.setError(LuceneQueryResult.OTHER_ERROR);
264 exception.printStackTrace();
265 }
266 return lucene_query_result;
267 }
268
269 public void setDefaultConjunctionOperator(String default_conjunction_operator) {
270 super.setDefaultConjunctionOperator(default_conjunction_operator);
271
272 if (default_conjunction_operator.equals("AND")) {
273 query_parser.setDefaultOperator(query_parser.AND_OPERATOR);
274 query_parser_no_stop_words.setDefaultOperator(query_parser.AND_OPERATOR);
275 } else { // default is OR
276 query_parser.setDefaultOperator(query_parser.OR_OPERATOR);
277 query_parser_no_stop_words.setDefaultOperator(query_parser.OR_OPERATOR);
278 }
279 }
280
281
282 public void cleanUp() {
283 super.cleanUp();
284 try {
285 if (searcher != null) {
286 searcher.close();
287 }
288 } catch (IOException exception) {
289 exception.printStackTrace();
290 }
291 }
292
293
294 protected Query parseQuery(IndexReader reader, QueryParser query_parser, String query_string, String fuzziness)
295 throws java.io.IOException, org.apache.lucene.queryParser.ParseException
296 {
297 // Split query string into the search terms and the filter terms
298 // * The first +(...) term contains the search terms so count
299 // up '(' and stop when we finish matching ')'
300 int offset = 0;
301 int paren_count = 0;
302 boolean seen_paren = false;
303 while (offset < query_string.length() && (!seen_paren || paren_count > 0)) {
304 if (query_string.charAt(offset) == '(') {
305 paren_count++;
306 seen_paren = true;
307 }
308 if (query_string.charAt(offset) == ')') {
309 paren_count--;
310 }
311 offset++;
312 }
313 String query_prefix = query_string.substring(0, offset);
314 String query_suffix = query_string.substring(offset);
315
316 ///ystem.err.println("Prefix: " + query_prefix);
317 ///ystem.err.println("Suffix: " + query_suffix);
318
319 Query query = query_parser.parse(query_prefix);
320 query = query.rewrite(reader);
321
322 // If this is a fuzzy search, then we need to add the fuzzy
323 // flag to each of the query terms
324 if (fuzziness != null && query.toString().length() > 0) {
325
326 // Revert the query to a string
327 System.err.println("Rewritten query: " + query.toString());
328 // Search through the string for TX:<term> query terms
329 // and append the ~ operator. Note that this search will
330 // not change phrase searches (TX:"<term> <term>") as
331 // fuzzy searching is not possible for these entries.
332 // Yahoo! Time for a state machine!
333 StringBuffer mutable_query_string = new StringBuffer(query.toString());
334 int o = 0; // Offset
335 // 0 = BASE, 1 = SEEN_T, 2 = SEEN_TX, 3 = SEEN_TX:
336 int s = 0; // State
337 while(o < mutable_query_string.length()) {
338 char c = mutable_query_string.charAt(o);
339 if (s == 0 && c == TEXTFIELD.charAt(0)) {
340 ///ystem.err.println("Found T!");
341 s = 1;
342 }
343 else if (s == 1) {
344 if (c == TEXTFIELD.charAt(1)) {
345 ///ystem.err.println("Found X!");
346 s = 2;
347 }
348 else {
349 s = 0; // Reset
350 }
351 }
352 else if (s == 2) {
353 if (c == ':') {
354 ///ystem.err.println("Found TX:!");
355 s = 3;
356 }
357 else {
358 s = 0; // Reset
359 }
360 }
361 else if (s == 3) {
362 // Don't process phrases
363 if (c == '"') {
364 ///ystem.err.println("Stupid phrase...");
365 s = 0; // Reset
366 }
367 // Found the end of the term... add the
368 // fuzzy search indicator
369 // Nor outside the scope of parentheses
370 else if (Character.isWhitespace(c) || c == ')') {
371 ///ystem.err.println("Yahoo! Found fuzzy term.");
372 mutable_query_string.insert(o, '~' + fuzziness);
373 o++;
374 s = 0; // Reset
375 }
376 }
377 o++;
378 }
379 // If we were in the state of looking for the end of a
380 // term - then we just found it!
381 if (s == 3) {
382
383 mutable_query_string.append('~' + fuzziness);
384 }
385 // Reparse the query
386 ///ystem.err.println("Fuzzy query: " + mutable_query_string.toString() + query_suffix);
387 query = query_parser.parse(mutable_query_string.toString() + query_suffix);
388 }
389 else {
390 query = query_parser.parse(query_prefix + query_suffix);
391 }
392
393 return query;
394 }
395
396 protected Filter parseFilterString(String filter_string)
397 {
398 Filter result = null;
399 Pattern pattern = Pattern.compile("\\s*\\+(\\w+)\\:([\\{\\[])(\\d+)\\s+TO\\s+(\\d+)([\\}\\]])\\s*");
400 Matcher matcher = pattern.matcher(filter_string);
401 if (matcher.matches()) {
402 String field_name = matcher.group(1);
403 boolean include_lower = matcher.group(2).equals("[");
404 String lower_term = matcher.group(3);
405 String upper_term = matcher.group(4);
406 boolean include_upper = matcher.group(5).equals("]");
407 result = new TermRangeFilter(field_name, lower_term, upper_term, include_lower, include_upper);
408 }
409 else {
410 System.err.println("Error: Could not understand filter string \"" + filter_string + "\"");
411 }
412 return result;
413 }
414
415
416 /** command line program and auxiliary methods */
417
418 // Fairly self-explanatory I should hope
419 static protected boolean query_result_caching_enabled = false;
420
421
422 static public void main (String args[])
423 {
424 if (args.length == 0) {
425 System.out.println("Usage: GS2LuceneQuery <index directory> [-fuzziness value] [-filter filter_string] [-sort sort_field] [-dco AND|OR] [-startresults number -endresults number] [query]");
426 return;
427 }
428
429 try {
430 String index_directory = args[0];
431
432 GS2LuceneQuery queryer = new GS2LuceneQuery();
433 queryer.setIndexDir(index_directory);
434
435 // Prepare the index cache directory, if query result caching is enabled
436 if (query_result_caching_enabled) {
437 // Make the index cache directory if it doesn't already exist
438 File index_cache_directory = new File(index_directory, "cache");
439 if (!index_cache_directory.exists()) {
440 index_cache_directory.mkdir();
441 }
442
443 // Disable caching if the index cache directory isn't available
444 if (!index_cache_directory.exists() || !index_cache_directory.isDirectory()) {
445 query_result_caching_enabled = false;
446 }
447 }
448
449 String query_string = null;
450
451 // Parse the command-line arguments
452 for (int i = 1; i < args.length; i++) {
453 if (args[i].equals("-sort")) {
454 i++;
455 queryer.setSortField(args[i]);
456 }
457 else if (args[i].equals("-filter")) {
458 i++;
459 queryer.setFilterString(args[i]);
460 }
461 else if (args[i].equals("-dco")) {
462 i++;
463 queryer.setDefaultConjunctionOperator(args[i]);
464 }
465 else if (args[i].equals("-fuzziness")) {
466 i++;
467 queryer.setFuzziness(args[i]);
468 }
469 else if (args[i].equals("-startresults")) {
470 i++;
471 if (args[i].matches("\\d+")) {
472 queryer.setStartResults(Integer.parseInt(args[i]));
473 }
474 }
475 else if (args[i].equals("-endresults")) {
476 i++;
477 if (args[i].matches("\\d+")) {
478 queryer.setEndResults(Integer.parseInt(args[i]));
479 }
480 }
481 else {
482 query_string = args[i];
483 }
484 }
485
486 if (!queryer.initialise()) {
487 return;
488 }
489
490 // The query string has been specified as a command-line argument
491 if (query_string != null) {
492 runQueryCaching(index_directory, queryer, query_string);
493 }
494
495 // Read queries from STDIN
496 else {
497 BufferedReader in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
498 while (true) {
499 // Read the query from STDIN
500 query_string = in.readLine();
501 if (query_string == null || query_string.length() == -1) {
502 break;
503 }
504
505 runQueryCaching(index_directory, queryer, query_string);
506
507 }
508 }
509 queryer.cleanUp();
510 }
511 catch (IOException exception) {
512 exception.printStackTrace();
513 }
514 }
515
516 protected static void runQueryCaching(String index_directory, GS2LuceneQuery queryer, String query_string)
517 throws IOException
518 {
519 StringBuffer query_results_xml = new StringBuffer();
520
521 // Check if this query result has been cached from a previous search (if it's enabled)
522 File query_result_cache_file = null;
523 if (query_result_caching_enabled) {
524 // Generate the cache file name from the query options
525 String query_result_cache_file_name = query_string + "-";
526 String fuzziness = queryer.getFuzziness();
527 query_result_cache_file_name += ((fuzziness != null) ? fuzziness : "") + "-";
528 String filter_string = queryer.getFilterString();
529 query_result_cache_file_name += ((filter_string != null) ? filter_string : "") + "-";
530 String sort_string = queryer.getSortField();
531 query_result_cache_file_name += ((sort_string != null) ? sort_string : "") + "-";
532 String default_conjunction_operator = queryer.getDefaultConjunctionOperator();
533 query_result_cache_file_name += default_conjunction_operator + "-";
534 int start_results = queryer.getStartResults();
535 int end_results = queryer.getEndResults();
536 query_result_cache_file_name += start_results + "-" + end_results;
537 query_result_cache_file_name = fileSafe(query_result_cache_file_name);
538
539 // If the query result cache file exists, just return its contents and we're done
540 File index_cache_directory = new File(index_directory, "cache");
541 query_result_cache_file = new File(index_cache_directory, query_result_cache_file_name);
542 if (query_result_cache_file.exists() && query_result_cache_file.isFile()) {
543 FileInputStream fis = new FileInputStream(query_result_cache_file);
544 InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
545 BufferedReader buffered_reader = new BufferedReader(isr);
546 String line = "";
547 while ((line = buffered_reader.readLine()) != null) {
548 query_results_xml.append(line + "\n");
549 }
550 String query_results_xml_string = query_results_xml.toString();
551 query_results_xml_string = query_results_xml_string.replaceFirst("cached=\"false\"", "cached=\"true\"");
552
553 utf8out.print(query_results_xml_string);
554 utf8out.flush();
555
556 return;
557 }
558 }
559
560 // not cached
561 query_results_xml.append("<ResultSet cached=\"false\">\n");
562 query_results_xml.append("<QueryString>" + LuceneQueryResult.xmlSafe(query_string) + "</QueryString>\n");
563 Filter filter = queryer.getFilter();
564 if (filter != null) {
565 query_results_xml.append("<FilterString>" + filter.toString() + "</FilterString>\n");
566 }
567
568 LuceneQueryResult query_result = queryer.runQuery(query_string);
569 if (query_result == null) {
570 System.err.println("Couldn't run the query");
571 return;
572 }
573
574 if (query_result.getError() != LuceneQueryResult.NO_ERROR) {
575 query_results_xml.append("<Error type=\""+query_result.getErrorString()+"\" />\n");
576 } else {
577 query_results_xml.append(query_result.getXMLString());
578 }
579 query_results_xml.append("</ResultSet>\n");
580
581 utf8out.print(query_results_xml);
582 utf8out.flush();
583
584 // Cache this query result, if desired
585 if (query_result_caching_enabled) {
586 // Catch any exceptions thrown trying to write the query result cache file and warn about them, but don't
587 // bother with the full stack trace. It won't affect the functionality if we can't write some cache
588 // files, it will just affect the speed of subsequent requests.
589 // Example exceptions are "permission denied" errors, or "filename too long" errors (the filter string
590 // can get very long in some collections)
591 try
592 {
593 FileWriter query_result_cache_file_writer = new FileWriter(query_result_cache_file);
594 query_result_cache_file_writer.write(query_results_xml.toString());
595 query_result_cache_file_writer.close();
596 }
597 catch (Exception exception)
598 {
599 System.err.println("Warning: Exception occurred trying to write query result cache file (" + exception + ")");
600 }
601 }
602 }
603
604 protected static String fileSafe(String text)
605 {
606 StringBuffer file_safe_text = new StringBuffer();
607 for (int i = 0; i < text.length(); i++) {
608 char character = text.charAt(i);
609 if ((character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') || character == '-') {
610 file_safe_text.append(character);
611 }
612 else {
613 file_safe_text.append('%');
614 file_safe_text.append((int) character);
615 }
616 }
617 return file_safe_text.toString();
618 }
619
620
621}
622
623
Note: See TracBrowser for help on using the repository browser.