source: gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3/util/SolrQueryWrapper.java@ 30050

Last change on this file since 30050 was 30050, checked in by Georgiy Litvinov, 9 years ago

Solr repo modifications for Solr side highlighing and snippets
  • Property svn:executable set to *
File size: 20.3 KB
Line 
1/**********************************************************************
2 *
3 * SolrQueryWrapper.java
4 *
5 * Copyright 2004 The New Zealand Digital Library Project
6 *
7 * A component of the Greenstone digital library software
8 * from the New Zealand Digital Library Project at the
9 * University of Waikato, New Zealand.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24 *
25 *********************************************************************/
26package org.greenstone.gsdl3.util;
27
28import java.lang.reflect.Type;
29import java.net.URLDecoder;
30import java.util.ArrayList;
31import java.util.Collection;
32import java.util.HashMap;
33import java.util.Iterator;
34import java.util.List;
35import java.util.Map;
36import java.util.Set;
37import java.util.HashSet;
38import java.util.regex.Pattern;
39import java.util.regex.Matcher;
40
41import org.apache.log4j.Logger;
42import org.apache.solr.client.solrj.SolrQuery; // subclass of ModifiableSolrParams
43import org.apache.solr.client.solrj.SolrServer;
44import org.apache.solr.client.solrj.SolrServerException;
45import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
46import org.apache.solr.client.solrj.response.QueryResponse;
47import org.apache.solr.client.solrj.response.TermsResponse;
48import org.apache.solr.core.CoreContainer;
49import org.apache.solr.core.SolrCore;
50import org.apache.solr.common.SolrDocument;
51import org.apache.solr.common.SolrDocumentList;
52import org.apache.solr.common.params.ModifiableSolrParams;
53import org.greenstone.LuceneWrapper4.SharedSoleneQuery;
54import org.greenstone.LuceneWrapper4.SharedSoleneQueryResult;
55import org.apache.lucene.search.Query; // Query, TermQuery, BooleanQuery, BooleanClause and more
56import org.apache.lucene.index.IndexReader;
57import org.apache.lucene.index.Term;
58import org.apache.solr.search.QParser;
59import org.apache.solr.search.SolrIndexSearcher;
60import org.apache.solr.request.LocalSolrQueryRequest;
61
62import com.google.gson.Gson;
63import com.google.gson.reflect.TypeToken;
64
65public class SolrQueryWrapper extends SharedSoleneQuery
66{
67 public static String SORT_ASCENDING = "asc";
68 public static String SORT_DESCENDING = "desc";
69 public static String SORT_BY_RANK = "score";
70 public static String SORT_BY_INDEX_ORDER = "_docid_";
71
72 static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.SolrQueryWrapper.class.getName());
73 protected int max_docs = 100;
74 protected String sort_order = SORT_DESCENDING;
75 protected String sort_field = SORT_BY_RANK; // don't want null default for solr
76 protected ArrayList<String> _facets = new ArrayList<String>();
77 protected ArrayList<String> _facetQueries = new ArrayList<String>();
78 SolrServer solr_core = null;
79
80 protected String highlight_field = null;
81
82 String collection_core_name_prefix = null;
83
84 public SolrQueryWrapper()
85 {
86 super();
87 start_results = 0;
88 }
89
90 public void setMaxDocs(int max_docs)
91 {
92 this.max_docs = max_docs;
93 }
94
95 public void setSolrCore(SolrServer solr_core)
96 {
97 this.solr_core = solr_core;
98 }
99
100 public void setCollectionCoreNamePrefix(String colCoreNamePrefix) {
101 this.collection_core_name_prefix = colCoreNamePrefix;
102 }
103
104 // make sure its not null.
105 public void setSortField(String sort_field) {
106 if (sort_field != null) {
107 this.sort_field = sort_field;
108 }
109 }
110 public void setHighlightField(String hl_field)
111 {
112 this.highlight_field = hl_field;
113 }
114 public void setSortOrder(String order)
115 {
116 this.sort_order = order;
117 }
118 public void addFacet(String facet)
119 {
120 if (!_facets.contains(facet))
121 {
122 _facets.add(facet);
123 }
124 }
125
126 public void clearFacets()
127 {
128 _facets.clear();
129 }
130
131 public void addFacetQuery(String facetQuery)
132 {
133 if (!_facetQueries.contains(facetQuery))
134 {
135 _facetQueries.add(facetQuery);
136 }
137 }
138
139 public void clearFacetQueries()
140 {
141 _facetQueries.clear();
142 }
143
144 public boolean initialise()
145 {
146 if (solr_core == null)
147 {
148 utf8out.println("Solr Core not loaded in ");
149 utf8out.flush();
150 return false;
151 }
152 return true;
153 }
154
155
156 /**
157 * UNUSED.
158 * Back when we used the EmbeddedSolrServer, this getTerms method would expand the terms of a query.
159 * Because of Solr/Lucene Index locking exceptions, we switched over to the HttpSolrServer instead
160 * of the Embedded kind.
161 *
162 * The functionality of getTerms has been moved to
163 * ../solrserver/Greenstone3SearchHandler.java, which will sit on the solrserver side (inside
164 * tomcat's solr webapp).
165 *
166 * Extracts the query terms from the query string. The query string can be a boolean
167 * combination of the various search fields with their search terms or phrases
168 */
169 public Term[] getTerms(SolrQuery solrQuery, String query_string)
170 {
171 Term terms[] = null;
172
173 if(solr_core instanceof EmbeddedSolrServer) {
174 EmbeddedSolrServer solrServer = (EmbeddedSolrServer)solr_core;
175
176 CoreContainer coreContainer = solrServer.getCoreContainer();
177
178 Collection<SolrCore> solrCores = coreContainer.getCores();
179 if(!solrCores.isEmpty()) {
180 Iterator<SolrCore> coreIterator = solrCores.iterator();
181
182 // Just use the first core that matches the collection name, since the term
183 // frequency of any term is the same regardless of whether its didx or sidx core
184 boolean foundCore = false;
185 while(coreIterator.hasNext() && !foundCore) {
186 SolrCore solrCore = coreIterator.next();
187 if(this.collection_core_name_prefix != null) {
188 if(!solrCore.getName().startsWith(this.collection_core_name_prefix)) {
189 //logger.error("### Skipping core not of this collection: " + solrCore.getName());
190 continue;
191 }
192 } else {
193 logger.error("### Collection_core_name_prefix not set. Won't try to find terms");
194 break;
195 }
196
197 //logger.error("### Found core " + solrCore.getName() + " of this collection " + this.collection_core_name_prefix);
198 foundCore = true;
199
200 LocalSolrQueryRequest solrQueryRequest = new LocalSolrQueryRequest(solrCore, solrQuery);
201 Query parsedQuery = null;
202
203 try {
204
205 // get the qparser, default is LuceneQParserPlugin, which is called "lucene" see http://wiki.apache.org/solr/QueryParser
206 QParser qParser = QParser.getParser(query_string, "lucene", solrQueryRequest);
207 parsedQuery = qParser.getQuery();
208
209 // For PrefixQuery or WildCardQuery (a subclass of AutomatonQuery, incl RegexpQ),
210 // like ZZ:econom* and ZZ:*date/regex queries, Query.extractTerms() throws an Exception
211 // because it has not done the Query.rewrite() step yet. So do that manually for them.
212 // This still doesn't provide us with the terms that econom* or *date break down into.
213
214 //if(parsedQuery instanceof PrefixQuery || parsedQuery instanceof AutomatonQuery) {
215 // Should we just check superclass MultiTermQuery?
216 // Can be a BooleanQuery containing PrefixQuery/WildCardQuery among its clauses, so
217 // just test for * in the query_string to determine if we need to do a rewrite() or not
218 if(query_string.contains("*")) {
219 SolrIndexSearcher searcher = solrQueryRequest.getSearcher();
220 IndexReader indexReader = searcher.getIndexReader(); // returns a DirectoryReader
221 parsedQuery = parsedQuery.rewrite(indexReader); // gets rewritten to ConstantScoreQuery
222 }
223
224 //System.err.println("#### Query type was: " + parsedQuery.getClass());
225 //logger.error("#### Query type was: " + parsedQuery.getClass());
226
227 // extract the terms
228 Set<Term> extractedQueryTerms = new HashSet<Term>();
229 parsedQuery.extractTerms(extractedQueryTerms);
230
231 terms = new Term[extractedQueryTerms.size()];
232
233 Iterator<Term> termsIterator = extractedQueryTerms.iterator();
234 for(int i = 0; termsIterator.hasNext(); i++) {
235 Term term = termsIterator.next();
236 ///System.err.println("#### Found query term: " + term);
237 ///logger.error("#### Found query term: " + term);
238
239 terms[i] = term; //(term.field(), term.text());
240 }
241
242 } catch(Exception queryParseException) {
243 queryParseException.printStackTrace();
244 System.err.println("Exception when parsing query: " + queryParseException.getMessage());
245 System.err.println("#### Query type was: " + parsedQuery.getClass());
246 logger.error("#### Query type was: " + parsedQuery.getClass());
247 }
248 // http://lucene.apache.org/solr/4_7_2/solr-core/org/apache/solr/request/SolrQueryRequestBase.html#close%28%29
249 // close() must be called when the object is no longer in use. Frees resources associated with this request
250 solrQueryRequest.close();
251 }
252
253 } else {
254 System.err.println("#### CoreContainer is empty");
255 logger.error("#### CoreContainer is empty");
256 }
257 } else {
258 System.err.println("#### Not an EmbeddedSolrServer. SolrQueryWrapper.getTerms() not yet implemented for " + solr_core.getClass());
259 logger.error("#### Not an EmbeddedSolrServer. SolrQueryWrapper.getTerms() not yet implemented for " + solr_core.getClass());
260 }
261
262
263 return terms;
264 }
265
266 public SharedSoleneQueryResult runQuery(String query_string)
267 {
268 if (query_string == null || query_string.equals(""))
269 {
270 utf8out.println("The query word is not indicated ");
271 utf8out.flush();
272 return null;
273 }
274
275 SolrQueryResult solr_query_result = new SolrQueryResult();
276 solr_query_result.clear();
277
278 if (_facetQueries.size() > 0)
279 {
280 HashMap<String, ArrayList<String>> grouping = new HashMap<String, ArrayList<String>>();
281 for (String currentQuery : _facetQueries)
282 {
283 //Facet queries are stored in JSON, so we have to decode it
284 Gson gson = new Gson();
285 Type type = new TypeToken<List<String>>()
286 {
287 }.getType();
288 List<String> queryElems = gson.fromJson(currentQuery, type);
289
290 //Group each query segment by the index it uses
291 for (String currentQueryElement : queryElems)
292 {
293 String decodedQueryElement = null;
294 try
295 {
296 decodedQueryElement = URLDecoder.decode(currentQueryElement, "UTF-8");
297 }
298 catch (Exception ex)
299 {
300 continue;
301 }
302
303 int colonIndex = currentQueryElement.indexOf(":");
304 String indexShortName = currentQueryElement.substring(0, colonIndex);
305
306 if (grouping.get(indexShortName) == null)
307 {
308 grouping.put(indexShortName, new ArrayList<String>());
309 }
310 grouping.get(indexShortName).add(decodedQueryElement);
311 }
312 }
313
314 //Construct the facet query string to add to the regular query string
315 StringBuilder facetQueryString = new StringBuilder();
316 int keysetCounter = 0;
317 for (String key : grouping.keySet())
318 {
319 StringBuilder currentFacetString = new StringBuilder("(");
320 int groupCounter = 0;
321 for (String queryElem : grouping.get(key))
322 {
323 currentFacetString.append(queryElem);
324
325 groupCounter++;
326 if (groupCounter < grouping.get(key).size())
327 {
328 currentFacetString.append(" OR ");
329 }
330 }
331 currentFacetString.append(")");
332
333 facetQueryString.append(currentFacetString);
334
335 keysetCounter++;
336 if (keysetCounter < grouping.keySet().size())
337 {
338 facetQueryString.append(" AND ");
339 }
340 }
341
342 if (facetQueryString.length() > 0)
343 {
344 query_string += " AND " + facetQueryString;
345 }
346 }
347
348
349 SolrQuery solrQuery = new SolrQuery(query_string);
350 solrQuery.addSort(this.sort_field, SolrQuery.ORDER.valueOf(this.sort_order)); // sort param, like "score desc" or "byORG asc"
351 solrQuery.setStart(start_results); // which result to start from
352 solrQuery.setRows((end_results - start_results) + 1); // how many results per "page"
353
354 // http://lucene.472066.n3.nabble.com/get-term-frequency-just-only-keywords-search-td4084510.html
355 // WORKS (search didx core):
356 //TI:farming
357 //docOID,score,termfreq(TI,'farming'),totaltermfreq(TI,'farming')
358
359
360 // which fields to return for each document, we'll add the request for totaltermfreq later
361 // fl=docOID score termfreq(TI,'farming') totaltermfreq(TI,'farming')
362 solrQuery.setFields("docOID", "score"); //solrParams.set("fl", "docOID score totaltermfreq(field,'queryterm')");
363
364 //Turn on highlighting
365 solrQuery.setHighlight(true);
366 //Return 3 snippets for each document
367 solrQuery.setParam("hl.snippets", "3");
368 solrQuery.setParam("hl.fl", highlight_field);
369 solrQuery.setHighlightSimplePre("&lt;span class=\"snippetText\"&gt;");
370
371 //Set text which appears after highlighted term
372 solrQuery.setHighlightSimplePost("&lt;/span&gt;");
373
374 //solrQuery.setTerms(true); // turn on the termsComponent
375 //solrQuery.set("terms.fl", "ZZ"); // which field to get the terms from. ModifiableSolrParams method
376
377 // http://wiki.apache.org/solr/TermVectorComponent and https://cwiki.apache.org/confluence/display/solr/The+Term+Vector+Component
378 // http://lucene.472066.n3.nabble.com/get-term-frequency-just-only-keywords-search-td4084510.html
379 // http://stackoverflow.com/questions/13031534/word-frequency-in-solr
380 // http://wiki.apache.org/solr/FunctionQuery#tf and #termfreq and #totaltermfreq
381 // https://wiki.apache.org/solr/TermsComponent
382
383 //solrParams.set("tv.tf", true);// turn on the terms vector Component
384 //solrParams.set("tv.fl", "ZZ");// which field to get the terms from /// ZZ
385
386
387 if (_facets.size() > 0)
388 {
389 // enable facet counts in the query response
390 solrQuery.setFacet(true); //solrParams.set("facet", "true");
391 for (int i = 0; i < _facets.size(); i++)
392 {
393 // add this field as a facet
394 solrQuery.addFacetField(_facets.get(i)); // solrParams.add("facet.field", _facets.get(i));
395 }
396 }
397
398 // the solrserver will now
399 // get the individual terms that make up the query, then request solr to return the totaltermfreq for each term
400
401 // do the query
402 try
403 {
404 QueryResponse solrResponse = solr_core.query(solrQuery); //solr_core.query(solrParams);
405 SolrDocumentList hits = solrResponse.getResults();
406 Map<String, Map<String, List<String>>> hlResponse = solrResponse.getHighlighting();
407 solr_query_result.setHighlightResults(hlResponse);
408 //TermsResponse termResponse = solrResponse.getTermsResponse(); // null unless termvectors=true in schema.xml
409
410 if (hits != null)
411 {
412 logger.info("*** hits size = " + hits.size());
413 logger.info("*** num docs found = " + hits.getNumFound());
414
415 logger.info("*** start results = " + start_results);
416 logger.info("*** end results = " + end_results);
417 logger.info("*** max docs = " + max_docs);
418
419 // numDocsFound is the total number of matching docs in the collection
420 // as opposed to the number of documents returned in the hits list
421
422 solr_query_result.setTotalDocs((int) hits.getNumFound());
423
424 solr_query_result.setStartResults(start_results);
425 solr_query_result.setEndResults(start_results + hits.size());
426
427 // get the first field we're searching in, this will be the fallback field
428 int sepIndex = query_string.indexOf(":");
429 String defaultField = query_string.substring(0, sepIndex);
430 //String query = query_string.substring(sepIndex + 2, query_string.length() - 1); // Replaced by call to getTerms()
431
432 //solr_query_result.addTerm(query, field, (int) hits.getNumFound(), -1);
433
434 // Output the matching documents
435 for (int i = 0; i < hits.size(); i++)
436 {
437 SolrDocument doc = hits.get(i);
438
439 // Need to think about how to support document term frequency. Make zero for now
440 int doc_term_freq = 0;
441 String docOID = (String) doc.get("docOID");
442 Float score = (Float) doc.get("score");
443
444 logger.info("**** docOID = " + docOID);
445 logger.info("**** score = " + score);
446
447
448 // solr returns each term's totaltermfreq, ttf, at the document level, even though
449 // the ttf is the same for each document. So extract this information just for the first document
450 // https://wiki.apache.org/solr/FunctionQuery#docfreq
451
452 if(i == 0) { // first document, all others repeat the same termfreq data
453 boolean foundTermInfo = false;
454
455 Collection<String> fieldNames = doc.getFieldNames();
456 for(Iterator<String> it = fieldNames.iterator(); it.hasNext(); ) {
457 String fieldName = it.next(); // e.g. looking for totaltermfreq(ZZ,'economically')
458 //logger.info("@@@@ found fieldName " + fieldName);
459
460
461 if(fieldName.startsWith("totaltermfreq")) {
462 //|| fieldName.startsWith("termfreq")) {
463
464 foundTermInfo = true;
465
466 // e.g. totaltermfreq(TI,'farming')
467 // e.g. termfreq(TI,'farming')
468 Pattern pattern = Pattern.compile("(.*?termfreq)\\((.*?),'(.*?)'\\)");
469 Matcher matcher = pattern.matcher(fieldName);
470 String metaField, indexField, queryTerm;
471 while (matcher.find()) {
472 metaField = matcher.group(1); // termfreq or totaltermfreq
473 indexField = matcher.group(2); //ZZ, TI
474 queryTerm = matcher.group(3);
475
476 //logger.info("\t@@@@ found field " + indexField);
477 //logger.info("\t@@@@ queryTerm " + queryTerm);
478
479 // Finally, can ask for the totaltermfreq value for this
480 // searchterm in its indexed field:
481 // e.g. totaltermfreq(TI,'farming'), e.g. termfreq(TI,'farming')
482 Long totaltermfreq = (Long)doc.get("totaltermfreq("+indexField+",'"+queryTerm+"')");
483
484 Integer termfreq = (Integer)doc.get("termfreq("+indexField+",'"+queryTerm+"')");
485
486 //System.err.println("**** ttf = " + totaltermfreq);
487 //System.err.println("**** tf = " + termfreq);
488 //logger.info("**** ttf = " + totaltermfreq);
489 //logger.info("**** tf = " + termfreq);
490 solr_query_result.addTerm(queryTerm, indexField, (int) hits.getNumFound(), totaltermfreq.intValue()); // long totaltermfreq to int
491 }
492 }
493 }
494 if(!foundTermInfo) { // no terms extracted from query_string
495 solr_query_result.addTerm(query_string, defaultField, (int) hits.getNumFound(), -1); // no terms
496 }
497 }
498
499 solr_query_result.addDoc(docOID, score.floatValue(), doc_term_freq); // doc_termfreq for which term????
500 }
501 }
502 else
503 {
504 solr_query_result.setTotalDocs(0);
505
506 solr_query_result.setStartResults(0);
507 solr_query_result.setEndResults(0);
508 }
509
510 solr_query_result.setFacetResults(solrResponse.getFacetFields());
511 }
512 catch (SolrServerException server_exception)
513 {
514 server_exception.printStackTrace();
515 solr_query_result.setError(SolrQueryResult.SERVER_ERROR);
516 }
517
518 return solr_query_result;
519 }
520// Highlighting query. Returns full highlighted text for document
521 public String runHighlightingQuery(String query,String hldocOID)
522 {
523
524 SolrQueryResult solr_query_result = new SolrQueryResult();
525 solr_query_result.clear();
526
527
528 /* Create Query*/
529
530 SolrQuery solrQuery = new SolrQuery(query);
531
532 /* Set Query Parameters*/
533
534 //Turn on highlighting
535 solrQuery.setHighlight(true);
536 //Extract default field from query
537
538 //Set field for highlighting
539 solrQuery.setParam("hl.fl", highlight_field);
540
541 //Get whole highlighted field
542 solrQuery.setHighlightFragsize(0);
543
544 //Return only required document by docOID
545 solrQuery.setFilterQueries("docOID:"+ hldocOID);
546
547 //Set text which appears before highlighted term
548 //solrQuery.setHighlightSimplePre("<annotation type=\"query_term\">");
549 solrQuery.setHighlightSimplePre("<span class=\"termHighlight\">");
550 //Set text which appears after highlighted term
551 //solrQuery.setHighlightSimplePost("</annotation>");
552 solrQuery.setHighlightSimplePost("</span>");
553 //Prepare results
554 String text = null;
555 // do the query
556 try
557 {
558 QueryResponse solrResponse = solr_core.query(solrQuery); //solr_core.query(solrParams);
559 //Get highliting results
560 Map<String,Map<String,List<String>>> highlightingResults = solrResponse.getHighlighting();
561 //Get highlited document text
562 text = highlightingResults.get(hldocOID).get(highlight_field).get(0);
563
564
565 }
566 catch (SolrServerException server_exception)
567 {
568 server_exception.printStackTrace();
569
570 }
571 return text;
572 }
573
574 //Greenstone universe operates with a base of 1 for "start_results"
575 //But Solr operates from 0
576 public void setStartResults(int start_results)
577 {
578 if (start_results < 0)
579 {
580 start_results = 0;
581 }
582 this.start_results = start_results - 1;
583 }
584
585 public void cleanUp()
586 {
587 super.cleanUp();
588 }
589
590}
Note: See TracBrowser for help on using the repository browser.