Changeset 29986 for gs3-extensions/solr/trunk/src/src

Timestamp:

2015-06-19T23:21:26+12:00 (9 years ago)

Author:

ak19

Message:

The getTerms() functionality previously used by the EmbeddedSolrServer has now been re-implemented for HttpSolrServer with the new custom Greenstone Solr RequestHandler class Greenstone3SearchHandler, which lives on the solr server side, in tomcat's solr webapp. The functionality has been improvemed, such as being able to search for: econom* cat, by recursively calling setRewriteMethods on any PrefixQuery and WildcardQuery MultiQueries within an overall BooleanQuery, and by handling BooleanQuery.TooManyClauses exceptions when the number of expanded terms is too large, such as for a search of a*.

Location:

gs3-extensions/solr/trunk/src/src/java/org/greenstone

Files:

• gsdl3/util/SolrQueryWrapper.java (modified) (4 diffs)
• solrserver (added)
• solrserver/Greenstone3SearchHandler.java (added)

gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3/util/SolrQueryWrapper.java

@@ -36 +36 @@
 import java.util.HashSet;
 
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
 import org.apache.log4j.Logger;
 import org.apache.solr.client.solrj.SolrQuery; // subclass of ModifiableSolrParams
@@ -149 +152 @@
 
 
-    /** Extracts the query terms from the query string. The query string can be a boolean 
+    /**
+     * UNUSED.
+     * Back when we used the EmbeddedSolrServer, this getTerms method would expand the terms of a query.
+     * Because of Solr/Lucene Index locking exceptions, we switched over to the HttpSolrServer instead
+     * of the Embedded kind. 
+     *
+     * The functionality of getTerms has been moved to 
+     * ../solrserver/Greenstone3SearchHandler.java, which will sit on the solrserver side (inside 
+     * tomcat's solr webapp).
+     *
+     * Extracts the query terms from the query string. The query string can be a boolean 
      * combination of the various search fields with their search terms or phrases
      */
@@ -371 +384 @@
         }
 
+        // the solrserver will now
         // get the individual terms that make up the query, then request solr to return the totaltermfreq for each term
-        Term[] terms = getTerms(solrQuery, query_string);
-        if(terms != null) {
-            for(int i = 0; i < terms.length; i++) {
-            Term term = terms[i];
-            String field = term.field();
-            String queryTerm = term.text();
-            // totaltermfreq(TI, 'farming') termfreq(TI, 'farming')
-            
-            solrQuery.addField("totaltermfreq(" + field + ",'" + queryTerm + "')");
-            solrQuery.addField("termfreq(" + field + ",'" + queryTerm + "')");
-            }
-        }
 
         // do the query
@@ -433 +435 @@
                     // solr returns each term's totaltermfreq, ttf, at the document level, even though 
                     // the ttf is the same for each document. So extract this information just for the first document
-                    if(i == 0) { // first document
-                        
-                        if(terms != null) {
-                        for(int j = 0; j < terms.length; j++) {
-                            Term term = terms[j];
-                            String field = term.field();
-                            String queryTerm = term.text();
-
-                            // totaltermfreq(TI, 'farming') termfreq(TI, 'farming')
-                            Long totaltermfreq = (Long)doc.get("totaltermfreq("+field+",'"+queryTerm+"')");
-                            Integer termfreq = (Integer)doc.get("termfreq("+field+",'"+queryTerm+"')");
-
-                            //System.err.println("**** ttf = " + totaltermfreq); 
-                            //System.err.println("**** tf = " + termfreq);
-                            //logger.info("**** ttf = " + totaltermfreq); 
-                            //logger.info("**** tf = " + termfreq);
+                    if(i == 0) { // first document, all others repeat the same termfreq data
+                        boolean foundTermInfo = false;
+
+                        Collection<String> fieldNames = doc.getFieldNames();
+                        for(Iterator<String> it = fieldNames.iterator(); it.hasNext(); ) {
+                        String fieldName = it.next(); // e.g. looking for totaltermfreq(ZZ,'economically')
+                        //logger.info("@@@@ found fieldName " + fieldName);
                         
-                            solr_query_result.addTerm(queryTerm, field, (int) hits.getNumFound(), totaltermfreq.intValue()); // long totaltermfreq to int
-                        }
-                        } else { // no terms extracted from query_string
+
+                        if(fieldName.startsWith("totaltermfreq")) {
+                           //|| fieldName.startsWith("termfreq")) {
+                            
+                            foundTermInfo = true;
+
+                            // e.g. totaltermfreq(TI,'farming') 
+                            // e.g. termfreq(TI,'farming')
+                            Pattern pattern = Pattern.compile("(.*?termfreq)\\((.*?),'(.*?)'\\)");
+                            Matcher matcher = pattern.matcher(fieldName);
+                            String metaField, indexField, queryTerm;
+                            while (matcher.find()) {
+                            metaField = matcher.group(1); // termfreq or totaltermfreq
+                            indexField = matcher.group(2); //ZZ, TI
+                            queryTerm = matcher.group(3);
+
+                            //logger.info("\t@@@@ found field " + indexField);
+                            //logger.info("\t@@@@ queryTerm " + queryTerm);
+
+                            // Finally, can ask for the totaltermfreq value for this
+                            // searchterm in its indexed field:
+                            // e.g. totaltermfreq(TI,'farming'), e.g. termfreq(TI,'farming')
+                            Long totaltermfreq = (Long)doc.get("totaltermfreq("+indexField+",'"+queryTerm+"')");
+                            
+                            Integer termfreq = (Integer)doc.get("termfreq("+indexField+",'"+queryTerm+"')");
+                            
+                            //System.err.println("**** ttf = " + totaltermfreq); 
+                            //System.err.println("**** tf = " + termfreq);
+                            //logger.info("**** ttf = " + totaltermfreq); 
+                            //logger.info("**** tf = " + termfreq);
+                            solr_query_result.addTerm(queryTerm, indexField, (int) hits.getNumFound(), totaltermfreq.intValue()); // long totaltermfreq to int
+                            }
+                        }                       
+                        }
+                        if(!foundTermInfo) { // no terms extracted from query_string
                         solr_query_result.addTerm(query_string, defaultField, (int) hits.getNumFound(), -1); // no terms
                         }