Changeset 29986 for gs3-extensions

Timestamp:

2015-06-19T23:21:26+12:00 (9 years ago)

Author:

ak19

Message:

The getTerms() functionality previously used by the EmbeddedSolrServer has now been re-implemented for HttpSolrServer with the new custom Greenstone Solr RequestHandler class Greenstone3SearchHandler, which lives on the solr server side, in tomcat's solr webapp. The functionality has been improvemed, such as being able to search for: econom* cat, by recursively calling setRewriteMethods on any PrefixQuery and WildcardQuery MultiQueries within an overall BooleanQuery, and by handling BooleanQuery.TooManyClauses exceptions when the number of expanded terms is too large, such as for a search of a*.

Location:

gs3-extensions/solr/trunk/src

Files:

• build.xml (modified) (4 diffs)
• conf/solrconfig.xml (modified) (2 diffs)
• src/java/org/greenstone/gsdl3/util/SolrQueryWrapper.java (modified) (4 diffs)
• src/java/org/greenstone/solrserver (added)
• src/java/org/greenstone/solrserver/Greenstone3SearchHandler.java (added)

gs3-extensions/solr/trunk/src/build.xml

@@ -130 +130 @@
     </jar>
     <copy file="${build.home}/gs3-solr.jar" todir="${web.libdir}"/>
+
+    <!-- customisations to the http solr server -->
+    <jar destfile="${build.home}/gs3-solrserver.jar">
+      <fileset dir="${build.home}">
+        <include name="org/greenstone/solrserver/**"/>
+      </fileset>
+      <manifest>
+        <attribute name="Built-By" value="greenstone3" />
+      </manifest>
+    </jar>
+    <!--<copy file="${build.home}/gs3-solrserver.jar" todir="${tomcat.dir}/webapps/solr/WEB-INF/lib"/>
+
+          But there's no guarantee the destination will exist at the beginning.
+          The jar file also needs to go into solr.war. Which is taken care of in the add-service task.
+          Instead, for testing, call ant compile-gs3-solrserver, which will copy it across to solr webapp. 
+      -->
   </target>
   
@@ -188 +204 @@
     <copy todir="${tomcat.dir}/webapps" file="webapps/solr.war" />
     <unwar src="${tomcat.dir}/webapps/solr.war" dest="${tomcat.dir}/webapps/solr"/>
-    <echo>Copying xalan related jar files from  ${web.libdir} into ${tomcat.dir}/webapps/solr.war</echo>
+    <echo>Copying xalan related jar files, morphology and gs3-solrserver jars from ${web.libdir} into ${tomcat.dir}/webapps/solr.war</echo>
     <copy todir="${tomcat.dir}/webapps/solr/WEB-INF/lib">
       <filelist dir="${web.libdir}" files="${shared-xalan-jars}" />
+      <file file="${build.home}/gs3-solrserver.jar" />
       <filelist dir="lib/russianmorphology" files="${russian-morph-jars}" />
     </copy>
@@ -224 +241 @@
 
 
-  <target name="add-service" depends="copy-solr-web,copy-files,solr-for-tomcat,compile" description="Run this target to setup the Solr extension for Greenstone3" />
+  <target name="add-service" depends="copy-solr-web,copy-files,compile,solr-for-tomcat" description="Run this target to setup the Solr extension for Greenstone3" />
   
   <target name="del-service" depends="del-files,del-solr-for-tomcat"
@@ -284 +301 @@
   </target>
   
+  <target name="compile-gs3-solrserver" description="TEST Target to compile the solr server side gs3-solserver.jar for testing">
+    <delete dir="build/org/greenstone/solrserver" />
+    <delete file="build/gs3-solrserver.jar" />
+    <antcall target="compile" />
+    <copy file="${build.home}/gs3-solrserver.jar" todir="${tomcat.dir}/webapps/solr/WEB-INF/lib" />
+  </target>
+
 </project>

gs3-extensions/solr/trunk/src/conf/solrconfig.xml

@@ -824 +824 @@
        queries across multiple shards
     -->
-  <requestHandler name="/select" class="solr.SearchHandler">
+  <!--<requestHandler name="/select" class="solr.SearchHandler">-->
+    <requestHandler name="/select" class="org.greenstone.solrserver.Greenstone3SearchHandler">
     <!-- default values for query parameters can be specified, these
          will be overridden by parameters in the request
@@ -888 +889 @@
 
   <!-- A request handler that returns indented JSON by default -->
-  <requestHandler name="/query" class="solr.SearchHandler">
+  <!--<requestHandler name="/query" class="solr.SearchHandler">-->
+  <requestHandler name="/query" class="org.greenstone.solrserver.Greenstone3SearchHandler">
      <lst name="defaults">
        <str name="echoParams">explicit</str>

gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3/util/SolrQueryWrapper.java

@@ -36 +36 @@
 import java.util.HashSet;
 
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
 import org.apache.log4j.Logger;
 import org.apache.solr.client.solrj.SolrQuery; // subclass of ModifiableSolrParams
@@ -149 +152 @@
 
 
-    /** Extracts the query terms from the query string. The query string can be a boolean 
+    /**
+     * UNUSED.
+     * Back when we used the EmbeddedSolrServer, this getTerms method would expand the terms of a query.
+     * Because of Solr/Lucene Index locking exceptions, we switched over to the HttpSolrServer instead
+     * of the Embedded kind. 
+     *
+     * The functionality of getTerms has been moved to 
+     * ../solrserver/Greenstone3SearchHandler.java, which will sit on the solrserver side (inside 
+     * tomcat's solr webapp).
+     *
+     * Extracts the query terms from the query string. The query string can be a boolean 
      * combination of the various search fields with their search terms or phrases
      */
@@ -371 +384 @@
         }
 
+        // the solrserver will now
         // get the individual terms that make up the query, then request solr to return the totaltermfreq for each term
-        Term[] terms = getTerms(solrQuery, query_string);
-        if(terms != null) {
-            for(int i = 0; i < terms.length; i++) {
-            Term term = terms[i];
-            String field = term.field();
-            String queryTerm = term.text();
-            // totaltermfreq(TI, 'farming') termfreq(TI, 'farming')
-            
-            solrQuery.addField("totaltermfreq(" + field + ",'" + queryTerm + "')");
-            solrQuery.addField("termfreq(" + field + ",'" + queryTerm + "')");
-            }
-        }
 
         // do the query
@@ -433 +435 @@
                     // solr returns each term's totaltermfreq, ttf, at the document level, even though 
                     // the ttf is the same for each document. So extract this information just for the first document
-                    if(i == 0) { // first document
-                        
-                        if(terms != null) {
-                        for(int j = 0; j < terms.length; j++) {
-                            Term term = terms[j];
-                            String field = term.field();
-                            String queryTerm = term.text();
-
-                            // totaltermfreq(TI, 'farming') termfreq(TI, 'farming')
-                            Long totaltermfreq = (Long)doc.get("totaltermfreq("+field+",'"+queryTerm+"')");
-                            Integer termfreq = (Integer)doc.get("termfreq("+field+",'"+queryTerm+"')");
-
-                            //System.err.println("**** ttf = " + totaltermfreq); 
-                            //System.err.println("**** tf = " + termfreq);
-                            //logger.info("**** ttf = " + totaltermfreq); 
-                            //logger.info("**** tf = " + termfreq);
+                    if(i == 0) { // first document, all others repeat the same termfreq data
+                        boolean foundTermInfo = false;
+
+                        Collection<String> fieldNames = doc.getFieldNames();
+                        for(Iterator<String> it = fieldNames.iterator(); it.hasNext(); ) {
+                        String fieldName = it.next(); // e.g. looking for totaltermfreq(ZZ,'economically')
+                        //logger.info("@@@@ found fieldName " + fieldName);
                         
-                            solr_query_result.addTerm(queryTerm, field, (int) hits.getNumFound(), totaltermfreq.intValue()); // long totaltermfreq to int
-                        }
-                        } else { // no terms extracted from query_string
+
+                        if(fieldName.startsWith("totaltermfreq")) {
+                           //|| fieldName.startsWith("termfreq")) {
+                            
+                            foundTermInfo = true;
+
+                            // e.g. totaltermfreq(TI,'farming') 
+                            // e.g. termfreq(TI,'farming')
+                            Pattern pattern = Pattern.compile("(.*?termfreq)\\((.*?),'(.*?)'\\)");
+                            Matcher matcher = pattern.matcher(fieldName);
+                            String metaField, indexField, queryTerm;
+                            while (matcher.find()) {
+                            metaField = matcher.group(1); // termfreq or totaltermfreq
+                            indexField = matcher.group(2); //ZZ, TI
+                            queryTerm = matcher.group(3);
+
+                            //logger.info("\t@@@@ found field " + indexField);
+                            //logger.info("\t@@@@ queryTerm " + queryTerm);
+
+                            // Finally, can ask for the totaltermfreq value for this
+                            // searchterm in its indexed field:
+                            // e.g. totaltermfreq(TI,'farming'), e.g. termfreq(TI,'farming')
+                            Long totaltermfreq = (Long)doc.get("totaltermfreq("+indexField+",'"+queryTerm+"')");
+                            
+                            Integer termfreq = (Integer)doc.get("termfreq("+indexField+",'"+queryTerm+"')");
+                            
+                            //System.err.println("**** ttf = " + totaltermfreq); 
+                            //System.err.println("**** tf = " + termfreq);
+                            //logger.info("**** ttf = " + totaltermfreq); 
+                            //logger.info("**** tf = " + termfreq);
+                            solr_query_result.addTerm(queryTerm, indexField, (int) hits.getNumFound(), totaltermfreq.intValue()); // long totaltermfreq to int
+                            }
+                        }                       
+                        }
+                        if(!foundTermInfo) { // no terms extracted from query_string
                         solr_query_result.addTerm(query_string, defaultField, (int) hits.getNumFound(), -1); // no terms
                         }