Ignore:
Timestamp:
2015-02-03T21:57:53+13:00 (9 years ago)
Author:
ak19
Message:

Moving from using the solr jetty server to solr using the GS3 tomcat server. Now localhost:8383/solr hosts the solr server RESTful pages. Changes: 1. Minor changes to GS3 build.xml. 2. GLI no longer does the temporary stopping of the GS3 server, launching jetty server for building a solr collection, stopping jetty, restarting GS3 tomcat server. GLI leaves the GS3 server running. 3. The main changes are to ext/solr. The ext/solr/gs3-setup.sh sets the new SOLR_PORT and SOLR_HOST variables read from the GS3 build.properties, as the jetty port and host variables are no longer used. ext/solr/build.xml now puts the solr war file into tomcat's webapps, as well as helper libraries necessary (xalan related); a solr.xml context file is created from a template file and placed into tomcat's conf/Catalina/localhost; additional solr jar files are copied into tomcat/lib, as well as the slf4j bridge being copied into GS3/web/WEB-INF/lib; the solr perl code has been changed to use the new RESTful URLs and particularly to work with solr running off the GS3 tomcat server, or stop and start it as required, rather than working with (or stopping and starting) the solr jetty server. A new run_solr_server.pl executable script runs the tomcat server rather than the jetty server; major changes to the Java Solr code to no longer work with the EmbeddedSolrServer (which caused a conflict when the index is accessed by solr jetty server upon rebuild of solr collections), our solr Java code now uses HttpSolrServer to contact the solr servlet running off tomcat. 5. Still a bug: when search results go over a page after rebuilding a solr collection in GLI against a running GS3 server, the 2nd page of search results aren't present and things break. But if the server is not running, solr collections rebuild fine, so the changes do everything that GS3.06 did and more.

Location:
gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3/service/GS2SolrSearch.java

    r29710 r29711  
    2020
    2121// Greenstone classes
    22 import java.io.File;
    2322import java.util.ArrayList;
    24 import java.util.Collection;
    2523import java.util.HashMap;
    2624import java.util.Iterator;
    2725import java.util.List;
    2826import java.util.Map;
     27import java.util.Properties;
    2928import java.util.Set;
    3029import java.util.Vector;
    3130
    3231import org.apache.log4j.Logger;
    33 import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
     32import org.apache.solr.client.solrj.SolrServer;
     33import org.apache.solr.client.solrj.impl.HttpSolrServer;
    3434import org.apache.solr.client.solrj.response.FacetField;
    35 import org.apache.solr.core.CoreContainer;
    36 import org.apache.solr.core.SolrCore;
    3735import org.greenstone.LuceneWrapper4.SharedSoleneQueryResult;
    3836import org.greenstone.gsdl3.util.FacetWrapper;
    … …  
    5048{
    5149
     50  public static final String SOLR_SERVLET_SUFFIX = "/solr";
    5251  protected static final String SORT_ORDER_PARAM = "sortOrder";
    5352  protected static final String SORT_ORDER_DESCENDING = "1";
    … …  
    5655    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2SolrSearch.class.getName());
    5756
    58     static protected CoreContainer all_solr_cores = null;
    59 
    60     protected HashMap solr_core_cache;
     57        protected String solr_servlet_base_url;
     58    protected HashMap<String, SolrServer> solr_core_cache;
    6159    protected SolrQueryWrapper solr_src = null;
    6260
    … …  
    7270        // based on 'level' parameter passed in to 'setUpQueryer()'
    7371
    74         solr_core_cache = new HashMap();
    75 
    76         if (all_solr_cores == null)
    77         {
    78             // Share one CoreContainer across all sites/collections
    79             try
    80             {
    81                 String gsdl3_writablehome = GlobalProperties.getGSDL3WritableHome();
    82                 String solr_ext_name = GlobalProperties.getProperty("gsdlext.solr.dirname", "solr");
    83 
    84                 String solr_home_str = GSFile.extHome(gsdl3_writablehome, solr_ext_name);
    85 
    86                 all_solr_cores = new CoreContainer(solr_home_str);
    87             }
    88             catch (Exception e)
    89             {
    90                 e.printStackTrace();
    91             }
    92         }
    93 
    94         this.solr_src = new SolrQueryWrapper();
     72        solr_core_cache = new HashMap<String, SolrServer>();
     73
     74        this.solr_src = new SolrQueryWrapper();     
     75
     76        // Create the solr servlet url on GS3's tomcat. By default it's "http://localhost:8383/solr"
     77        // Don't do this in configure(), since the tomcat url will remain unchanged while tomcat is running
     78        try {
     79            Properties globalProperties = new Properties();
     80            globalProperties.load(Class.forName("org.greenstone.util.GlobalProperties").getClassLoader().getResourceAsStream("global.properties"));
     81            String host = globalProperties.getProperty("tomcat.server", "localhost");
     82            String port = globalProperties.getProperty("tomcat.port", "8383");
     83            String protocol = globalProperties.getProperty("tomcat.protocol", "http");
     84           
     85            String portStr = port.equals("80") ? "" : ":"+port;
     86            solr_servlet_base_url = protocol+"://"+host+portStr+SOLR_SERVLET_SUFFIX;
     87        } catch(Exception e) {
     88            logger.error("Error reading greenstone's tomcat solr server properties from global.properties", e);
     89        }
    9590    }
    9691
    … …  
    10095        boolean success = super.configure(info, extra_info);
    10196
    102         // 1. Make the CoreContainer reload solr.xml
    103         // This is particularly needed for when activate.pl is executed during
    104         // a running GS3 server. At that point, the solr collection is reactivated and
    105         // we need to tell Greenstone that the solr index has changed. This requires
    106         // the CoreContainer to reload the solr.xml file, and it all works again.
    107 
    108         solr_core_cache.clear(); // clear the map of solr cores for this collection added to the map upon querying
    109        
    110         // Reload the updated solr.xml into the CoreContainer
    111         // (Doing an all_solr_cores.shutdown() first doesn't seem to be required)
    112         try {   
    113             String solr_home_str = all_solr_cores.getSolrHome();
    114             File solr_home = new File(solr_home_str);
    115             File solr_xml = new File( solr_home,"solr.xml" );
    116            
    117             //all_solr_cores.load(solr_home_str,solr_xml);
    118             all_solr_cores.load();
    119 
    120         } catch (Exception e) {
    121             logger.error("Exception in GS2SolrSearch.configure(): " + e.getMessage());
    122             e.printStackTrace();
    123             return false;
    124         }
    125        
     97        // clear the map of solr cores for this collection added to the map upon querying
     98        solr_core_cache.clear();
     99
    126100        if(!success) {
    127101            return false;
    128102        }
    129103       
    130         // 2. Setting up facets
     104        // Setting up facets
    131105        // TODO - get these from build config, in case some haven't built
    132106        Element searchElem = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
    … …  
    170144        this.solr_src.cleanUp();
    171145
    172 
    173         // 1. clear the map keeping track of the solrcores' EmbeddedSolrServers in this collection
     146        // clear the map keeping track of the SolrServers in this collection
    174147        solr_core_cache.clear();
    175 
    176         // 2. For solr 4.7.2., GLI (and ant stop from cmd) is unable to shutdown the tomcat server fully,
    177         // IF any collection has been previewed AND if there are any solr collections in the collect folder.
    178         // This is because although the GS3 server seems to have stopped running at this stage, running a
    179         // `ps aux | grep tomcat` reveals that some part of tomcat is still running. It seems to be still
    180         // holding on to the cores. Doing an all_cores.shutdown() here, stops GS3 from hanging on to the cores
    181         // while still preserving the core desciptions in web/ext/solr.xml as needed when restarting the GS3 server.
    182 
    183         // Need GS3 server (tomcat) to release the cores, else a part of tomcat is still running in the background
    184         // on ant stop, holding a lock on the cores. Doing shutdown() preserves core descriptions in solr.xml
    185         all_solr_cores.shutdown();
    186         all_solr_cores = null;
    187148    }
    188149
    … …  
    356317        // solr-core, (caching the result in 'solr_core_cache')
    357318        String core_name = getCollectionCoreNamePrefix() + "-" + index;
    358 
    359         EmbeddedSolrServer solr_core = null;
     319       
     320        SolrServer solr_core = null;
    360321
    361322        if (!solr_core_cache.containsKey(core_name))
    362         {
    363             solr_core = new EmbeddedSolrServer(all_solr_cores, core_name);
    364 
    365             solr_core_cache.put(core_name, solr_core);
     323        {           
     324            solr_core = new HttpSolrServer(this.solr_servlet_base_url+"/"+core_name);
     325            solr_core_cache.put(core_name, solr_core);         
    366326        }
    367327        else
    368328        {
    369             solr_core = (EmbeddedSolrServer) solr_core_cache.get(core_name);
     329            solr_core = solr_core_cache.get(core_name);
    370330        }
    371331
    … …  
    381341        try
    382342        {
    383             //SharedSoleneQueryResult sqr = this.solr_src.runQuery(query);
    384343            SharedSoleneQueryResult sqr = this.solr_src.runQuery(query);
    385344

  • gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3/service/SolrSearch.java

    r29710 r29711  
    1313import org.w3c.dom.NodeList;
    1414
     15import java.util.ArrayList;
    1516import java.util.HashMap;
    16 import java.util.ArrayList;
    17 import java.util.Iterator;
     17import java.util.Properties;
    1818
    19 
     19import org.apache.log4j.Logger;
    2020import org.apache.solr.client.solrj.SolrQuery;
    2121import org.apache.solr.client.solrj.SolrServer;
    2222import org.apache.solr.client.solrj.SolrServerException;
    23 import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
     23import org.apache.solr.client.solrj.impl.HttpSolrServer;
    2424import org.apache.solr.client.solrj.response.QueryResponse;
    2525import org.apache.solr.common.SolrDocument;
    2626import org.apache.solr.common.SolrDocumentList;
    27 import org.apache.solr.common.params.ModifiableSolrParams;
    28 import org.apache.solr.common.params.SolrParams;
    2927import org.apache.solr.common.util.NamedList;
    3028import org.apache.solr.servlet.SolrRequestParsers;
    31 import org.apache.solr.core.CoreContainer;
    32 import org.apache.solr.core.SolrCore;
    33 
    34 import java.io.File;
    35 import java.util.Collection;
    36 
    37 
    38 import org.apache.log4j.*;
    3929
    4030
    4131public class SolrSearch extends LuceneSearch {
    4232
     33    public static final String SOLR_SERVLET_SUFFIX = "/solr";
    4334    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.SolrSearch.class.getName());
    4435
    45     static protected CoreContainer solr_cores = null;
    46     protected HashMap solr_server;
     36    protected String solr_servlet_base_url;
     37    protected HashMap<String, SolrServer> solr_server;
    4738
    4839    public SolrSearch()
    4940    {
    50     solr_server = new HashMap();
     41    solr_server = new HashMap<String, SolrServer>();   
    5142
    52     if (solr_cores == null) {
    53         // Share one CoreContainer across all sites/collections
    54         try {
    55        
    56         String gsdl3_home = GlobalProperties.getGSDL3Home();
    57         String solr_ext_name = GlobalProperties.getProperty("gsdlext.solr.dirname","solr");
    58        
    59         String solr_home_str = GSFile.extHome(gsdl3_home,solr_ext_name);
    60 
    61         solr_cores = new CoreContainer(solr_home_str);
    62         }
    63         catch (Exception e) {
    64         e.printStackTrace();
    65         }
     43    // Create the solr servlet url on GS3's tomcat. By default it's "http://localhost:8383/solr"
     44    // Don't do this in configure(), since the tomcat url will remain unchanged while tomcat is running
     45    try {
     46        Properties globalProperties = new Properties();
     47        globalProperties.load(Class.forName("org.greenstone.util.GlobalProperties").getClassLoader().getResourceAsStream("global.properties"));
     48        String host = globalProperties.getProperty("tomcat.server", "localhost");
     49        String port = globalProperties.getProperty("tomcat.port", "8383");
     50        String protocol = globalProperties.getProperty("tomcat.protocol", "http");
     51       
     52        String portStr = port.equals("80") ? "" : ":"+port;
     53        solr_servlet_base_url = protocol+"://"+host+portStr+SOLR_SERVLET_SUFFIX;
     54    } catch(Exception e) {
     55        logger.error("Error reading greenstone's tomcat solr server properties from global.properties", e);
    6656    }
    6757    }
    … …  
    7666    super.cleanUp();
    7767   
    78         // 1. clear the map keeping track of the solrcores' EmbeddedSolrServers in this collection
    79         solr_server.clear();
    80 
    81         // 2. Need GS3 server (tomcat) to release the cores, else a part of tomcat is still running in the background
    82         // on ant stop, holding a lock on the cores. Doing shutdown() preserves core descriptions in solr.xml
    83         solr_cores.shutdown();
    84         solr_cores = null;
     68    // clear the map keeping track of the SolrServers in this collection
     69    solr_server.clear();
    8570    }
    8671
    … …  
    8873    // adjusted configure to bring it up to speed with changes in GS2SolrSearch (for activate.pl) - not yet tested
    8974    public boolean configure(Element info, Element extra_info) {
    90         boolean success = super.configure(info, extra_info);
     75    boolean success = super.configure(info, extra_info);
    9176   
    92         // 1. Make the CoreContainer reload solr.xml
    93         // This is particularly needed for when activate.pl is executed during
    94         // a running GS3 server. At that point, the solr collection is reactivated and
    95         // we need to tell Greenstone that the solr index has changed. This requires
    96         // the CoreContainer to reload the solr.xml file, and it all works again.
    97 
    98         solr_server.clear(); // clear the map of solr cores for this collection added to the map upon querying
    99        
    100         // Reload the updated solr.xml into the CoreContainer
    101         // (Doing a solr_cores.shutdown() first doesn't seem to be required)
    102         try {   
    103             String solr_home_str = solr_cores.getSolrHome();
    104             File solr_home = new File(solr_home_str);
    105             File solr_xml = new File( solr_home,"solr.xml" );
    106            
    107             //solr_cores.load(solr_home_str,solr_xml);
    108             solr_cores.load();
    109         } catch (Exception e) {
    110             logger.error("Exception in SolrSearch.configure(): " + e.getMessage());
    111             e.printStackTrace();
    112             return false;
    113         }
     77    // clear the map of solr cores for this collection added to the map upon querying
     78    solr_server.clear();
     79   
     80    if(!success) {
     81        return false;
     82    }
    11483
    11584    // initialize required number of SolrCores based on values
    … …  
    12392        String core_name = core_name_prefix + "-" + idx_name;
    12493
    125         EmbeddedSolrServer solr_core
    126         = new EmbeddedSolrServer(solr_cores,core_name);
    127 
     94        SolrServer solr_core = new HttpSolrServer(this.solr_servlet_base_url+"/"+core_name);
    12895        solr_server.put(core_name,solr_core);
    12996    }
     97   
    13098
    13199    return success;
    … …  
    165133
    166134        try {
     135       
     136        // Use SolrQuery with HttpSolrServer instead of ModifiableSolrParams,
     137        // see http://stackoverflow.com/questions/13944567/querying-solr-server-using-solrj     
     138        SolrQuery solrParams = new SolrQuery(query_string); // initialised with q url-parameter
     139        //solrparams.setRequestHandler("/select"); // default. Or try "select"
    167140
    168         ModifiableSolrParams solrParams = new ModifiableSolrParams();
    169         solrParams.set("q", query_string);
    170         //solrParams.set("start", start);
    171         //solrParams.set("rows", nbDocuments);
     141        ///solrParams.set("start", start);
     142        ///solrParams.set("rows", nbDocuments);
     143
    172144
    173145        String core_name = getCollectionCoreNamePrefix() + "-" + index;
    174146
    175         EmbeddedSolrServer solr_core = (EmbeddedSolrServer)solr_server.get(core_name);
    176 
     147        // http://stackoverflow.com/questions/17026530/accessing-a-cores-default-handler-through-solrj-using-setquerytype
     148        // default request handler is /select, see http://wiki.apache.org/solr/CoreQueryParameters
     149        SolrServer solr_core = solr_server.get(core_name);
    177150        QueryResponse solrResponse = solr_core.query(solrParams);
    178151

  • gs3-extensions/solr/trunk/src/src/java/org/greenstone/gsdl3/util/SolrQueryWrapper.java

    r29664 r29711  
    241241        }
    242242    } else {
    243         System.err.println("#### Not an EmbeddedSolrServer. This shouldn't happen." + solr_core.getClass());
    244         logger.error("#### Not an EmbeddedSolrServer. This shouldn't happen."  + solr_core.getClass());
     243        System.err.println("#### Not an EmbeddedSolrServer. SolrQueryWrapper.getTerms() not yet implemented for " + solr_core.getClass());
     244        logger.error("#### Not an EmbeddedSolrServer. SolrQueryWrapper.getTerms() not yet implemented for "  + solr_core.getClass());
    245245    }
    246246   
Note: See TracChangeset for help on using the changeset viewer.