root/gs3-extensions/solr/trunk/src/build.xml @ 29711

Revision 29711, 11.1 KB (checked in by ak19, 6 years ago)

Moving from using the solr jetty server to solr using the GS3 tomcat server. Now localhost:8383/solr hosts the solr server RESTful pages. Changes: 1. Minor changes to GS3 build.xml. 2. GLI no longer does the temporary stopping of the GS3 server, launching jetty server for building a solr collection, stopping jetty, restarting GS3 tomcat server. GLI leaves the GS3 server running. 3. The main changes are to ext/solr. The ext/solr/gs3-setup.sh sets the new SOLR_PORT and SOLR_HOST variables read from the GS3 build.properties, as the jetty port and host variables are no longer used. ext/solr/build.xml now puts the solr war file into tomcat's webapps, as well as helper libraries necessary (xalan related); a solr.xml context file is created from a template file and placed into tomcat's conf/Catalina/localhost; additional solr jar files are copied into tomcat/lib, as well as the slf4j bridge being copied into GS3/web/WEB-INF/lib; the solr perl code has been changed to use the new RESTful URLs and particularly to work with solr running off the GS3 tomcat server, or stop and start it as required, rather than working with (or stopping and starting) the solr jetty server. A new run_solr_server.pl executable script runs the tomcat server rather than the jetty server; major changes to the Java Solr code to no longer work with the EmbeddedSolrServer? (which caused a conflict when the index is accessed by solr jetty server upon rebuild of solr collections), our solr Java code now uses HttpSolrServer? to contact the solr servlet running off tomcat. 5. Still a bug: when search results go over a page after rebuilding a solr collection in GLI against a running GS3 server, the 2nd page of search results aren't present and things break. But if the server is not running, solr collections rebuild fine, so the changes do everything that GS3.06 did and more.

Line 
1<?xml version="1.0"?>
2<project name="gs3-solr-ext" default="usage" basedir=".">
3
4  <!-- DIRECTORY LOCATIONS -->
5  <!-- local ext paths -->
6  <property name="src.home" value="${basedir}/src/java"/>
7  <property name="build.home" value="${basedir}/build"/>
8  <available property="ext.web.exists" file="${basedir}/web" type="dir" />
9
10  <!-- greenstone3 paths -->
11  <!-- http://stackoverflow.com/questions/3136849/how-do-i-convert-a-relative-path-in-ant-to-an-absolute-path
12    http://stackoverflow.com/questions/8295860/pathconvert-with-relative-file-names -->
13  <property name="web.home" location="${basedir}/../../web"/> <!-- location property creates an absolute path -->
14  <property name="localsite.collectdir" value="${web.home}/sites/localsite/collect"/>
15  <property name="web.classesdir" value="${web.home}/WEB-INF/classes"/>
16  <property name="web.libdir" value="${web.home}/WEB-INF/lib"/> 
17  <property name="web.extdir" value="${web.home}/ext/solr"/>
18
19  <property name="tomcat.dir" location="${basedir}/../../packages/tomcat"/> <!-- location property creates an absolute path -->
20  <property name="tomcat.lib.dir" value="${tomcat.dir}/lib"/>
21  <property name="tomcat.context.dir" value="${tomcat.dir}/conf/Catalina/localhost"/>
22
23  <path id="compile.classpath">
24    <!-- gs3 jar files we need -->
25    <pathelement location="${web.libdir}/gsdl3.jar"/>
26    <pathelement location="${web.libdir}/gutil.jar"/>
27    <pathelement location="${web.libdir}/LuceneWrapper4.jar"/>
28    <pathelement location="${web.libdir}/gson-1.7.1.jar"/>
29    <!-- local jar files -->
30    <fileset dir="lib/java">
31      <include name="*.jar"/>
32    </fileset>
33    <pathelement location="lib/servlet-api-3.0.jar"/>
34  </path>
35
36
37  <!-- FILE LISTINGS.-->
38  <!-- Created as property elements rather than as filelist elements, since
39    they can then be reused for the add-service and delete-service targets. -->
40  <property name="jars"
41        value="solr-core-4.7.2.jar
42        solr-solrj-4.7.2.jar
43        lucene-core-4.7.2.jar
44        lucene-analyzers-common-4.7.2.jar
45        lucene-analyzers-kuromoji-4.7.2.jar
46        lucene-analyzers-phonetic-4.7.2.jar
47        lucene-grouping-4.7.2.jar
48        lucene-highlighter-4.7.2.jar
49        lucene-queries-4.7.2.jar
50        lucene-queryparser-4.7.2.jar
51        lucene-memory-4.7.2.jar
52        lucene-misc-4.7.2.jar
53        lucene-spatial-4.7.2.jar
54        lucene-suggest-4.7.2.jar
55        httpclient-4.3.1.jar
56        httpcore-4.3.jar
57        httpmime-4.3.1.jar
58        commons-fileupload-1.2.1.jar
59        commons-io-2.1.jar
60        commons-lang-2.6.jar
61        velocity-1.6.1.jar
62        log4j-over-slf4j-1.6.6.jar
63        slf4j-api-1.6.6.jar
64        slf4j-jdk14-1.6.6.jar
65        spatial4j-0.4.1.jar
66        zookeeper-3.4.5.jar
67        guava-14.0.1.jar
68        noggit-0.5.jar"/>
69<!--        jcl-over-slf4j-1.6.6.jar
70        jul-to-slf4j-1.6.6.jar
71        log4j-1.2.16.jar
72        wstx-asl-3.2.7.jar
73-->
74<!--
75   lucene-spellchecker-3.3.0.jar replaced with lucene-suggest-4.7.2.jar, since both contain packages org/.../spell and org/.../suggest
76   
77   commons-fileupload-1.2.1.jar - in solr war file
78   velocity-1.6.1.jar
79   log4j-over-slf4j-1.6.1.jar - have to download slf4j jar file for version 1.6.6
80   slf4j-jdk14-1.6.1.jar - have to download slf4j jar file for version 1.6.6
81  -->
82
83  <property name="java-service-files"
84    value="SolrSearch.java         
85    GS2SolrSearch.java
86    GS2SolrRetrieve.java"/> 
87           
88  <property name="java-util-files"
89    value="SolrFacetWrapper.java
90    SolrQueryWrapper.java           
91    SolrQueryResult.java"/>           
92 
93 <property name="property-files"
94    value="GS2SolrSearch.properties"/> 
95
96           
97  <!-- TARGETS -->
98  <target name="usage" description="Print a help message">
99    <echo message="  Execute 'ant -projecthelp' for a list of targets."/>
100    <echo message="  Execute 'ant -help' for Ant help."/>
101    <echo>To install the Solr extension for Greenstone3, run 'ant add-service'.
102    To remove the files and folders installed by add-service, run 'ant del-service'.
103    </echo>
104  </target>         
105 
106  <target name="compile" description="Compile up the solr java classes, jar them up, and install gs3-solr.jar into Greenstone's web/WEB-INF/lib dir">
107    <mkdir dir="${build.home}"/>
108    <javac srcdir="${src.home}" destdir="${build.home}" debug="true">
109      <classpath>
110    <path refid="compile.classpath"/>
111      </classpath>
112    </javac>
113    <jar destfile="${build.home}/gs3-solr.jar">
114      <fileset dir="${build.home}">
115        <include name="org/greenstone/gsdl3/**"/>
116      </fileset>
117      <manifest>
118        <attribute name="Built-By" value="${user.name}" />
119      </manifest>
120    </jar>
121    <copy file="${build.home}/gs3-solr.jar" todir="${web.libdir}"/>
122  </target>
123 
124  <target name="copy-files" description="Helper-target: copy files across for add-service target">
125   
126    <echo/>
127    <echo>Adding to gsdl3 properties area: properties/${property-files}</echo>
128    <copy todir="${web.classesdir}">
129          <filelist id="prop-files" dir="properties" files="${property-files}"/>
130    </copy>
131    <echo/>
132    <echo>Adding to gsdl3 web jar lib directory: ${basedir}/lib/java's ${jars}</echo>
133    <copy todir="${web.libdir}">
134          <filelist id="jar-files" dir="lib/java" files="${jars}"/>
135    </copy>
136   
137    <echo/>
138    <echo>Creating web extension directory: ${web.extdir}</echo>
139    <mkdir dir="${web.extdir}"/>
140   
141    <echo/>
142    <echo>Adding to gsdl3 web solr ext directory: solr.xml and solr.xml.in</echo>
143    <copy file="solr.xml.in" tofile="${web.extdir}/solr.xml.in"/>
144    <pathconvert targetos="unix" property="src.gsdl3.home.unix">
145      <path path="../../web"/>
146    </pathconvert>
147    <filter token="gsdl3home" value="${src.gsdl3.home.unix}"/>
148    <copy file="solr.xml.in" tofile="${web.extdir}/solr.xml" filtering="true" overwrite="true"/>
149   
150    <echo/>
151    <echo>Adding example solr-jdbm-demo collection to ${localsite.collectdir}</echo>
152    <copy todir="${localsite.collectdir}/solr-jdbm-demo"
153          preservelastmodified="true"
154          failonerror="true" > 
155      <fileset dir="${basedir}/collect/solr-jdbm-demo" includes="**"/> 
156    </copy>
157    <echo>Unzipping pre-built index</echo>
158    <unzip dest="${localsite.collectdir}/solr-jdbm-demo" src="${localsite.collectdir}/solr-jdbm-demo/index.zip" />
159  </target>
160
161  <!-- Setting up solr to work with tomcat server instead of jetty server -->
162  <target name="solr-for-tomcat" description="Helper-target: setting up solr to work tomcat">
163   
164    <echo>Copying solr jars needed for running solr with tomcat: ${basedir}/lib/ext</echo> 
165    <copy todir="${tomcat.lib.dir}">     
166      <fileset dir="lib/ext">
167    <include name="*.jar"/>
168      </fileset>
169    </copy>
170   
171    <!-- slf4j and commons logging bridge needed to avoid exception about incompatibility in tomcat log files-->
172    <echo>Copying ${basedir}/lib/ext/jcl-over-slf4j-1.6.6.jar again to ${web.libdir}</echo>
173    <copy todir="${web.libdir}">
174      <filelist id="logging-bridge" dir="lib/ext" files="jcl-over-slf4j-1.6.6.jar"/>
175    </copy>
176
177    <echo>Moving xalan related jar files from  ${web.libdir} to ${tomcat.lib.dir}, to share with solr</echo>
178    <move todir="${tomcat.lib.dir}">
179      <filelist dir="${web.libdir}">
180    <file name="xalan.jar"/>
181    <file name="xercesImpl.jar"/>
182    <file name="xml-apis.jar"/>
183    <file name="xsltc.jar"/>
184    <file name="serializer.jar"/>
185      </filelist>
186    </move>
187   
188    <echo>Copying ${basedir}/webapps/solr.war to ${tomcat.dir}/webapps</echo>
189    <copy todir="${tomcat.dir}/webapps" file="webapps/solr.war" />
190    <echo>Generating solr context file in ${tomcat.context.dir}</echo>
191   
192    <!-- we want unix paths (forward slashes) in the tomcat context file -->
193    <pathconvert targetos="unix" property="gsdl3.web.home">
194      <path path="${web.home}"/><!-- creates an absolute path-->
195    </pathconvert>
196    <pathconvert targetos="unix" property="tomcat.home">
197      <path path="${tomcat.dir}"/><!-- creates an absolute path-->
198    </pathconvert>
199    <filter token="gsdl3home" value="${gsdl3.web.home}"/>
200    <filter token="tomcathome" value="${tomcat.home}"/>
201    <copy file="solr-tomcat-context.xml.in" tofile="${tomcat.context.dir}/solr.xml" filtering="true" overwrite="true"/>
202
203  </target> 
204
205  <!-- copy the content of the web folder (avoiding the top-level .svn directory) -->
206  <target name="copy-solr-web" if="ext.web.exists">
207    <echo/>
208    <echo>Copy to gsdl3 web: the content of the ${basedir}/web folder (excluding .svn)</echo>
209    <copy todir="${web.extdir}">
210      <dirset dir="${basedir}/web">
211    <exclude name=".svn"/>
212      </dirset>
213    </copy>
214  </target>
215
216
217  <target name="add-service" depends="copy-solr-web,copy-files,solr-for-tomcat,compile" description="Run this target to setup the Solr extension for Greenstone3" />
218 
219  <target name="del-service" depends="del-files,del-solr-for-tomcat"
220    description="Run this target to unset the Solr extension for Greenstone3" />
221
222
223  <target name="del-files" description="Helper-target to delete files for del-service target">
224    <!-- failonerror is set to false in case some files don't exist
225    and can't be deleted therefore -->
226
227    <echo/>
228    <echo>Removing from gsdl3 properties area: properties/${property-files}</echo>
229    <delete failonerror="false">
230          <filelist dir="${web.classesdir}" files="${property-files}"/>
231    </delete>
232
233    <echo/>
234    <echo>Removing from gsdl3 web jar lib directory: ${basedir}/lib/java's ${jars} gs3-solr.jar</echo>
235    <delete failonerror="false">
236          <filelist dir="${web.libdir}" files="${jars} gs3-solr.jar"/>
237    </delete>   
238   
239    <echo/>
240    <echo>Removing web solr extension directory: ${web.extdir}</echo>
241    <delete failonerror="false" includeEmptyDirs="true" dir="${web.extdir}"/>   
242
243    <echo/>
244    <echo>Removing solr-jdbm-demo collection from: ${localsite.collectdir}</echo>
245    <delete failonerror="false" includeEmptyDirs="true" dir="${localsite.collectcdir}/solr-jdbm-demo"/>
246
247  </target>
248 
249  <target name="del-solr-for-tomcat" description="Helper-target to remove files for getting solr to work with tomcat">
250
251    <echo/>
252    <echo>Removing solr modifications to tomcat: context file, solr.war and deployed version, jar files</echo>
253
254    <!-- Actually need to ensure tomcat is not running at this point
255         But if solr.war is deleted first, it can't be re-deployed when deleting the solr folder subsequently
256      -->
257    <delete failonerror="false" file="${tomcat.dir}/webapps/solr.war"/>
258    <delete failonerror="false" includeEmptyDirs="true" dir="${tomcat.dir}/webapps/solr"/>
259
260    <delete failonerror="false" file="${tomcat.context.dir}/solr.xml"/>
261
262    <!-- delete all the jar files in tomcat/lib that were copied from ext/solr/lib/ext
263         which are all the jar files that are present in both tomcat/lib and ext/solr/lib/ext
264         https://ant.apache.org/manual/Types/selectors.html#presentselect -->
265    <delete failonerror="false">     
266      <fileset dir="${tomcat.lib.dir}" includes="**/*.jar">     
267        <present present="both" targetdir="lib/ext"/>
268      </fileset>
269    </delete>
270
271    <!-- remove logging bridge jar file that was added into greenstone 3 web lib area -->
272    <delete failonerror="false" file="${web.libdir}/jcl-over-slf4j-1.6.6.jar"/>
273
274    <!-- moving shared xalan related jar files back from tomcat lib to GS3 web lib -->
275    <move todir="${web.libdir}">
276      <filelist dir="${tomcat.lib.dir}">
277        <file name="xalan.jar"/>
278        <file name="xercesImpl.jar"/>
279        <file name="xml-apis.jar"/>
280        <file name="xsltc.jar"/>
281        <file name="serializer.jar"/>
282      </filelist>
283    </move>
284
285    <!--http://stackoverflow.com/questions/2140637/how-do-i-build-a-list-of-file-names-->
286  </target>
287 
288</project>
Note: See TracBrowser for help on using the browser.