source: other-projects/nightly-tasks/diffcol/trunk/task@ 27540

Last change on this file since 27540 was 27540, checked in by ak19, 8 years ago
  1. Reports better sent to the greenstone mail id 2. Need to import with OIDtype set to hashing on the full filename, to generate stable doc ids. This is how the model collections are generated, with which the locally built ones are to be compared. 3. Need to ensure the upload directory exists.
File size: 6.9 KB
Line 
1#!/bin/bash
2
3if [ "x$TASK_HOME" == "x" ]; then
4 TASK_HOME=`pwd`;
5fi
6
7export DATA_DIR=$TASK_HOME/diffcol-data
8export UPLOAD_DIR=$TASK_HOME/diffcol-reports
9export MONITOR_EMAIL=greenstone_team@cs.waikato.ac.nz
10
11#check key environment vars are set
12if [ "$UPLOAD_DIR" == "" ]; then
13 echo "Please set a UPLOAD_DIR for the test in an environment.sh file"
14 exit
15fi
16if [ "$DATA_DIR" == "" ]; then
17 echo "Please set a DATA_DIR for the test in an environment.sh file"
18 exit
19fi
20if [ "$MONITOR_EMAIL" == "" ]; then
21 echo "Please set a MONITOR_EMAIL for the test in an environment.sh file"
22 exit
23fi
24
25
26if [ "$DATA_DIR" == "/" ]; then
27 echo "DATA_DIR should not be the fs root"
28 exit
29fi
30
31echo "DATA_DIR: $DATA_DIR"
32echo "UPLOAD_DIR: $UPLOAD_DIR"
33
34#create an id for this test
35dateid=`date +'%Y-%m-%d'`
36
37echo "Starting test '$dateid'"
38
39#set classpath
40CLASSPATH=
41for file in $TASK_HOME/lib/*.jar; do
42 CLASSPATH=$file:$CLASSPATH
43done
44export CLASSPATH
45
46#set the location of the full report
47xmlout=$DATA_DIR'/full-report-'$dateid'.xml'
48
49function setup_greenstone() {
50
51 #clean up from previous tests
52 echo -n "about to clean up any old tests (Ctrl-C to cancel)"
53 for i in x x x x x; do sleep 1; echo -n .; done
54 echo
55
56 echo "cleaning up previous tests"
57 rm -rf $DATA_DIR
58
59 echo "creating the data dir"
60 mkdir -p $DATA_DIR || exit
61
62 cd $DATA_DIR
63
64 #svn checkout of main gsdl directory
65 echo "checkout gsdl:"
66 svn co http://svn.greenstone.org/main/trunk/greenstone2 gsdl
67 echo "done"
68
69 cd $DATA_DIR/gsdl
70
71
72 # since this is a bash script, we're on linux/darwin, need gnome-lib
73 # for the correct architecture. Assume all darwin is intel, not ppc
74 echo "setting up gnome-lib-minimal for compilation"
75
76# http://stackoverflow.com/questions/394230/detect-the-os-from-a-bash-script
77 os='linux';
78 gl_suffix=$os;
79 if [[ "$OSTYPE" == "darwin"* ]]; then
80 os='darwin';
81 gl_suffix=$os'-intel';
82 else
83 # linux, worl out the bit arch
84 arch=`uname -m`
85 if [[ "$arch" == *"64"* ]]; then
86 arch='64';
87 gl_suffix=$gl_suffix'-x64';
88 else
89 arch='32';
90 fi
91 fi
92
93
94 #svn checkout gnome-lib for this linux/darwin
95 cd $DATA_DIR/gsdl/ext
96
97# svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
98 svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-$gl_suffix.tar.gz gl.tar.gz
99 tar xvzf gl.tar.gz
100 cd gnome-lib-minimal
101 source devel.bash
102
103 cd ../..
104
105
106 #configure
107 echo "configure gsdl: "
108 echo "<configure>" >> $xmlout
109 ./configure 2>> $DATA_DIR/compilation-errors
110 echo "</configure>" >> $xmlout
111 echo "done"
112
113 #make
114 echo "make gsdl: "
115 echo "<make>" >> $xmlout
116 make 2>> $DATA_DIR/compilation-errors
117 echo "</make>" >> $xmlout
118 echo "done"
119
120 #make install
121 echo "make install gsdl: "
122 echo "<make-install>" >> $xmlout
123 make install $DATA_DIR/compilation-errors
124 echo "</make-install>" >> $xmlout
125 echo "done"
126
127
128}
129
130
131function run_test() {
132
133 #perform the requested subcommands, outputting xml information
134 echo '<test time="'$dateid'" id="'$dateid'">' > $xmlout
135
136 cd $DATA_DIR/gsdl
137
138 #get svn info
139 echo "getting svn info: "
140 echo "<svn-info>" >> $xmlout
141 svn info >> $xmlout
142 echo "</svn-info>" >> $xmlout
143 echo "done"
144
145 #set up the gsdl environment with the setup.bash script
146 echo "setting up gsdl environment: "
147 export GSDLHOME=
148 source setup.bash
149 echo "done"
150
151 #make two copies of the model-collect directory in gsdl
152 #one to be rebuild and one as the basis for comparison
153 #strip both of all .svn directories
154
155 #copy the model collections to the collect folder to be rebuilt
156 echo -n "installing test collections and model collections to new gsdl installation... "
157
158 #clean up
159 rm -rf collect
160 rm -rf model-collect
161
162 #copy to collect and strip
163 cp -r $TASK_HOME/model-collect collect
164 find collect -type d -name .svn -exec rm -rf {} \;
165
166 #make the model copy
167 cp -r collect model-collect
168
169 echo "done"
170
171 #for each collection, import, build and diff with its model counterpart
172 for collection in collect/*; do
173
174 #escape the filename (in case of space)
175 collection=`echo $collection | sed 's@ @\\\ @g'`
176
177 #get just the basename
178 collection=`basename $collection`
179
180 echo '<collection-test name="'$collection'">' >> $xmlout
181
182 #import
183 #Ensure the OIDtype for importing is hash_on_full_filename
184 # "to make document identifiers more stable across
185 # upgrades of the software, although it means that
186 # duplicate documents contained in the collection are no
187 # longer detected automatically."
188 echo $collection' - Importing:'
189 echo "<import>" >> $xmlout
190 import.pl -OIDtype hash_on_full_filename $collection -removeold
191 echo "</import>" >> $xmlout
192 echo "done"
193
194 #build
195 echo $collection' - Building:'
196 echo "<build>" >> $xmlout
197 buildcol.pl $collection -removeold
198 echo "</build>" >> $xmlout
199 echo "done"
200
201 #rename the intermediate 'building' directory 'index'
202 echo -n $collection' - Move "building" to "index"... "'
203 rm -rf collect/$collection/index
204 mv collect/$collection/building collect/$collection/index
205 echo "done"
206
207 #diffcol
208 echo $collection' - Diffing:'
209 cd $TASK_HOME/diffcol
210 ./diffcol.pl -output xml -verbosity 10 $collection >> $xmlout
211 cd $DATA_DIR/gsdl
212 echo "done"
213
214 echo "</collection-test>" >> $xmlout
215 done
216
217 echo "</test>" >> $xmlout
218
219 # email out if failed
220 echo "Checking if failed... "
221 result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
222 echo "result: "$result
223 if [ "$result" != "yes" ]; then
224 echo 'gsdl regression test for '$dateid' failed' | mail -s $1' Regression Test Failed' $MONITOR_EMAIL
225 #$WEBMASTER_EMAIL
226 fi
227 echo "done"
228}
229
230function upload() {
231 # ensure the upload directory exists
232 # before trying to transfer the xml and html files across
233 if [ ! -d "$UPLOAD_DIR" ]; then
234 mkdir -p "$UPLOAD_DIR";
235 fi
236
237 scp $DATA_DIR/*.xml $DATA_DIR/*.html $UPLOAD_DIR
238}
239
240function summarise() {
241
242 # make a summarised Xml report
243 echo -n "Summarizing the xml report... "
244 java org.apache.xalan.xslt.Process -IN $xmlout -XSL $TASK_HOME/xsl/xml-report.xsl -OUT $DATA_DIR/report-$dateid.xml
245 echo "done"
246
247 # make a summarised HTml report
248 echo -n "Creating an html summary report... "
249 java org.apache.xalan.xslt.Process -IN $DATA_DIR/report-$dateid.xml -XSL $TASK_HOME/xsl/html-report.xsl -OUT $DATA_DIR/report-$dateid.html
250 echo "done"
251
252}
253
254#parse arguments
255if [ "$1" == "" ]; then
256 action=all
257else
258 if [ "$1" == "setup_greenstone" ]; then
259 action=setup_greenstone
260 elif [ "$1" == "run_test" ]; then
261 action=run_test
262 elif [ "$1" == "summarise" ]; then
263 action=summarise
264 elif [ "$1" == "upload" ]; then
265 action=upload
266 elif [ "$1" == "all" ]; then
267 action=all
268 else
269 echo "bad subcommand"
270 exit
271 fi
272fi
273
274#do the requested action
275if [ "$action" == "setup_greenstone" ]; then
276 setup_greenstone
277elif [ "$action" == "run_test" ]; then
278 run_test
279elif [ "$action" == "summarise" ]; then
280 summarise
281elif [ "$action" == "upload" ]; then
282 upload
283elif [ "$action" == "all" ]; then
284 setup_greenstone
285 run_test
286 summarise
287 upload
288fi
289
Note: See TracBrowser for help on using the repository browser.