root/other-projects/nightly-tasks/diffcol/trunk/task @ 27658

Revision 27658, 9.2 KB (checked in by ak19, 7 years ago)

Although task(.sh) is now deprecated, commiting a local change

Line 
1#!/bin/bash
2
3if [ "x$TASK_HOME" == "x" ]; then
4    TASK_HOME=`pwd`;
5fi
6
7export DATA_DIR=$TASK_HOME/diffcol-data
8export UPLOAD_DIR=$TASK_HOME/diffcol-reports
9export MONITOR_EMAIL=greenstone_team@cs.waikato.ac.nz
10
11# control if an existing compiled greenstone is used
12# or, if one should be checked out, which revision to checkout from svn
13SVN_OPT_REV="-r head"
14#export GSDLHOME=
15#export GSDL3SRCHOME=
16
17#check key environment vars are set
18if [ "$UPLOAD_DIR" == "" ]; then
19    echo "Please set a UPLOAD_DIR for the test in an environment.sh file"
20    exit
21fi
22if [ "$DATA_DIR" == "" ]; then
23    echo "Please set a DATA_DIR for the test in an environment.sh file"
24    exit
25fi
26if [ "$MONITOR_EMAIL" == "" ]; then
27    echo "Please set a MONITOR_EMAIL for the test in an environment.sh file"
28    exit
29fi
30
31
32if [ "$DATA_DIR" == "/" ]; then
33    echo "DATA_DIR should not be the fs root"
34    exit
35fi
36
37echo "DATA_DIR: $DATA_DIR"
38echo "UPLOAD_DIR: $UPLOAD_DIR"
39
40#create an id for this test
41dateid=`date +'%Y-%m-%d'`
42
43echo "Starting test '$dateid'"
44
45#set classpath
46CLASSPATH=
47for file in $TASK_HOME/lib/*.jar; do
48    CLASSPATH=$file:$CLASSPATH
49done
50export CLASSPATH
51
52#set the location of the full report
53xmlout=$DATA_DIR'/full-report-'$dateid'.xml'
54
55# the toplevel folder of the greenstone installation being used
56greenstone_home=
57# gsdl is the checkout folder and can be greenstone2 or greenstone3
58gsdl=greenstone2
59
60
61# Check if using existing compiled-up greenstone installation
62# and set the greenstone_home location accordingly
63if [ "x$GSDL3SRCHOME" != "x" -o "x$GSDLHOME" != "x" ]; then
64    echo "Found existing Greenstone home, will use that instead"
65    greenstone_home=$GSDLHOME
66else
67    greenstone_home="$DATA_DIR/$gsdl";
68fi
69
70
71
72function setup_greenstone() {
73
74    #clean up from previous tests
75    echo -n "about to clean up any old tests (Ctrl-C to cancel)"
76    for i in x x x x x; do sleep 1; echo -n .; done
77    echo
78
79    echo "cleaning up previous tests"
80    rm -rf $DATA_DIR
81
82    echo "creating the data dir"
83    mkdir -p $DATA_DIR || exit
84
85    cd $DATA_DIR
86
87
88        # use existing compiled-up greenstone installation
89    if [ "x$GSDL3SRCHOME" != "x" -o "x$GSDLHOME" != "x" ]; then
90        echo "Found existing Greenstone home, will use that instead"
91        return;
92    fi
93
94    # Else checkout a GS from svn into DATA_DIR
95
96
97    #svn checkout of main gsdl directory
98    echo "checkout $gsdl:"
99    svn co $SVN_OPT_REV http://svn.greenstone.org/main/trunk/greenstone2 $gsdl
100    echo "done"
101
102    cd $DATA_DIR/$gsdl
103
104
105    # since this is a bash script, we're on linux/darwin, need gnome-lib
106    # for the correct architecture. Assume all darwin is intel, not ppc
107    echo "setting up gnome-lib-minimal for compilation"
108
109# http://stackoverflow.com/questions/394230/detect-the-os-from-a-bash-script
110    os='linux';
111    gl_suffix=$os;
112    if [[ "$OSTYPE" == "darwin"* ]]; then
113        os='darwin';
114        gl_suffix=$os'-intel';
115    else
116        # linux, work out the bit architecture
117        arch=`uname -m`
118        if [[ "$arch" == *"64"* ]]; then
119        arch='64';
120        gl_suffix=$gl_suffix'-x64';
121        else
122        arch='32';
123        fi
124    fi
125
126
127    #svn checkout gnome-lib for this linux/darwin
128    cd $DATA_DIR/$gsdl/ext
129
130#   svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
131    svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-$gl_suffix.tar.gz gl.tar.gz
132    tar xvzf gl.tar.gz
133        cd gnome-lib-minimal
134        source devel.bash
135
136        cd ../..
137
138
139    #configure
140    echo "configure $gsdl: "
141    echo "<configure>" >> $xmlout
142    ./configure 2>> $DATA_DIR/compilation-errors
143    echo "</configure>" >> $xmlout
144    echo "done"
145
146    #make
147    echo "make $gsdl: "
148    echo "<make>" >> $xmlout
149    make 2>> $DATA_DIR/compilation-errors
150    echo "</make>" >> $xmlout
151    echo "done"
152
153    #make install
154    echo "make install $gsdl: "
155    echo "<make-install>" >> $xmlout
156    make install 2>> $DATA_DIR/compilation-errors
157    echo "</make-install>" >> $xmlout
158    echo "done"
159
160    # set the path to the greenstone_home variable
161    greenstone_home=$DATA_DIR/$gsdl
162}
163
164
165function run_test() {
166
167    #perform the requested subcommands, outputting xml information
168    echo '<test time="'$dateid'" id="'$dateid'">' > $xmlout
169
170    # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
171    echo "Updating $TASK_HOME/model-collect:"
172    #cd $TASK_HOME/model-collect
173    svn up $TASK_HOME/model-collect
174
175    # go to whichever greenstone_home we're using
176    cd $greenstone_home
177
178    #get svn info
179    echo "getting svn info: "
180    echo "<svn-info>" >> $xmlout
181    svn info >> $xmlout
182    echo "</svn-info>" >> $xmlout
183    echo "done"
184
185    #set up the gsdl environment with the setup.bash script
186    echo "setting up $gsdl environment: "
187        export GSDLHOME=
188        source setup.bash
189    echo "done"
190
191    #make two copies of the model-collect directory in gsdl
192    #one to be rebuild and one as the basis for comparison
193    #strip both of all .svn directories
194
195    #copy the model collections to the collect folder to be rebuilt
196    echo -n "installing test collections and model collections to new $gsdl installation... "
197
198        #clean up
199        rm -rf collect
200        rm -rf model-collect
201
202        #copy to collect and strip
203        cp -r $TASK_HOME/model-collect collect
204        find collect -type d -name .svn -exec rm -rf {} \;
205
206        #make the model copy
207        cp -r collect model-collect
208
209    echo "done"
210
211    #for each collection, import, build and diff with its model counterpart
212    for collection in collect/*; do
213
214        #escape the filename (in case of space)
215        collection=`echo $collection | sed 's@ @\\\ @g'`
216
217        #get just the basename
218        collection=`basename $collection`
219
220        echo '<collection-test name="'$collection'">' >> $xmlout
221
222        #import
223        #Ensure the OIDtype for importing is hash_on_full_filename
224        # "to make document identifiers more stable across
225        # upgrades of the software, although it means that
226        # duplicate documents contained in the collection are no
227        # longer detected automatically."
228        echo $collection' - Importing:'
229        echo "<import>" >> $xmlout
230        import.pl -OIDtype hash_on_full_filename -removeold $collection
231        echo "</import>" >> $xmlout
232        echo "done"
233
234        #build
235        echo $collection' - Building:'
236        echo "<build>" >> $xmlout
237        buildcol.pl -removeold $collection
238        echo "</build>" >> $xmlout
239        echo "done"
240
241        #rename the intermediate 'building' directory 'index'
242        echo -n $collection' - Move "building" to "index"... "'
243        rm -rf collect/$collection/index
244        mv collect/$collection/building collect/$collection/index
245        echo "done"
246
247        #diffcol
248        echo $collection' - Diffing:'
249        cd $TASK_HOME/diffcol
250        ./diffcol.pl -output xml -verbosity 10 $collection >> $xmlout
251        cd $greenstone_home
252        echo "done"
253
254        echo "</collection-test>" >> $xmlout
255    done
256
257    echo "</test>" >> $xmlout
258
259    echo "done"
260
261}
262
263function upload() {
264    # ensure the upload directory exists
265    # before trying to transfer the xml and html files across
266    # if it already existed, clear it of contents
267    if [ ! -d "$UPLOAD_DIR" ]; then
268    mkdir -p "$UPLOAD_DIR";
269#    else
270#   rm $UPLOAD_DIR/*
271    fi
272
273    scp $DATA_DIR/*.xml $DATA_DIR/*.html $UPLOAD_DIR
274}
275
276function mail_simple_message() {
277    # email out brief failure message if failed                                                       
278    echo "Checking if successful... "
279    result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
280    echo "result: "$result
281    if [ "$result" != "yes" ]; then
282        echo $gsdl' regression test for '$dateid' failed' | mail -s 'Regression Test Failed' $MONITOR_EMAIL #$WEBMASTER_EMAIL
283    fi
284}
285
286
287function mail_with_report_attached() {
288    # http://stackoverflow.com/questions/17359/how-do-i-send-a-file-as-an-email-attachment-using-linux-command-line
289
290    #echo "This is the message body" | mutt -a "/path/to/file.to.attach" -s "subject of message" -- recipient@domain.com
291
292
293    # email out with report attached, if the tests failed                                                     
294    echo "Checking if successful... "
295    result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
296    echo "result: "$result
297    if [ "$result" != "yes" ]; then
298    echo $gsdl' regression test for '$dateid' failed' | mutt -a $DATA_DIR'/report-'$dateid'.html' -s 'Regression Test Failed' -- $MONITOR_EMAIL
299    echo "Sent mail with report attached."
300    fi
301}
302
303function summarise() {
304
305    # make a summarised Xml report
306    echo -n "Summarizing the xml report... "
307    java org.apache.xalan.xslt.Process -IN $xmlout -XSL $TASK_HOME/xsl/xml-report.xsl -OUT $DATA_DIR/report-$dateid.xml
308    echo "done"
309
310    # make a summarised HTml report
311    echo -n "Creating an html summary report... "
312    java org.apache.xalan.xslt.Process -IN $DATA_DIR/report-$dateid.xml -XSL $TASK_HOME/xsl/html-report.xsl -OUT $DATA_DIR/report-$dateid.html
313    echo "done"
314
315}
316
317
318#parse arguments
319if [ "$1" == "" ]; then
320    action=all
321else
322    if [ "$1" == "setup_greenstone" ]; then
323        action=setup_greenstone
324    elif [ "$1" == "run_test" ]; then
325        action=run_test
326    elif [ "$1" == "summarise" ]; then
327        action=summarise
328    elif [ "$1" == "upload" ]; then
329        action=upload
330    elif [ "$1" == "all" ]; then
331        action=all
332    else
333        echo "bad subcommand"
334        exit
335    fi
336fi
337
338#do the requested action
339if [ "$action" == "setup_greenstone" ]; then
340    setup_greenstone
341elif [ "$action" == "run_test" ]; then
342    run_test
343#   mail_simple_message
344elif [ "$action" == "summarise" ]; then
345    summarise
346elif [ "$action" == "upload" ]; then
347    upload
348#   mail_simple_message
349    mail_with_report_attached
350elif [ "$action" == "all" ]; then
351    setup_greenstone
352    run_test
353    summarise
354    upload
355    mail_with_report_attached
356fi
357
Note: See TracBrowser for help on using the browser.