root/other-projects/nightly-tasks/diffcol/trunk/task_old @ 32165

Revision 28003, 9.2 KB (checked in by ak19, 6 years ago)

The task file (bash script) has been replaced with the perl version task.pl for some time, but don't want people to accidentally run the task bash script, so I'm renaming it.

Line 
1#!/bin/bash
2
3if [ "x$TASK_HOME" == "x" ]; then
4    TASK_HOME=`pwd`;
5fi
6
7export DATA_DIR=$TASK_HOME/diffcol-data
8export UPLOAD_DIR=$TASK_HOME/diffcol-reports
9export MONITOR_EMAIL=greenstone_team@cs.waikato.ac.nz
10
11# control if an existing compiled greenstone is used
12# or, if one should be checked out, which revision to checkout from svn
13SVN_OPT_REV="-r head"
14#export GSDLHOME=
15#export GSDL3SRCHOME=
16
17#check key environment vars are set
18if [ "$UPLOAD_DIR" == "" ]; then
19    echo "Please set a UPLOAD_DIR for the test in an environment.sh file"
20    exit
21fi
22if [ "$DATA_DIR" == "" ]; then
23    echo "Please set a DATA_DIR for the test in an environment.sh file"
24    exit
25fi
26if [ "$MONITOR_EMAIL" == "" ]; then
27    echo "Please set a MONITOR_EMAIL for the test in an environment.sh file"
28    exit
29fi
30
31
32if [ "$DATA_DIR" == "/" ]; then
33    echo "DATA_DIR should not be the fs root"
34    exit
35fi
36
37echo "DATA_DIR: $DATA_DIR"
38echo "UPLOAD_DIR: $UPLOAD_DIR"
39
40#create an id for this test
41dateid=`date +'%Y-%m-%d'`
42
43echo "Starting test '$dateid'"
44
45#set classpath
46CLASSPATH=
47for file in $TASK_HOME/lib/*.jar; do
48    CLASSPATH=$file:$CLASSPATH
49done
50export CLASSPATH
51
52#set the location of the full report
53xmlout=$DATA_DIR'/full-report-'$dateid'.xml'
54
55# the toplevel folder of the greenstone installation being used
56greenstone_home=
57# gsdl is the checkout folder and can be greenstone2 or greenstone3
58gsdl=greenstone2
59
60
61# Check if using existing compiled-up greenstone installation
62# and set the greenstone_home location accordingly
63if [ "x$GSDL3SRCHOME" != "x" -o "x$GSDLHOME" != "x" ]; then
64    echo "Found existing Greenstone home, will use that instead"
65    greenstone_home=$GSDLHOME
66else
67    greenstone_home="$DATA_DIR/$gsdl";
68fi
69
70
71
72function setup_greenstone() {
73
74    #clean up from previous tests
75    echo -n "about to clean up any old tests (Ctrl-C to cancel)"
76    for i in x x x x x; do sleep 1; echo -n .; done
77    echo
78
79    echo "cleaning up previous tests"
80    rm -rf $DATA_DIR
81
82    echo "creating the data dir"
83    mkdir -p $DATA_DIR || exit
84
85    cd $DATA_DIR
86
87
88        # use existing compiled-up greenstone installation
89    if [ "x$GSDL3SRCHOME" != "x" -o "x$GSDLHOME" != "x" ]; then
90        echo "Found existing Greenstone home, will use that instead"
91        return;
92    fi
93
94    # Else checkout a GS from svn into DATA_DIR
95
96
97    #svn checkout of main gsdl directory
98    echo "checkout $gsdl:"
99    svn co $SVN_OPT_REV http://svn.greenstone.org/main/trunk/greenstone2 $gsdl
100    echo "done"
101
102    cd $DATA_DIR/$gsdl
103
104
105    # since this is a bash script, we're on linux/darwin, need gnome-lib
106    # for the correct architecture. Assume all darwin is intel, not ppc
107    echo "setting up gnome-lib-minimal for compilation"
108
109# http://stackoverflow.com/questions/394230/detect-the-os-from-a-bash-script
110    os='linux';
111    gl_suffix=$os;
112    if [[ "$OSTYPE" == "darwin"* ]]; then
113        os='darwin';
114        gl_suffix=$os'-intel';
115    else
116        # linux, work out the bit architecture
117        arch=`uname -m`
118        if [[ "$arch" == *"64"* ]]; then
119        arch='64';
120        gl_suffix=$gl_suffix'-x64';
121        else
122        arch='32';
123        fi
124    fi
125
126
127    #svn checkout gnome-lib for this linux/darwin
128    cd $DATA_DIR/$gsdl/ext
129
130#   svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
131    svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-$gl_suffix.tar.gz gl.tar.gz
132    tar xvzf gl.tar.gz
133        cd gnome-lib-minimal
134        source devel.bash
135
136        cd ../..
137
138
139    #configure
140    echo "configure $gsdl: "
141    echo "<configure>" >> $xmlout
142    ./configure 2>> $DATA_DIR/compilation-errors
143    echo "</configure>" >> $xmlout
144    echo "done"
145
146    #make
147    echo "make $gsdl: "
148    echo "<make>" >> $xmlout
149    make 2>> $DATA_DIR/compilation-errors
150    echo "</make>" >> $xmlout
151    echo "done"
152
153    #make install
154    echo "make install $gsdl: "
155    echo "<make-install>" >> $xmlout
156    make install 2>> $DATA_DIR/compilation-errors
157    echo "</make-install>" >> $xmlout
158    echo "done"
159
160    # set the path to the greenstone_home variable
161    greenstone_home=$DATA_DIR/$gsdl
162}
163
164
165function run_test() {
166
167    #perform the requested subcommands, outputting xml information
168    echo '<test time="'$dateid'" id="'$dateid'">' > $xmlout
169
170    # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
171    echo "Updating $TASK_HOME/model-collect:"
172    #cd $TASK_HOME/model-collect
173    svn up $TASK_HOME/model-collect
174
175    # go to whichever greenstone_home we're using
176    cd $greenstone_home
177
178    #get svn info
179    echo "getting svn info: "
180    echo "<svn-info>" >> $xmlout
181    svn info >> $xmlout
182    echo "</svn-info>" >> $xmlout
183    echo "done"
184
185    #set up the gsdl environment with the setup.bash script
186    echo "setting up $gsdl environment: "
187        export GSDLHOME=
188        source setup.bash
189    echo "done"
190
191    #make two copies of the model-collect directory in gsdl
192    #one to be rebuild and one as the basis for comparison
193    #strip both of all .svn directories
194
195    #copy the model collections to the collect folder to be rebuilt
196    echo -n "installing test collections and model collections to new $gsdl installation... "
197
198        #clean up
199        rm -rf collect
200        rm -rf model-collect
201
202        #copy to collect and strip
203        cp -r $TASK_HOME/model-collect collect
204        find collect -type d -name .svn -exec rm -rf {} \;
205
206        #make the model copy
207        cp -r collect model-collect
208
209    echo "done"
210
211    #for each collection, import, build and diff with its model counterpart
212    for collection in collect/*; do
213
214        #escape the filename (in case of space)
215        collection=`echo $collection | sed 's@ @\\\ @g'`
216
217        #get just the basename
218        collection=`basename $collection`
219
220        echo '<collection-test name="'$collection'">' >> $xmlout
221
222        #import
223        #Ensure the OIDtype for importing is hash_on_full_filename
224        # "to make document identifiers more stable across
225        # upgrades of the software, although it means that
226        # duplicate documents contained in the collection are no
227        # longer detected automatically."
228        echo $collection' - Importing:'
229        echo "<import>" >> $xmlout
230        import.pl -OIDtype hash_on_full_filename -removeold $collection
231        echo "</import>" >> $xmlout
232        echo "done"
233
234        #build
235        echo $collection' - Building:'
236        echo "<build>" >> $xmlout
237        buildcol.pl -removeold $collection
238        echo "</build>" >> $xmlout
239        echo "done"
240
241        #rename the intermediate 'building' directory 'index'
242        echo -n $collection' - Move "building" to "index"... "'
243        rm -rf collect/$collection/index
244        mv collect/$collection/building collect/$collection/index
245        echo "done"
246
247        #diffcol
248        echo $collection' - Diffing:'
249        cd $TASK_HOME/diffcol
250        ./diffcol.pl -output xml -verbosity 10 $collection >> $xmlout
251        cd $greenstone_home
252        echo "done"
253
254        echo "</collection-test>" >> $xmlout
255    done
256
257    echo "</test>" >> $xmlout
258
259    echo "done"
260
261}
262
263function upload() {
264    # ensure the upload directory exists
265    # before trying to transfer the xml and html files across
266    # if it already existed, clear it of contents
267    if [ ! -d "$UPLOAD_DIR" ]; then
268    mkdir -p "$UPLOAD_DIR";
269#    else
270#   rm $UPLOAD_DIR/*
271    fi
272
273    scp $DATA_DIR/*.xml $DATA_DIR/*.html $UPLOAD_DIR
274}
275
276function mail_simple_message() {
277    # email out brief failure message if failed                                                       
278    echo "Checking if successful... "
279    result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
280    echo "result: "$result
281    if [ "$result" != "yes" ]; then
282        echo $gsdl' regression test for '$dateid' failed' | mail -s 'Regression Test Failed' $MONITOR_EMAIL #$WEBMASTER_EMAIL
283    fi
284}
285
286
287function mail_with_report_attached() {
288    # http://stackoverflow.com/questions/17359/how-do-i-send-a-file-as-an-email-attachment-using-linux-command-line
289
290    #echo "This is the message body" | mutt -a "/path/to/file.to.attach" -s "subject of message" -- recipient@domain.com
291
292
293    # email out with report attached, if the tests failed                                                     
294    echo "Checking if successful... "
295    result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
296    echo "result: "$result
297    if [ "$result" != "yes" ]; then
298    echo $gsdl' regression test for '$dateid' failed' | mutt -a $DATA_DIR'/report-'$dateid'.html' -s 'Regression Test Failed' -- $MONITOR_EMAIL
299    echo "Sent mail with report attached."
300    fi
301}
302
303function summarise() {
304
305    # make a summarised Xml report
306    echo -n "Summarizing the xml report... "
307    java org.apache.xalan.xslt.Process -IN $xmlout -XSL $TASK_HOME/xsl/xml-report.xsl -OUT $DATA_DIR/report-$dateid.xml
308    echo "done"
309
310    # make a summarised HTml report
311    echo -n "Creating an html summary report... "
312    java org.apache.xalan.xslt.Process -IN $DATA_DIR/report-$dateid.xml -XSL $TASK_HOME/xsl/html-report.xsl -OUT $DATA_DIR/report-$dateid.html
313    echo "done"
314
315}
316
317
318#parse arguments
319if [ "$1" == "" ]; then
320    action=all
321else
322    if [ "$1" == "setup_greenstone" ]; then
323        action=setup_greenstone
324    elif [ "$1" == "run_test" ]; then
325        action=run_test
326    elif [ "$1" == "summarise" ]; then
327        action=summarise
328    elif [ "$1" == "upload" ]; then
329        action=upload
330    elif [ "$1" == "all" ]; then
331        action=all
332    else
333        echo "bad subcommand"
334        exit
335    fi
336fi
337
338#do the requested action
339if [ "$action" == "setup_greenstone" ]; then
340    setup_greenstone
341elif [ "$action" == "run_test" ]; then
342    run_test
343#   mail_simple_message
344elif [ "$action" == "summarise" ]; then
345    summarise
346elif [ "$action" == "upload" ]; then
347    upload
348#   mail_simple_message
349    mail_with_report_attached
350elif [ "$action" == "all" ]; then
351    setup_greenstone
352    run_test
353    summarise
354    upload
355    mail_with_report_attached
356fi
357
Note: See TracBrowser for help on using the browser.