source: other-projects/nightly-tasks/diffcol/trunk/task@ 27956

#!/bin/bash

if [ "x$TASK_HOME" == "x" ]; then
    TASK_HOME=`pwd`;
fi

export DATA_DIR=$TASK_HOME/diffcol-data
export UPLOAD_DIR=$TASK_HOME/diffcol-reports
export [email protected]

# control if an existing compiled greenstone is used 
# or, if one should be checked out, which revision to checkout from svn
SVN_OPT_REV="-r head"
#export GSDLHOME=
#export GSDL3SRCHOME=

#check key environment vars are set
if [ "$UPLOAD_DIR" == "" ]; then
    echo "Please set a UPLOAD_DIR for the test in an environment.sh file"
    exit
fi
if [ "$DATA_DIR" == "" ]; then
    echo "Please set a DATA_DIR for the test in an environment.sh file"
    exit
fi
if [ "$MONITOR_EMAIL" == "" ]; then
    echo "Please set a MONITOR_EMAIL for the test in an environment.sh file"
    exit
fi


if [ "$DATA_DIR" == "/" ]; then
    echo "DATA_DIR should not be the fs root"
    exit
fi

echo "DATA_DIR: $DATA_DIR"
echo "UPLOAD_DIR: $UPLOAD_DIR"

#create an id for this test
dateid=`date +'%Y-%m-%d'`

echo "Starting test '$dateid'"

#set classpath
CLASSPATH=
for file in $TASK_HOME/lib/*.jar; do
    CLASSPATH=$file:$CLASSPATH
done
export CLASSPATH

#set the location of the full report
xmlout=$DATA_DIR'/full-report-'$dateid'.xml'

# the toplevel folder of the greenstone installation being used
greenstone_home=
# gsdl is the checkout folder and can be greenstone2 or greenstone3
gsdl=greenstone2


# Check if using existing compiled-up greenstone installation
# and set the greenstone_home location accordingly
if [ "x$GSDL3SRCHOME" != "x" -o "x$GSDLHOME" != "x" ]; then
    echo "Found existing Greenstone home, will use that instead"
    greenstone_home=$GSDLHOME
else
    greenstone_home="$DATA_DIR/$gsdl";
fi



function setup_greenstone() {

    #clean up from previous tests
    echo -n "about to clean up any old tests (Ctrl-C to cancel)"
    for i in x x x x x; do sleep 1; echo -n .; done
    echo

    echo "cleaning up previous tests"
    rm -rf $DATA_DIR

    echo "creating the data dir"
    mkdir -p $DATA_DIR || exit

    cd $DATA_DIR


        # use existing compiled-up greenstone installation 
    if [ "x$GSDL3SRCHOME" != "x" -o "x$GSDLHOME" != "x" ]; then
        echo "Found existing Greenstone home, will use that instead"
        return;
    fi

    # Else checkout a GS from svn into DATA_DIR


    #svn checkout of main gsdl directory
    echo "checkout $gsdl:"
    svn co $SVN_OPT_REV http://svn.greenstone.org/main/trunk/greenstone2 $gsdl
    echo "done"

    cd $DATA_DIR/$gsdl


    # since this is a bash script, we're on linux/darwin, need gnome-lib
    # for the correct architecture. Assume all darwin is intel, not ppc
    echo "setting up gnome-lib-minimal for compilation"

# http://stackoverflow.com/questions/394230/detect-the-os-from-a-bash-script
    os='linux';
    gl_suffix=$os;
    if [[ "$OSTYPE" == "darwin"* ]]; then
        os='darwin';
        gl_suffix=$os'-intel';
    else
        # linux, work out the bit architecture
        arch=`uname -m`
        if [[ "$arch" == *"64"* ]]; then
        arch='64';
        gl_suffix=$gl_suffix'-x64';
        else
        arch='32';
        fi
    fi


    #svn checkout gnome-lib for this linux/darwin
    cd $DATA_DIR/$gsdl/ext

#   svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
    svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-$gl_suffix.tar.gz gl.tar.gz
    tar xvzf gl.tar.gz
        cd gnome-lib-minimal
        source devel.bash

        cd ../..


    #configure
    echo "configure $gsdl: "
    echo "<configure>" >> $xmlout
    ./configure 2>> $DATA_DIR/compilation-errors
    echo "</configure>" >> $xmlout
    echo "done"

    #make
    echo "make $gsdl: "
    echo "<make>" >> $xmlout
    make 2>> $DATA_DIR/compilation-errors
    echo "</make>" >> $xmlout
    echo "done"

    #make install
    echo "make install $gsdl: "
    echo "<make-install>" >> $xmlout
    make install 2>> $DATA_DIR/compilation-errors
    echo "</make-install>" >> $xmlout
    echo "done"

    # set the path to the greenstone_home variable
    greenstone_home=$DATA_DIR/$gsdl
}


function run_test() {

    #perform the requested subcommands, outputting xml information
    echo '<test time="'$dateid'" id="'$dateid'">' > $xmlout

    # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
    echo "Updating $TASK_HOME/model-collect:"
    #cd $TASK_HOME/model-collect
    svn up $TASK_HOME/model-collect

    # go to whichever greenstone_home we're using
    cd $greenstone_home

    #get svn info
    echo "getting svn info: "
    echo "<svn-info>" >> $xmlout
    svn info >> $xmlout
    echo "</svn-info>" >> $xmlout
    echo "done"

    #set up the gsdl environment with the setup.bash script
    echo "setting up $gsdl environment: "
        export GSDLHOME=
        source setup.bash
    echo "done"

    #make two copies of the model-collect directory in gsdl
    #one to be rebuild and one as the basis for comparison
    #strip both of all .svn directories

    #copy the model collections to the collect folder to be rebuilt
    echo -n "installing test collections and model collections to new $gsdl installation... "

        #clean up
        rm -rf collect
        rm -rf model-collect

        #copy to collect and strip
        cp -r $TASK_HOME/model-collect collect
        find collect -type d -name .svn -exec rm -rf {} \;

        #make the model copy
        cp -r collect model-collect

    echo "done"

    #for each collection, import, build and diff with its model counterpart
    for collection in collect/*; do

        #escape the filename (in case of space)
        collection=`echo $collection | sed 's@ @\\\ @g'`

        #get just the basename
        collection=`basename $collection`

        echo '<collection-test name="'$collection'">' >> $xmlout

        #import
        #Ensure the OIDtype for importing is hash_on_full_filename
        # "to make document identifiers more stable across 
        # upgrades of the software, although it means that 
        # duplicate documents contained in the collection are no
        # longer detected automatically."
        echo $collection' - Importing:'
        echo "<import>" >> $xmlout
        import.pl -OIDtype hash_on_full_filename -removeold $collection
        echo "</import>" >> $xmlout 
        echo "done"

        #build
        echo $collection' - Building:'
        echo "<build>" >> $xmlout
        buildcol.pl -removeold $collection
        echo "</build>" >> $xmlout
        echo "done"

        #rename the intermediate 'building' directory 'index'
        echo -n $collection' - Move "building" to "index"... "'
        rm -rf collect/$collection/index
        mv collect/$collection/building collect/$collection/index
        echo "done"

        #diffcol
        echo $collection' - Diffing:'
        cd $TASK_HOME/diffcol
        ./diffcol.pl -output xml -verbosity 10 $collection >> $xmlout
        cd $greenstone_home
        echo "done"

        echo "</collection-test>" >> $xmlout
    done

    echo "</test>" >> $xmlout

    echo "done"

}

function upload() {
    # ensure the upload directory exists 
    # before trying to transfer the xml and html files across
    # if it already existed, clear it of contents
    if [ ! -d "$UPLOAD_DIR" ]; then
    mkdir -p "$UPLOAD_DIR";
#    else
#   rm $UPLOAD_DIR/*
    fi

    scp $DATA_DIR/*.xml $DATA_DIR/*.html $UPLOAD_DIR
}

function mail_simple_message() {
    # email out brief failure message if failed                                                       
    echo "Checking if successful... "
    result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
    echo "result: "$result
    if [ "$result" != "yes" ]; then
        echo $gsdl' regression test for '$dateid' failed' | mail -s 'Regression Test Failed' $MONITOR_EMAIL #$WEBMASTER_EMAIL 
    fi
}


function mail_with_report_attached() {
    # http://stackoverflow.com/questions/17359/how-do-i-send-a-file-as-an-email-attachment-using-linux-command-line

    #echo "This is the message body" | mutt -a "/path/to/file.to.attach" -s "subject of message" -- [email protected]


    # email out with report attached, if the tests failed                                                      
    echo "Checking if successful... "
    result=`java org.apache.xalan.xslt.Process -IN "$xmlout" -XSL "$TASK_HOME/xsl/passed-or-not.xsl"`
    echo "result: "$result
    if [ "$result" != "yes" ]; then
    echo $gsdl' regression test for '$dateid' failed' | mutt -a $DATA_DIR'/report-'$dateid'.html' -s 'Regression Test Failed' -- $MONITOR_EMAIL
    echo "Sent mail with report attached."
    fi
}

function summarise() {

    # make a summarised Xml report
    echo -n "Summarizing the xml report... "
    java org.apache.xalan.xslt.Process -IN $xmlout -XSL $TASK_HOME/xsl/xml-report.xsl -OUT $DATA_DIR/report-$dateid.xml
    echo "done"

    # make a summarised HTml report
    echo -n "Creating an html summary report... "
    java org.apache.xalan.xslt.Process -IN $DATA_DIR/report-$dateid.xml -XSL $TASK_HOME/xsl/html-report.xsl -OUT $DATA_DIR/report-$dateid.html
    echo "done"

}


#parse arguments
if [ "$1" == "" ]; then
    action=all
else
    if [ "$1" == "setup_greenstone" ]; then
        action=setup_greenstone
    elif [ "$1" == "run_test" ]; then
        action=run_test
    elif [ "$1" == "summarise" ]; then
        action=summarise
    elif [ "$1" == "upload" ]; then
        action=upload
    elif [ "$1" == "all" ]; then
        action=all
    else
        echo "bad subcommand"
        exit
    fi
fi

#do the requested action
if [ "$action" == "setup_greenstone" ]; then
    setup_greenstone
elif [ "$action" == "run_test" ]; then
    run_test
#   mail_simple_message
elif [ "$action" == "summarise" ]; then
    summarise
elif [ "$action" == "upload" ]; then
    upload
#   mail_simple_message
    mail_with_report_attached
elif [ "$action" == "all" ]; then
    setup_greenstone
    run_test
    summarise
    upload
    mail_with_report_attached
fi