Changeset 27621 for other-projects/nightly-tasks/diffcol/trunk/task.pl

Timestamp:

2013-06-13T20:55:14+12:00 (11 years ago)

Author:

ak19

Message:

Ported most of the task.sh functionality across to task.pl. The email and attachment functions still need to be added, as also the main function. And there's also some issue with collection subfolders not getting deleted during a build which results in unnecessary differences between the model and test collections.

File:

• other-projects/nightly-tasks/diffcol/trunk/task.pl (modified) (1 diff)

other-projects/nightly-tasks/diffcol/trunk/task.pl

@@ -1 @@
-#!/usr/bin/perl -w
-
-
-# Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
-# Doing so will call this pl file and pass in "summarise" in ARGV
-# This pl file will in turn call the task executable in this folder 
-# passing in "summarise" as a parameter.
-system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
-
-#system("/bin/bash -c \"./task @ARGV\"");
-#print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
+#!/usr/bin/perl -w  
+
+# This program is meant to run the nightly diffcol
+# It is meant to be an equivalent for the existing task bash script
+# But it is intended to be expanded to work for Windows and GS3
+# For windows, need to REMEMBER to set the correct shebangs at the top
+
+
+use Cwd; 
+use File::Path; # for rmdir and mkdir type functions
+use File::Copy; # for recursive copying of folders but skipping .svn
+use File::Basename;
+
+
+my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
+my $sep = $isWin ? "\\" : "/";
+my $pathsep = $isWin ? ";" : ":";
+#my $script_ext = $isWin ? ".bat" : ".bash";
+my $setup_script = "setup"; # needs to become gs3-setup for GS3
+
+# TASK_HOME should be the toplevel diffcol folder
+$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
+##    print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
+
+
+$ENV{'DATA_DIR'} = filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
+$ENV{'UPLOAD_DIR'} = filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
+$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
+##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
+
+# control if an existing compiled greenstone is used
+# or, if one should be checked out, which revision to checkout from svn
+$ENV{'SVN_OPT_REV'} = "-r head";
+#export GSDLHOME=
+#export GSDL3SRCHOME=
+
+#check key environment vars are set
+if(!defined $ENV{'UPLOAD_DIR'}) {
+    print STDOUT "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
+    #return 1;
+}
+if(!defined $ENV{'DATA_DIR'}) {
+    print STDOUT "Please set a DATA_DIR for the test in an environment.sh file\n";
+    #return 1;
+}
+if(!defined $ENV{'MONITOR_EMAIL'}) {
+    print STDOUT "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
+    #return 1;
+}
+
+if($ENV{'DATA_DIR'} eq "/") {
+    print STDOUT "DATA_DIR should not be the fs root\n";
+    #return 1;
+} 
+
+print STDOUT "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
+print STDOUT "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
+
+
+#create an id for this test
+my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
+$year += 1900;
+$mon += 1;
+$mon = "0$mon" if ($mon < 10);
+$mday = "0$mday" if ($mday < 10);
+my $dateid="$year-$mon-$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
+
+print STDOUT "Starting test '$dateid'\n";
+
+
+# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
+$ENV{'CLASSPATH'} = "";
+my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
+##print STDERR "@@@ $jar_lib_path\n";
+my @files = <$jar_lib_path/*.jar>; # /full/path/to/diffcol/lib/*jar
+foreach $file (@files) {
+    $ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
+}
+##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
+
+
+#set the location of the full report                                               
+my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$dateid.xml");
+##print STDERR "XML: $xmlout\n";
+
+# the toplevel folder of the greenstone installation being used
+my $greenstone_home="";
+# gsdl is the checkout folder and can be greenstone2 or greenstone3
+my $gsdl="greenstone2";
+
+
+# Check if using existing compiled-up greenstone installation
+# and set the greenstone_home location accordingly
+
+if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
+    print STDOUT "Found existing Greenstone home, will use that instead\n";
+    $greenstone_home=$ENV{'GSDLHOME'};
+} else {
+    $greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
+}
+##print STDERR "GSHOME: $greenstone_home\n";
+
+#&setup_greenstone;
+&run_test;
+&summarise;
+&upload;
+
+##********************************
+
+
+
+#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
+
+sub setup_greenstone
+{
+    #clean up from previous tests
+    print STDOUT "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
+    for my $i ( 1..5 ) {
+    sleep 1; # 1 second
+    print STDOUT ".";
+    }
+    print STDOUT "\n";
+
+    # http://perldoc.perl.org/File/Path.html
+    print STDOUT "cleaning up previous tests\n";
+    &File::Path::remove_tree($ENV{'DATA_DIR'});
+
+    print STDOUT "creating the data dir\n";
+    &File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
+
+    chdir($ENV{'DATA_DIR'});
+
+    # use existing compiled-up greenstone installation, if a GSDLHOME set
+    if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
+    print STDOUT "Found existing Greenstone home, will use that instead";
+    return;
+    }
+
+    # Else checkout a GS from svn into DATA_DIR
+
+    #svn checkout of main gsdl directory                                       
+    print STDOUT "checkout $gsdl:\n";
+    my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone2 $gsdl";
+    ##print STDERR "Checkout CMD: $cmd\n";
+
+    # # unlike backticks operator, system() will print the output of the command to the screen as it executes
+    # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
+    my $status = system "$cmd"; #my $status = `$cmd`;
+    print STDOUT "done\n";
+
+    ##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
+
+    chdir("$ENV{'DATA_DIR'}$sep$gsdl");
+
+    ##print STDERR "@@@ OS:  $^O.|".$Config{'archname64'}."|\n";
+
+    # if we're on linux/darwin, need gnome-lib for the correct architecture.
+    if(!$isWin) {
+
+    print STDOUT "setting up gnome-lib-minimal for compilation\n";
+
+    # To get gnome-lib, need to determine bit architecture of the linux/darwin
+    # http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
+    # $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
+    # well on ActivePerl. 
+    # But since we know we're on a Linux/Darwin machine at this point, wecan just run `uname -m` and other linux cmds
+
+    my $gnome_lib_file = ($^O =~ m/macos/i) ? "darwin-intel" : "linux"; # assuming all darwin is intel, not ppc!!
+
+    my $bit_arch=`uname -m`;
+    $gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/);
+
+    #svn checkout gnome-lib for this linux/darwin
+    chdir("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
+
+    ##print STDERR "**** gnomelib: $gnome_lib_file\n";
+
+    # checkout and unpack gnome-lib-minimal
+
+    #svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
+    $cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
+    system $cmd;
+    system ("tar -xvzf gl.tar.gz");
+
+    chdir("gnome-lib-minimal");
+    ##print STDERR "*** ARCH: $bit_arch\n";
+
+    # need to run source devel.bash on gnome-lib followed by configure, make, make install 
+    # in one go, in order to preserve the compile environment set up by sourcing devel.bash
+
+    # http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
+    # http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash 
+    
+#   $cmd = "/bin/bash \"source devel.bash && cd ../.. && ./configure --enable-apache httpd && make && make install\"";
+#   $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
+
+    $cmd = "bash -c \"";    
+
+    $cmd .= "source ./devel.bash";
+    $cmd .= " && cd ../..";
+
+    #configure
+    # $cmd .= " && ./configure";
+    $cmd .= " && echo 'configure $gsdl: ' ";
+        $cmd .= " && echo '<configure>' >> $xmlout";    
+        $cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
+        $cmd .= " && echo '</configure>' >> $xmlout";
+        $cmd .= " && echo 'done'";
+
+    #make
+    $cmd .= " && echo 'make $gsdl: '";
+    $cmd .= " && echo '<make>' >> $xmlout";
+        $cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
+        $cmd .= " && echo '</make>' >> $xmlout";
+        $cmd .= " && echo 'done'";
+
+    #make install
+    $cmd .= " && echo 'make install $gsdl: '";
+        $cmd .= " && echo '<make-install>' >> $xmlout";
+    $cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
+        $cmd .= " && echo '</make-install>' >> $xmlout";
+        $cmd .= " && echo 'done'";
+    
+    $cmd .= "\""; # close off cmd to bash and run it
+    system $cmd;    
+    }
+
+    # set the path to the greenstone_home variable                             
+    $greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
+
+}
+
+# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
+sub run_test
+{
+    open (my $xml_fh, '>'.$xmlout) || die "Could not open xml file $xmlout for appending: $!\n";   
+
+    # perform the requested subcommands, outputting xml information
+    print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
+    
+    # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home                                                                            
+    print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
+    my $cmd = "svn up $ENV{'TASK_HOME'}/model-collect"; #chdir("$ENV{'TASK_HOME'}/model-collect");
+    my $status = system "$cmd";
+
+    # go to whichever greenstone_home we're using
+    chdir($greenstone_home);
+
+    # get svn info
+    print STDOUT "getting svn info: $xmlout\n";
+    print $xml_fh "<svn-info>\n";
+    run_and_print_cmd("svn info", $xml_fh);
+    print $xml_fh "</svn-info>\n";
+    print STDOUT "done\n";
+
+    #make two copies of the model-collect directory in gsdl
+    #one to be rebuilt and one as the basis for comparison
+    #strip both of all .svn directories
+    
+    #copy the model collections to the collect folder to be rebuilt
+    print STDOUT "installing test collections and model collections to new $gsdl installation... ";
+    
+    #clean up
+    if(-d "collect") {
+    &File::Path::remove_tree("collect") || die "Error could not delete collect: $!";
+    }
+    &File::Path::remove_tree("model-collect");
+
+    #copy to collect and strip .svn subfolders
+    &File::Path::make_path("collect"); # create the folder and copy contents across
+    &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect"), "collect", ".svn");
+
+    #make the model copy
+    &File::Path::make_path("model-collect"); 
+    &copy_recursively("collect", "model-collect"); # copy contents across
+    
+    print STDOUT "done\n";    
+    
+
+    #for each collection, import, build and diff with its model counterpart
+    opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
+    for my $collection (readdir $collect_handle) {
+    next if ($collection eq "." || $collection eq "..");
+    next if ($collection ne "Small-HTML"); ## TEMPORARY, FOR TESTING THIS SCRIPT
+
+    #escape the filename (in case of space)
+    $collection =~ s@ @\\ @g;
+    #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
+
+    print STDERR "*** Found collection $collection\n";
+    print $xml_fh "<collection-test name=\"$collection\">\n";
+
+    #import
+# Ensure the OIDtype for importing is hash_on_full_filename
+# "to make document identifiers more stable across upgrades of the software, 
+# although it means that duplicate documents contained in the collection are
+# no longer detected automatically."
+    print STDOUT "$collection - Importing:\n";
+    print $xml_fh "<import>\n";
+    $cmd = "import.pl -OIDtype hash_on_full_filename $collection -removeold";
+    &run_build_script($cmd);
+    print $xml_fh "</import>\n";
+    print STDOUT "done\n";  
+
+    #build
+    print STDOUT "$collection - Building:\n";
+    print $xml_fh "<build>\n";
+    $cmd = "buildcol.pl $collection -removeold";
+    &run_build_script($cmd);
+    print $xml_fh "</build>\n";
+    print STDOUT "done\n";
+    
+    #rename the intermediate 'building' directory 'index'
+    print STDOUT "$collection - Move \"building\" to \"index\"... ";
+    my $index = &filename_concat("collect", $collection, "index");
+    my $building = &filename_concat("collect", $collection, "building");
+    &File::Path::remove_tree($index);
+    # Renaming Directories, http://www.perlmonks.org/?node_id=177421
+    move($building, $index) or die "copy failed: $!"; # File::Copy::move
+    print STDOUT "done\n";
+
+    #diffcol
+    print STDOUT "$collection - Diffing:\n";
+    my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
+    $cmd = "./diffcol.pl -output xml -verbosity 10 $collection";    
+    &run_diff_script($cmd, $xml_fh, $diffcol_dir);
+
+    chdir($greenstone_home); # this is actually where we are
+    print STDOUT "done\n";
+    print $xml_fh "</collection-test>\n";
+    }
+    closedir $collect_handle; # close handle to collect dir
+
+    print $xml_fh "</test>\n";
+    close($xml_fh);
+
+    print STDOUT "done\n";
+}
+
+##***************************************************************
+# runs setup in greenstone_home before running the diff command
+sub run_diff_script {    
+    my ($cmd, $fh, $diffcol_dir) = @_;
+
+    # we're in greenstone_home now
+    if(!$isWin) {   
+    $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && cd $diffcol_dir && $cmd\"";
+
+    } else { # Need to prefix cmd -c/-k as necessary
+    $cmd = "cd $greenstone_home && set GSDLHOME=&& source $setup_script.bat && cd $diffcol_dir && $cmd";
+    }
+
+    return &run_and_print_cmd($cmd, $fh);
+}
+
+# runs setup in greenstone_home before running the given build command
+sub run_build_script {    
+    my ($cmd, $fh) = @_;
+
+#    chdir($greenstone_home);
+
+    # we are in $greenstone_home, can directly run the build cmd on the collection
+    if(!$isWin) {
+    $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && $cmd\"";
+
+    } else { # Need to prefix cmd -c/-k as necessary
+    $cmd = "set GSDLHOME=&& source $setup_script.bat && $cmd";
+    }
+
+    return &run_and_print_cmd($cmd, $fh);
+}
+#ERROR (36096) running bash -c "export GSDLHOME=&& source setup.bash && buildcol.pl Small-HTML -removeold": Illegal seek
+
+
+# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
+# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
+sub run_and_print_cmd {
+    my ($cmd, $fh) = @_;
+    
+    open my $pin, "$cmd|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-|', 'powercfg -l') or die $!;
+
+    if(defined $fh) { # print cmd output both to the filehandle and to stdout
+    while (my $line = <$pin>) {
+        print $fh $line;
+#       print STDOUT $line; # if also printing cmd output to STDOUT
+    }
+    } 
+    else { # no filehandle, so just need to print to stdout
+
+    # unlike backticks operator, system() will print the output of the command to the screen as it executes
+    # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
+        
+    my $status = system $cmd;
+    if($status != 0) {
+        print STDERR "ERROR ($status) running $cmd: $!\n";
+    }
+    }
+    close($pin);
+}
+
+sub filename_concat {
+    my $first_file = shift(@_);
+    my (@filenames) = @_;
+
+    # If first_file is not null or empty, then add it back into the list
+    if (defined $first_file && $first_file =~ /\S/)
+    {
+    unshift(@filenames, $first_file);
+    }
+
+    my $filename = join($sep, @filenames);
+    $filename =~ s/[\\\/]$//; # remove trailing slashes if any
+    return $filename;
+}
+
+
+# The following code is from 
+# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
+# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
+sub copy_recursively {
+    my ($from_dir, $to_dir, $regex) = @_;
+    opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
+
+#    if(-d !$to_dir) {
+#   mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
+#    }
+
+    for my $entry (readdir $dh) {
+    next if ($entry eq "." || $entry eq "..");
+        next if (defined $regex && $entry =~ /$regex/);
+        my $source = "$from_dir/$entry";
+        my $destination = "$to_dir/$entry";
+        if (-d $source) {
+            mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
+            copy_recursively($source, $destination, $regex);
+        } else {
+            copy($source, $destination) or die "copy failed: $!";
+        }
+    }
+    closedir $dh;
+    return;
+}
+
+sub summarise {
+
+    # make a summarised Xml report
+    print STDOUT "Summarizing the xml report... ";
+    my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.xml";
+    my $status = system($cmd);
+    print STDOUT "done\n";
+
+    # make a summarised HTMl report
+    print STDOUT "Creating an html summary report... ";
+    $cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.html";
+    $status = system($cmd);
+    print STDOUT "done\n";
+}
+
+sub upload {
+    # ensure the upload directory exists before trying to transfer the xml and html files across
+    # if it already existed, clear it of contents
+    if (-d $ENV{'UPLOAD_DIR'}) {
+    &File::Path::make_path($ENV{'UPLOAD_DIR'});
+    } # else rm $UPLOAD_DIR/*    
+
+    # copy all *.xml and *.html files across to UPLOAD_DIR
+    opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
+    for my $entry (readdir $dh) {
+    next if ($entry !~ m/(\.xml|\.html)$/);
+    copy($entry, $ENV{'UPLOAD_DIR'});
+    }
+    closedir $dh;
+
+    print STDOUT "Finished uploading\n";
+}
+
+
+# TO DO:
+# Sending emails with perl: http://learn.perl.org/examples/email.html
+# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
+
+
+# The old version of this program contained the following, consisting of 1 line of active code:
+
+  # Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
+  # Doing so will call this pl file and pass in "summarise" in ARGV
+  # This pl file will in turn call the task executable in this folder 
+  # passing in "summarise" as a parameter.
+#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
+
+  ##system("/bin/bash -c \"./task @ARGV\"");
+  ##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"