Context Navigation

source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 28661

Last change on this file since 28661 was 28661, checked in by ak19, 10 years ago
Committing the next installment of code to handle diffcol for GS3. Now it successfully compiles up GS3, while diffcol still works for GS2.
Property svn:executable set to ``*
File size: 41.8 KB

Rev	Line
[27621]	1	#!/usr/bin/perl -w
[27556]	2
[27621]	3	# This program is meant to run the nightly diffcol
	4	# It is meant to be an equivalent for the existing task bash script
	5	# But it is intended to be expanded to work for Windows and GS3
	6	# For windows, need to REMEMBER to set the correct shebangs at the top
[27556]	7
[27663]	8
	9	# TODO:
	10	# Have a caveat mode and a stable mode (as in snapshot/task.pl)
	11	#
	12	#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat\|stable)" ) {
	13	# $major_version = 2;
	14	# $prefix="2t";
	15	# $rk="tk2"; # test kit
	16	#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat\|stable)" ) {
	17	# $major_version = 3;
	18	# $prefix="3t";
	19	# $rk="tk3"; # test kit
	20
[27628]	21	package diffcoltask;
[27556]	22
[27621]	23	use Cwd;
[27628]	24	use Switch; # for switch(val) { case: ; ...}
[27621]	25	use File::Path; # for rmdir and mkdir type functions
	26	use File::Copy; # for recursive copying of folders but skipping .svn
	27	use File::Basename;
	28
[27628]	29	use strict;
	30	no strict 'subs'; # allow barewords (eg STDERR) as function arguments
[27621]	31
[27764]	32
[27621]	33	my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
[27764]	34	my $isMac = ($^O =~ m/macos\|darwin/i) ? 1 : 0;
[28571]	35
	36	my $osversion="";
	37	# Need to get the correct gnome-lib-minimal for the OS
	38	# darwin11* Lion, darwin12* Mountain Lion, darwin9* and darwin10* are Leopard and Snow Leopard
	39	if ($^O eq "darwin") {
	40	$osversion=`uname -r`; # e.g. 12.x.x
	41	#$osversion =~ s@\..*$@@; # e.g.12
[28634]	42	$osversion = ($osversion =~ m@^1[1-9](\.)?@i) ? "Lion-" : "";
[28571]	43	}
	44
	45
[27621]	46	my $sep = $isWin ? "\\" : "/";
	47	my $pathsep = $isWin ? ";" : ":";
	48	#my $script_ext = $isWin ? ".bat" : ".bash";
[28661]	49	my $setup_script = "setup"; # will become gs3-setup for GS3
[27687]	50	my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
[27621]	51
[28616]	52	my $install_type = "svn";
	53	my $install_version = "2";
	54
[28106]	55	my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
[28072]	56	my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
[27701]	57
[28072]	58	# if use_local_rebuild is on, use_static_model should be on
	59	if ($use_local_rebuild && !$use_static_model) {
	60	$use_static_model = 1;
	61	}
	62
[28172]	63	my $test_os = $isWin ? "windows" : ($isMac ? "darwin" : "linux");
	64	my $model_os = "linux"; # default
[28072]	65
[27621]	66	# TASK_HOME should be the toplevel diffcol folder
	67	$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
[27701]	68	if($isWin) {
	69	$ENV{'TASK_HOME'} =~ s@\/@\\@g;
	70	# need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
	71	# run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
	72	# At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
	73	# end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
	74	require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
	75	$ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
	76	}
[27621]	77	## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
	78
	79
[27678]	80	$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
	81
	82	# we'll be using BLAT to send mail attachments on Windows
[27687]	83	my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
	84	if($isWin && $use_blat && ! -e $blat) {
	85	print STDERR "\n***********************************\n";
	86	print STDERR "No blat.exe found in $blat.\n";
	87	print STDERR "Blat needed to send mail with attachments on Windows.\n";
	88	print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
	89	print STDERR "for your bit architecture and name the folder 'blat'\n";
	90	print STDERR "***********************************\n\n";
[27678]	91	$blat = 0;
	92	}
	93
	94
[27668]	95	$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
	96	$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
[27621]	97	$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
[27678]	98	$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
[27621]	99	##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
	100
	101	# control if an existing compiled greenstone is used
	102	# or, if one should be checked out, which revision to checkout from svn
	103	$ENV{'SVN_OPT_REV'} = "-r head";
[27656]	104	#$ENV{'GSDLHOME'}=
	105	#$ENV{'GSDL3SRCHOME'}=
[27621]	106
[27628]	107
[27677]	108	# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
	109	my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
[27628]	110
	111	#parse arguments
	112	my $action = "all";
[28114]	113	my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
[28620]	114	my @collections = (); # list of collections that run_test should process
[28114]	115
[28620]	116	if(scalar(@ARGV) == 0) {
	117	$action="all";
	118	}
	119
	120	# process any arguments that are --gs2\|--gs3 and --bin\|--svn, and delete them from the array
	121	# if none provided, it's gs2 and svn by default.
	122	for (my $i = $#ARGV; $i >= 0; --$i) {
	123	if($ARGV[$i] =~ m/--(bin\|svn)/) {
	124	$install_type = $1;
	125	splice @ARGV, $i, 1; # remove the element from the argument array
	126	} elsif($ARGV[$i] =~ m/--gs(2\|3)/) {
	127	$install_version = $1;
[28661]	128	$setup_script = $install_version eq "3" ? "gs3-setup" : "setup"; # needs to become gs3-setup for GS3
[28620]	129	splice @ARGV, $i, 1; # remove the element from the argument array
	130	}
	131	}
	132
	133
[28172]	134	# run_test can take any number of args
[28620]	135	if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
[28172]	136	print STDERR "**** Wrong number of arguments\n";
[28114]	137	&printusage();
	138	exit -1;
[27628]	139	}
	140
[28620]	141	if(scalar(@ARGV) > 0) {
[27628]	142	switch ($ARGV[0]) {
[28114]	143	case qr/^(-h\|--?help\|help)$/i { &printusage; exit 0; }
[27628]	144	case qr/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/ { $action=$ARGV[0]; }
	145	else {
[28172]	146	print STDERR "**** Bad subcommand.\n";
[27628]	147	&printusage;
	148	exit -1;
	149	}
	150	}
[28616]	151
[28172]	152	# run_test action can take a subaction: nosvn\|justdiff. It can also take --modelOS (windows\|linux\|darwin)
[28114]	153	# nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
	154	# it out from the svn model-collect again.
	155	# justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
[28616]	156
[28620]	157	if($action eq "run_test" && scalar(@ARGV) >= 2) {
[28114]	158	push(@collections, @ARGV);
	159	shift @collections; # remove action from array
	160
[28172]	161	for (my $i=0; $i < scalar(@ARGV); $i++) {
	162	if($ARGV[$i] =~ m@^--@) {
	163	shift @collections; # remove subaction/flag from array
[28114]	164
[28172]	165	$subaction = $ARGV[$i];
[28114]	166	if($subaction eq "--justdiff") {
[28172]	167	$use_local_rebuild = $use_static_model = 1;
[28114]	168	} elsif ($subaction eq "--nosvn") {
[28172]	169	$use_static_model = 1;
	170	#} elsif ($subaction =~ m/\-\-testOS/i && defined $ARGV[$i+1]) {
	171	# $test_os = $ARGV[$i+1];
	172	# $i++;
	173	# shift @collections; # remove test_os value from array
	174	} elsif ($subaction =~ m/\-\-modelOS/i && defined $ARGV[$i+1] && $ARGV[$i+1] =~ m/windows\|linux\|darwin/i) {
	175	$model_os = $ARGV[$i+1];
	176	$i++;
	177	shift @collections; # remove model_os value from array
	178	#print STDERR "Model_os specified: $model_os\n";
[28114]	179	} else {
[28172]	180	print STDERR "**** Bad subaction/value: ".$ARGV[$i]."\n";
	181	&printusage;
	182	exit -1;
	183	}
	184	}
[28114]	185	}
	186
	187	# foreach my $col (@collections) {
	188	# print STDERR "Collection: $col\n";
	189	# }
	190	}
[27628]	191	}
	192
[28661]	193	print STDERR "Install type $install_type\n";
	194	print STDERR "Install version $install_version\n";
[28114]	195
[27621]	196	#check key environment vars are set
	197	if(!defined $ENV{'UPLOAD_DIR'}) {
[27629]	198	print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
[27621]	199	#return 1;
	200	}
	201	if(!defined $ENV{'DATA_DIR'}) {
[27629]	202	print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
[27621]	203	#return 1;
	204	}
	205	if(!defined $ENV{'MONITOR_EMAIL'}) {
[27629]	206	print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
[27621]	207	#return 1;
	208	}
	209
	210	if($ENV{'DATA_DIR'} eq "/") {
[27629]	211	print STDERR "DATA_DIR should not be the fs root\n";
[27621]	212	#return 1;
	213	}
	214
[27629]	215	print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
	216	print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
[27621]	217
	218	#create an id for this test
	219	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	220	$year += 1900;
	221	$mon += 1;
	222	$mon = "0$mon" if ($mon < 10);
	223	$mday = "0$mday" if ($mday < 10);
[27702]	224	my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
[27621]	225
[27629]	226	print STDERR "Starting test '$dateid'\n";
[27621]	227
	228
	229	# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
	230	$ENV{'CLASSPATH'} = "";
	231	my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
	232	my @files = <$jar_lib_path/.jar>; # /full/path/to/diffcol/lib/jar
[27628]	233	foreach my $file (@files) {
[27668]	234	$file =~ s@\/@\\@g if $isWin;
[27621]	235	$ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
	236	}
	237	##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
	238
	239
	240	#set the location of the full report
[28661]	241	my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$install_version-$dateid.xml");
[27621]	242	##print STDERR "XML: $xmlout\n";
	243
	244	# the toplevel folder of the greenstone installation being used
[28661]	245	my $greenstone3_home="";
	246	my $greenstone_home=""; # gs2build for gs3, toplevel install folder for gs2
[27621]	247	# gsdl is the checkout folder and can be greenstone2 or greenstone3
	248	my $gsdl="greenstone2";
	249
	250	# Check if using existing compiled-up greenstone installation
	251	# and set the greenstone_home location accordingly
	252
	253	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27629]	254	print STDERR "Found existing Greenstone home, will use that instead\n";
[28661]	255	$greenstone_home=$ENV{'GSDLHOME'} if defined $ENV{'GSDLHOME'};
	256
	257	if(defined $ENV{'GSDL3SRCHOME'}) {
	258	print STDERR "*** GSDL3SRCHOME set, changing to using installed GS3\n";
	259	$gsdl = "greenstone3";
	260	$install_version = "3";
	261	$greenstone3_home=$ENV{'GSDL3SRCHOME'};
	262	$greenstone_home=filename_concat($greenstone3_home, "gs2build") unless defined $ENV{'GSDLHOME'};
	263	}
[27621]	264	} else {
[28661]	265	if($install_version eq "3") {
	266	$gsdl = "greenstone3";
	267	$greenstone3_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	268	$greenstone_home=filename_concat($greenstone3_home, "gs2build");
	269	} else {
	270	$greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	271	}
[27621]	272	}
	273	##print STDERR "GSHOME: $greenstone_home\n";
	274
[27628]	275	#do the requested action
	276	if($action eq "setup_greenstone") {
	277	&setup_greenstone;
	278	}
	279	elsif ($action eq "run_test") {
	280	&run_test;
	281	}
	282	elsif ($action eq "summarise") {
	283	&summarise;
	284	}
	285	elsif ($action eq "upload") {
	286	&upload;
	287	&mail_with_report_attached;
	288	}
	289	elsif ($action eq "all") {
	290	&setup_greenstone;
	291	&run_test;
	292	&summarise;
	293	&upload;
	294	&mail_with_report_attached;
	295	}
[27621]	296
	297	##********************************
	298
[27628]	299	sub printusage
	300	{
[28172]	301	# print STDERR "Run as: $0 (help\|setup_greenstone\|run_test <--modelOS windows\|darwin\|linux> <--justdiff\|--nosvn> <col1 col2 ...> \|summarise\|upload\|all)\n";
	302	print STDERR "Run as: $0 (help\|setup_greenstone\|run_test\|summarise\|upload\|all)\n";
	303	print STDERR "where run_test can further take the following optional parameters:\n";
	304	print STDERR "\t--modelOS (windows\|darwin\|linux)\n";
	305	print STDERR "\t--justdiff\|--nosvn\n";
	306	print STDERR "\t<col1 col2 ...>\n";
[28616]	307	print STDERR "where setup_greenstone can further take the following optional parameters:\n";
	308	print STDERR "\t--gs2\|--gs3\n";
	309	print STDERR "\t--svn\|--bin\n";
[27628]	310	}
[27621]	311
	312	#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
	313
	314	sub setup_greenstone
	315	{
[28661]	316	if($install_version eq "3" && !defined $ENV{'JAVA_HOME'}) {
	317	print "*** JAVA_HOME not set. Set it and add its bin folder to the PATH\n";
	318	exit 0;
	319	}
	320
[27621]	321	#clean up from previous tests
[27628]	322	print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
[27621]	323	for my $i ( 1..5 ) {
	324	sleep 1; # 1 second
[27628]	325	print STDERR ".";
[27621]	326	}
[27628]	327	print STDERR "\n";
[27621]	328
	329	# http://perldoc.perl.org/File/Path.html
[27629]	330	print STDERR "cleaning up previous tests\n";
[28661]	331	#&File::Path::remove_tree($ENV{'DATA_DIR'});
	332	if(-d $greenstone_home && $install_version eq "2") {
	333	&File::Path::remove_tree($greenstone_home);
	334	}
	335	if(-d $greenstone3_home && $install_version eq "3") {
	336	&File::Path::remove_tree($greenstone3_home);
	337	}
	338	unlink glob "$ENV{'DATA_DIR'}$sep"."report-$install_version";
	339	unlink "$ENV{'DATA_DIR'}$sep"."compilation-errors";
[27629]	340	print STDERR "creating the data dir\n";
[28661]	341	#&File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
[27621]	342
	343	chdir($ENV{'DATA_DIR'});
	344
	345	# use existing compiled-up greenstone installation, if a GSDLHOME set
	346	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27710]	347	print STDERR "Found existing Greenstone home, will use that instead\n";
[27621]	348	return;
	349	}
	350
	351	# Else checkout a GS from svn into DATA_DIR
[28661]	352	if($install_type eq "svn") {
	353	&checkout_gs_from_svn();
	354	##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
[27621]	355
[28661]	356	if($install_version eq "2") {
	357	&compile_gs2_svn();
	358	} else {
	359	&compile_gs3_svn();
	360	}
	361	}
	362
	363	}
	364
	365	sub checkout_gs_from_svn {
	366
[27621]	367	#svn checkout of main gsdl directory
[27629]	368	print STDERR "checkout $gsdl:\n";
[28661]	369	my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone$install_version $gsdl";
[27621]	370	##print STDERR "Checkout CMD: $cmd\n";
	371
	372	# # unlike backticks operator, system() will print the output of the command to the screen as it executes
	373	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	374	my $status = system "$cmd"; #my $status = `$cmd`;
[28661]	375	if($status != 0) {
	376	print STDERR "@@@ SVN checkout of $gsdl failed\n";
	377	exit -1;
	378	}
[27629]	379	print STDERR "done\n";
[28661]	380	}
[27621]	381
[28661]	382	sub getImageMagickBins {
	383	my ($cmd, $status);
	384	my $imagickzip = "";
[27621]	385
[28661]	386	chdir ("$greenstone_home");
	387	if(!$isWin) { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
[27710]	388
	389	my $bit_arch=`uname -m`;
[28661]	390
	391	# imagemagick binary
[27710]	392	print STDERR "Getting imagemagick binary\n";
[28661]	393
[27724]	394	my $os = $isMac ? "darwin" : "linux";
[28661]	395	$imagickzip = "imagemagick-$os";
[27668]	396
[27710]	397	if($isMac) {
[28661]	398	# $imagickzip .= "-10.5.tar.gz";
[27962]	399	# at present, only the Imagemagick binaries created by Max for darwin work on the Macs
[28661]	400	# &File::Path::make_path("$greenstone_home$sep"."bin"."$sep$os"); # need to ensure gsdl/bin/darwin exists
[27962]	401	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
	402	$status = system($cmd);
[28661]	403	if($status != 0) {
	404	print STDERR "@@@ Unable to get imagemagick for darwin\n";
	405	}
	406
	407	# need ghostscript mac binary too for pdf to img conversions on mac
[27963]	408	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
	409	$status = system($cmd);
[28661]	410	if($status != 0) {
	411	print STDERR "@@@ Unable to get ghostscript for darwin\n";
	412	}
	413
	414	# the imagemagick and ghostscript binaries have been set to executable on svn trac now
	415	# system("chmod -R u+x $greenstone_home/bin/darwin/imagemagick/bin/*");
	416	# system("chmod -R u+x $greenstone_home/bin/darwin/ghostscript/bin/*");
[27710]	417	} else { # linux
	418	my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
[27724]	419	$imagickzip .= "$extension64.tar.gz";
[27962]	420
	421	# now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
	422	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
	423	$status = system ($cmd);
	424	system("cd ext && tar -xvzf $imagickzip");
[27724]	425	}
[28661]	426	}
	427	return $imagickzip;
	428	}
[27710]	429
[28661]	430	sub getGnomeLibExt
	431	{
	432	my ($cmd, $status);
	433	my $bit_arch=`uname -m`;
	434	my $os = $isMac ? "darwin" : "linux";
	435
[27710]	436	# gnomelib binary
[27629]	437	print STDERR "setting up gnome-lib-minimal for compilation\n";
[27621]	438
	439	# To get gnome-lib, need to determine bit architecture of the linux/darwin
	440	# http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
	441	# $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
	442	# well on ActivePerl.
[28621]	443	# But since we know we're on a Linux/Darwin machine at this point, we can just run `uname -m` and other linux cmds
[27621]	444
[28571]	445	# osversion will be "Lion" or ""
	446	# and assuming all darwin is intel, not ppc!!
[28572]	447	my $gnome_lib_file = $isMac ? "darwin-".$osversion."intel" : "linux";
[27621]	448
[28571]	449	$gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/ && !$isMac); # linux only case
[27621]	450
	451	#svn checkout gnome-lib for this linux/darwin
[28661]	452	chdir("$greenstone_home$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
[27621]	453
	454	##print STDERR "**** gnomelib: $gnome_lib_file\n";
	455
	456	# checkout and unpack gnome-lib-minimal
	457
	458	#svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
	459	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
	460	system $cmd;
	461	system ("tar -xvzf gl.tar.gz");
	462
	463	##print STDERR "*** ARCH: $bit_arch\n";
[28661]	464	}
[27621]	465
[28661]	466
	467
	468	sub compile_gs3_svn() {
	469	my ($cmd, $status);
	470	chdir ("$greenstone3_home");
	471
	472	$cmd = "ant"; # creates the build.properties file from the .in template
	473	$status = system $cmd;
	474	if($status != 0) {
	475	print STDERR "Unable to run the ant command: $status\n";
	476	exit -1;
	477	}
	478
	479	$cmd = "ant -Dproperties.ok=y -Dcheckout.gnomelib.ext=true prepare"; # pass in confirmation to ant prepare step
	480	$status = system $cmd;
	481	if($status != 0) {
	482	print STDERR "Failed to run $cmd command: $status\n";
	483	exit -1;
	484	}
	485
	486	my $imagickzip = &getImageMagickBins();
	487	chdir ("$greenstone3_home");
	488
	489	$cmd = "ant -Dcheckout.gnomelib.ext=true install"; # Compile with gnome-lib.
	490	$status = system $cmd;
	491	if($status != 0) {
	492	print STDERR "Failed to run $cmd command: $status\n";
	493	exit -1;
	494	}
	495	}
	496
	497	sub compile_gs2_svn() {
	498	my $imagickzip = &getImageMagickBins();
	499	&getGnomeLibExt();
	500
	501	my ($cmd, $status);
	502	my $os = $isMac ? "darwin" : "linux";
	503
	504	chdir("$greenstone_home"); #chdir("$ENV{'DATA_DIR'}$sep$gsdl"); # goes into toplevel gs2 or gs3 folder
	505
	506	##print STDERR "@@@ OS: $^O.\|".$Config{'archname64'}."\|\n";
	507
	508	if($isWin) {
	509	print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
	510
	511	# we're now in the GS2 folder, call makegs2 with silent param
	512	$cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
	513	$status = system $cmd;
	514	if($status != 0) {
	515	print STDERR "Greenstone compilation on Windows failed\n";
	516	exit -1;
	517	}
	518
	519	} else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
	520
	521	chdir("ext$sep"."gnome-lib-minimal");
	522
[27621]	523	# need to run source devel.bash on gnome-lib followed by configure, make, make install
	524	# in one go, in order to preserve the compile environment set up by sourcing devel.bash
	525
	526	# http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
	527	# http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
	528
	529	# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
	530	$cmd = "bash -c \"";
	531
	532	$cmd .= "source ./devel.bash";
	533	$cmd .= " && cd ../..";
	534
	535	#configure
	536	# $cmd .= " && ./configure";
	537	$cmd .= " && echo 'configure $gsdl: ' ";
	538	$cmd .= " && echo '<configure>' >> $xmlout";
	539	$cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
	540	$cmd .= " && echo '</configure>' >> $xmlout";
	541	$cmd .= " && echo 'done'";
	542
	543	#make
	544	$cmd .= " && echo 'make $gsdl: '";
	545	$cmd .= " && echo '<make>' >> $xmlout";
	546	$cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
	547	$cmd .= " && echo '</make>' >> $xmlout";
	548	$cmd .= " && echo 'done'";
	549
	550	#make install
	551	$cmd .= " && echo 'make install $gsdl: '";
	552	$cmd .= " && echo '<make-install>' >> $xmlout";
	553	$cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
	554	$cmd .= " && echo '</make-install>' >> $xmlout";
	555	$cmd .= " && echo 'done'";
	556
	557	$cmd .= "\""; # close off cmd to bash and run it
[27668]	558	$status = system $cmd;
[27724]	559
[27962]	560	if(!$isMac) { # Linux
[28661]	561	# Moving imagemagick after instead of before compilation, since bin/darwin and bin/linux gets overwritten during compilation
	562	move("$greenstone_home$sep"."ext/imagemagick/$os", "$greenstone_home$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
	563	unlink "$greenstone_home$sep"."ext$sep$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
	564	&File::Path::remove_tree("$greenstone_home$sep"."ext$sep"."imagemagick"); # the untarred parent folder
[27962]	565	}
[27724]	566
[27621]	567	}
[27668]	568
[28661]	569	if($status != 0) {
	570	print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
	571	exit -1;
	572	}
	573
	574	&getIsisGdl("$greenstone_home"); #&getIsisGdl("$ENV{'DATA_DIR'}/$gsdl");
[27764]	575
[27621]	576	# set the path to the greenstone_home variable
[28661]	577	#$greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
[27764]	578	}
[27621]	579
[27764]	580	sub getPDFBox
	581	{
	582	# current revision is 27763, but using "head" works
	583	my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
	584	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
	585
	586	# now get the PDFBox extension for PDFBox tutorial
	587	print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
	588
	589	chdir($greenstone_home);
	590	my $cmd = "";
	591	if ($isWin) {
	592	$cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
	593
	594	} elsif ($isMac) { # need to use curl not wget
	595	$cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
	596	}
	597	else { # linux
[27765]	598	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
[27764]	599	}
	600	my $status = system $cmd;
	601	if($status != 0) {
	602	print STDERR "@@@ Failed to set up PDFBox\n";
[27765]	603	exit -1; # or proceed to testing other tutorials?
[27764]	604	}
[27621]	605	}
	606
[28615]	607	sub getOpenOfficeExt
	608	{
	609	# current revision is 27763, but using "head" works
	610	my $OOEXT_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/open-office/trunk/open-office-java"; # both for .zip and .tar.gz extension
	611	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/open-office/trunk/open-office-java";
	612
	613	# now get the OpenOffice extension for the AllDocTypes collection
	614	print STDERR "Getting open office ext from $OOEXT_TRAC_URL:\n";
	615
	616	chdir($greenstone_home);
	617	my $cmd = "";
	618	if ($isWin) {
	619	$cmd = "setup.bat && cd ext && wget $OOEXT_TRAC_URL.zip && unzip open-office-java.zip";
	620
	621	} elsif ($isMac) { # need to use curl not wget
	622	$cmd = "cd ext && curl $OOEXT_TRAC_URL.tar.gz > open-office-java.tar.gz && tar -xzf open-office-java.tar.gz";
	623	}
	624	else { # linux
	625	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $OOEXT_TRAC_URL.tar.gz && tar -xzf open-office-java.tar.gz\"";
	626	}
	627	my $status = system $cmd;
	628	if($status != 0) {
	629	print STDERR "@@@ Failed to set up the Open Office Extension\n";
	630	exit -1; # or proceed to testing other tutorials?
	631	}
	632	}
	633
[28605]	634	sub getIsisGdl {
[28606]	635	my $gsfolder = shift(@_);
[28605]	636
[28606]	637	if(!$isWin) {
[28661]	638	chdir($greenstone_home);
[28605]	639	my $bit_arch=`uname -m`;
	640	if ($bit_arch =~ m/64$/) {
[28606]	641	my $cmd = "";
	642	if($isMac) {
	643	$cmd = "cd $gsfolder/bin/darwin && curl http://www.greenstone.org/caveat-emptor/IsisGdl.macleopard > IsisGdl && chmod u+x IsisGdl";
	644	} else { # linux
	645	$cmd = "cd $gsfolder/bin/linux && wget http://www.greenstone.org/caveat-emptor/IsisGdl.bin32 && mv IsisGdl.bin32 IsisGdl && chmod u+x IsisGdl";
	646	}
[28605]	647	my $isis_status = system $cmd;
	648	if($isis_status != 0) {
[28606]	649	print STDERR "Unable to get IsisGdl from caveat page\n";
[28605]	650	}
	651	}
	652	}
	653	}
	654
[27621]	655	# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
	656	sub run_test
[28661]	657	{
	658	my $collect_parent = $greenstone_home;
	659	my $model_collect = "model-collect";
	660	my $build_options = "";
	661	if($install_version eq "3") {
	662	$collect_parent = &filename_concat($greenstone3_home,"web","sites","localsite");
	663	$model_collect = "gs3-model-collect";
	664	$build_options = " -site localsite ";
	665	chdir($greenstone3_home);
	666	} else {
	667	chdir($greenstone_home);
	668	}
	669
[28114]	670	my $num_cols = scalar(@collections); # remember the empty case
	671
	672	if($num_cols == 0) { # deal with all collections
	673	push (@collections, "");
[28615]	674	# putting the empty string in the array so that the "all collections" case
	675	# can be handled similar to how the case of user-specified collections is handled
[28114]	676
	677	} else { # deal with user specified set of collections
	678	# prefix the directory separator to each collection name
	679	@collections = map { $sep.$_ } @collections;
	680	}
	681
[27764]	682	my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
	683	if(!-d $pdfbox) {
	684	&getPDFBox();
	685	}
	686
[28615]	687	my $openofficeext = &filename_concat($greenstone_home, "ext", "open-office");
	688	if(!-d $openofficeext) {
	689	&getOpenOfficeExt();
	690	}
	691
	692	#&getIsisGdl("$greenstone_home");
[28605]	693
[27668]	694	open (my $xml_fh, '>'.$xmlout) \|\| die "Could not open xml file $xmlout for appending: $!\n";
[27621]	695
	696	# perform the requested subcommands, outputting xml information
	697	print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
[28072]	698
[28114]	699	my ($cmd, $status);
[27724]	700	# make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
[28072]	701
	702	if(!$use_local_rebuild) {
[28661]	703	print $xml_fh "Updating $ENV{'TASK_HOME'}/$model_collect:\n";
[28114]	704	for my $col (@collections) {
[28661]	705	$cmd = "svn up $ENV{'TASK_HOME'}/$model_collect$col"; #chdir("$ENV{'TASK_HOME'}/$model_collect");
[28114]	706	$status = system "$cmd";
	707	}
[28072]	708	}
[27724]	709
[28661]	710	# go to whichever collecthome parent we're using
	711	chdir($collect_parent);
[27621]	712
	713	# get svn info
[27629]	714	print STDERR "getting svn info: $xmlout\n";
[27621]	715	print $xml_fh "<svn-info>\n";
[27668]	716	&run_and_print_cmd("svn info", $xml_fh);
[27621]	717	print $xml_fh "</svn-info>\n";
[27629]	718	print STDERR "done\n";
[27621]	719
[28114]	720	if(!$use_local_rebuild) {
	721
	722	#make two copies of the model-collect directory in gsdl
	723	#one to be rebuilt and one as the basis for comparison
	724	#strip both of all .svn directories
	725
	726	#copy the model collections to the collect folder to be rebuilt
	727	print STDERR "installing test collections and model collections to new $gsdl installation... ";
[28072]	728	#clean up
	729	if(-d "collect") {
[28114]	730	for my $col (@collections) {
[28172]	731	if(-d "collect$col") {
	732	&File::Path::remove_tree("collect$col") \|\| die "Error could not delete collect: $!";
	733	}
[28114]	734	}
[28072]	735	}
[27621]	736
[28072]	737	if($use_static_model) {
[28114]	738	for my $col (@collections) {
	739	#copy to collect and strip .svn subfolders
	740	&File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
	741	&copy_recursively("model-collect$col", "collect$col", ".svn");
	742	}
[27621]	743
[28072]	744	} else { # the default situation: where we check out the model-collect from svn
[28114]	745	for my $col (@collections) {
	746	&File::Path::remove_tree("model-collect$col");
	747
	748	#copy to collect and strip .svn subfolders
	749	&File::Path::make_path("collect$col"); # create the folder and copy contents across
[28661]	750	&copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","$model_collect$col"), "collect$col", ".svn");
[28114]	751
	752	#make the model copy
	753	&File::Path::make_path("model-collect$col");
	754	&copy_recursively("collect$col", "model-collect$col"); # copy contents across
	755	}
[28072]	756	}
	757
	758	print STDERR "done\n";
	759	}
	760
[28114]	761	#for each collection, import, build and diff with its model counterpart
	762
	763	# if working with all collections, read the list of collections from the folders in collect
	764	if($num_cols == 0) {
	765	@collections = (); # get rid of the empty string put in the array to represent "all collections"
	766
[28661]	767	opendir my($collect_handle), "collect" or die "Could not open dir $collect_parent/collect: $!";
[28114]	768
	769	for my $collection (readdir $collect_handle) {
	770	next if ($collection eq "." \|\| $collection eq "..");
	771	next if ($collection eq "modelcol");
	772	push(@collections, $collection);
	773	}
	774	closedir $collect_handle; # close handle to collect dir
	775	}
	776
	777	for my $collection (@collections) {
[27621]	778
[28114]	779	# next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
	780	# next if ($collection !~ m/OAI\|METS\|DSpace\|MGPP\|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
	781
[27621]	782	#escape the filename (in case of space)
	783	$collection =~ s@ @\\ @g;
	784	#getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
[28114]	785
	786	$collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
	787
	788	if (! -d "collect$sep$collection") {
	789	print STDERR "Collection $collection does not exist\n";
	790	next;
	791	}
	792
[27621]	793	print STDERR "*** Found collection $collection\n";
	794	print $xml_fh "<collection-test name=\"$collection\">\n";
	795
[28661]	796	# run the building scripts from the toplevel of the GS installation
	797	if($install_version eq "3") {
	798	chdir($greenstone3_home);
	799	} else {
	800	chdir($greenstone_home);
	801	}
	802
[28072]	803	if(!$use_local_rebuild) {
	804	#import
	805	# Ensure the OIDtype for importing is hash_on_full_filename
	806	# "to make document identifiers more stable across upgrades of the software,
	807	# although it means that duplicate documents contained in the collection are
	808	# no longer detected automatically."
	809	print STDERR "$collection - Importing:\n";
	810	print $xml_fh "<import>\n";
[28661]	811	&run_build_script("import.pl $build_options -removeold $collection"); #-OIDtype hash_on_full_filename
[28072]	812	print $xml_fh "</import>\n";
	813	print STDERR "done\n";
[27621]	814
[28072]	815	#build
	816	print STDERR "$collection - Building:\n";
	817	print $xml_fh "<build>\n";
[28661]	818	&run_build_script("buildcol.pl $build_options -removeold $collection");
[28072]	819	print $xml_fh "</build>\n";
	820	print STDERR "done\n";
	821
	822	#rename the intermediate 'building' directory 'index'
	823	print STDERR "$collection - Move \"building\" to \"index\"... ";
[28661]	824	my $index = &filename_concat($collect_parent, "collect", $collection, "index");
	825	my $building = &filename_concat($collect_parent, "collect", $collection, "building");
[28072]	826	&File::Path::remove_tree($index);
	827	# Renaming Directories, http://www.perlmonks.org/?node_id=177421
	828	move($building, $index) or die "copy failed: $!"; # File::Copy::move
	829	print STDERR "done\n";
	830	}
[27621]	831	#diffcol
[27629]	832	print STDERR "$collection - Diffing:\n";
[27621]	833	my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
[28172]	834
[28661]	835	# chdir($collect_parent); # this is actually where we are
[28172]	836	# help diffcol to know on what os the model cols were generated
	837	# and what os this test machine is (on which the test cols will be generated)
	838	$cmd = "diffcol.pl -testos $test_os -modelos $model_os -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
[27621]	839	&run_diff_script($cmd, $xml_fh, $diffcol_dir);
	840
[28661]	841	if($install_version eq "3") {
	842	chdir($greenstone3_home); # this is actually where we are
	843	} else {
	844	chdir($greenstone_home); # this is actually where we are
	845	}
[27629]	846	print STDERR "done\n";
[27621]	847	print $xml_fh "</collection-test>\n";
	848	}
	849
	850	print $xml_fh "</test>\n";
	851	close($xml_fh);
	852
[27629]	853	print STDERR "done\n";
[27621]	854	}
	855
	856	##***************************************************************
	857	# runs setup in greenstone_home before running the diff command
	858	sub run_diff_script {
	859	my ($cmd, $fh, $diffcol_dir) = @_;
	860
[28661]	861	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
	862
[27621]	863	# we're in greenstone_home now
	864	if(!$isWin) {
[28661]	865	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && cd $diffcol_dir && ./$cmd\"";
[27621]	866
	867	} else { # Need to prefix cmd -c/-k as necessary
[28661]	868	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
[27668]	869	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	870	}
	871
	872	return &run_and_print_cmd($cmd, $fh);
	873	}
	874
	875	# runs setup in greenstone_home before running the given build command
	876	sub run_build_script {
	877	my ($cmd, $fh) = @_;
[28661]	878	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
[27621]	879
	880	# chdir($greenstone_home);
[27687]	881	# we are in $greenstone_home already, can directly run the build cmd on the collection
[27621]	882	if(!$isWin) {
[28661]	883	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && $cmd\"";
[27621]	884
	885	} else { # Need to prefix cmd -c/-k as necessary
[28661]	886	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
[27621]	887	}
[27687]	888	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	889
[27668]	890	return system($cmd);
[27629]	891	#return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
[27621]	892	}
	893
	894
	895	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
	896	# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
	897	sub run_and_print_cmd {
	898	my ($cmd, $fh) = @_;
	899
	900	open my $pin, "$cmd\|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-\|', 'powercfg -l') or die $!;
	901
	902	if(defined $fh) { # print cmd output both to the filehandle and to stdout
	903	while (my $line = <$pin>) {
	904	print $fh $line;
	905	# print STDOUT $line; # if also printing cmd output to STDOUT
	906	}
	907	}
	908	else { # no filehandle, so just need to print to stdout
	909
	910	# unlike backticks operator, system() will print the output of the command to the screen as it executes
	911	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	912
	913	my $status = system $cmd;
	914	if($status != 0) {
	915	print STDERR "ERROR ($status) running $cmd: $!\n";
	916	}
	917	}
	918	close($pin);
	919	}
	920
	921	sub filename_concat {
	922	my $first_file = shift(@_);
	923	my (@filenames) = @_;
	924
	925	# If first_file is not null or empty, then add it back into the list
	926	if (defined $first_file && $first_file =~ /\S/)
	927	{
	928	unshift(@filenames, $first_file);
	929	}
	930
	931	my $filename = join($sep, @filenames);
	932	$filename =~ s/[\\\/]$//; # remove trailing slashes if any
	933	return $filename;
	934	}
	935
	936
	937	# The following code is from
	938	# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
	939	# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
	940	sub copy_recursively {
	941	my ($from_dir, $to_dir, $regex) = @_;
	942	opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
	943
	944	# if(-d !$to_dir) {
	945	# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
	946	# }
	947
	948	for my $entry (readdir $dh) {
	949	next if ($entry eq "." \|\| $entry eq "..");
	950	next if (defined $regex && $entry =~ /$regex/);
	951	my $source = "$from_dir/$entry";
	952	my $destination = "$to_dir/$entry";
	953	if (-d $source) {
	954	mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
	955	copy_recursively($source, $destination, $regex);
	956	} else {
	957	copy($source, $destination) or die "copy failed: $!";
	958	}
	959	}
	960	closedir $dh;
	961	return;
	962	}
	963
	964	sub summarise {
	965
	966	# make a summarised Xml report
[27629]	967	print STDERR "Summarizing the xml report... ";
[28661]	968	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml";
[27621]	969	my $status = system($cmd);
[27629]	970	print STDERR "done\n";
[27621]	971
	972	# make a summarised HTMl report
[27629]	973	print STDERR "Creating an html summary report... ";
[28661]	974	$cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.html";
[27621]	975	$status = system($cmd);
[27629]	976	print STDERR "done\n";
[28005]	977
	978	# Print whether the tests passed or failed
	979	print STDERR "*******************************************\n";
	980	print STDERR "Checking if successful... \n";
	981	$cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	982	$status = `$cmd`; #$status = system($cmd);
	983	print STDERR "result: $status\n";
	984	print STDERR "*******************************************\n";
[27621]	985	}
	986
	987	sub upload {
[27657]	988	# if the upload dir already existed, clear it of contents
	989	if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
	990	# don't want to keep previous days reports
	991	# else we will have to manually clear them at some point
	992	# just generate the set of reports for this run of task.pl upload
	993	# and
	994	&File::Path::remove_tree($ENV{'UPLOAD_DIR'});
	995	}
	996	# recreate the upload directory
	997	&File::Path::make_path($ENV{'UPLOAD_DIR'});
[27621]	998
	999	# copy all .xml and .html files across to UPLOAD_DIR
	1000	opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
	1001	for my $entry (readdir $dh) {
[27687]	1002	next if ($entry !~ m/(\.xml\|\.html?)$/);
[28661]	1003	next if ($entry !~ m/(report-$install_version)/);
[27687]	1004
	1005	# copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
	1006	# html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
[27694]	1007	my $os_entry = $entry;
	1008	$os_entry =~ s@\[email protected]@ if $isWin;
[28571]	1009	if($isMac) {
[28634]	1010	$osversion = "Leopard-" if ($osversion eq "");
[28572]	1011	$os_entry = "diffcol-".$^O."-".$osversion."$os_entry"; # darwin-Lion for Lion/Mountain Lion
[28571]	1012	} else {
	1013	$os_entry = "diffcol-".$^O."-$os_entry";
	1014	}
[27687]	1015
[28621]	1016	# if the test failed, prefix "failed" to the report so that it shows up with an error icon on the caveat page
	1017	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1018	my $result = `$cmd`;
	1019	if($result ne "yes") {
	1020	$os_entry =~ s/diffcol-/diffcol-FAIL-/;
	1021	}
	1022
[27687]	1023	# get the absolute path to the original files before copying them over
	1024	$entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
	1025
	1026	# copy them over with their new names
	1027	## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
	1028	copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
[27621]	1029	}
	1030	closedir $dh;
	1031
[27637]	1032
	1033	# Upload the html file to puka
	1034	#default identity dir
	1035	if ( ! exists $ENV{'IDENTITY_DIR'} ) {
[27725]	1036	$ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
[27637]	1037	}
	1038	if (! exists $ENV{'SNAPSHOT_MODE'} ) {
	1039	$ENV{'SNAPSHOT_MODE'} = "caveat";
	1040	}
	1041
[27678]	1042	#use the correct key for uploading
[27637]	1043	$ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
	1044	if(-f $ENV{'IDENTITY_FILE'}) {
[27678]	1045	# if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
	1046
[28661]	1047	# the report we want to upload is actually just os-diffcol-report-$install_version-$dateid.html
[27687]	1048	my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c .htm \| "; #&& cat .html \| "; # && tar -c \|
[27637]	1049	$command .= ($^O eq "MSWin32" ? "plink" : "ssh");
	1050	$command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
	1051	#print "$command\n";
	1052	my $status = system("$command");
	1053	if($status != 0) {
[27678]	1054	print STDERR "*** Failed to upload test report to nzdl $status\n";
[27637]	1055	}
	1056	} else {
	1057	print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
	1058	}
	1059
[27629]	1060	print STDERR "Finished uploading\n";
[27621]	1061	}
	1062
	1063	# Sending emails with perl: http://learn.perl.org/examples/email.html
	1064	# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
[27628]	1065	# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
	1066	sub mail_with_report_attached
	1067	{
	1068	# email out with report attached, if the tests failed
[27629]	1069	print STDERR "Checking if successful... \n";
[27628]	1070	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1071	#my $result = system($cmd);
	1072	my $result = `$cmd`;
[27621]	1073
[27629]	1074	print STDERR "result: $result\n";
[27621]	1075
[27628]	1076	if($result ne "yes") {
	1077	my $msg = "$gsdl regression test for $dateid failed";
	1078	my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
[28661]	1079	my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$install_version-$dateid.html");
[27628]	1080
[27687]	1081	if($isWin) {
	1082	if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
[27678]	1083	# http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
	1084	#blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
[27628]	1085
[27678]	1086	# need to install blat on windows
	1087	$cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
	1088	$result = system($cmd);
	1089	}
	1090	else {
[27687]	1091	$result = 1; # status from running mail command is 0 if success, 1 if fail
[27678]	1092	print STDERR "********************************************\n";
[27687]	1093	if ($use_blat) {
	1094	print STDERR "Need blat and SMTP set to send mail attachment\n" ;
	1095	} else {
	1096	print STDERR "Not set up to send mail on Windows\n";
	1097	}
[27678]	1098	print STDERR "Inspect report at: $attach_file\n";
	1099	print STDERR "********************************************\n";
	1100	}
	1101	} else { # linux
[27628]	1102	my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
	1103
	1104	if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
	1105	$cmd="echo '$gsdl regression test for $dateid failed.' \| mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
	1106
	1107	print STDERR "********************************************\n";
	1108	print STDERR "No mutt installed, unable to mail attachment\n";
	1109	print STDERR "Inspect report at: $attach_file\n";
	1110	print STDERR "********************************************\n";
	1111	} else {
	1112	#$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
	1113	$cmd = "echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
	1114	}
[27678]	1115
	1116	# run the mail command
	1117	$result = system($cmd); #&run_and_print_cmd($cmd);
[27628]	1118	}
	1119
[27678]	1120
[27637]	1121	if($result != 0) {
	1122	print STDERR "*** Unable to send email: $?\n";
	1123	}
	1124	else {
	1125	print STDERR "Sent mail with report attached.\n";
	1126	}
[27628]	1127	} else {
	1128	print STDERR "********************************************\n";
[27629]	1129	print STDERR "Tests were successful. Not sending mail.\n";
[27628]	1130	print STDERR "********************************************\n";
	1131	}
	1132	}
	1133
[27621]	1134	# The old version of this program contained the following, consisting of 1 line of active code:
	1135
	1136	# Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
	1137	# Doing so will call this pl file and pass in "summarise" in ARGV
	1138	# This pl file will in turn call the task executable in this folder
	1139	# passing in "summarise" as a parameter.
	1140	#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
	1141
	1142	##system("/bin/bash -c \"./task @ARGV\"");
	1143	##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
[27694]	1144

Note: See TracBrowser for help on using the repository browser.

Download in other formats: