Context Navigation

source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 29493

Last change on this file since 29493 was 29493, checked in by sjs49, 9 years ago
Minor changes
Property svn:executable set to ``*
File size: 42.7 KB

Rev	Line
[27621]	1	#!/usr/bin/perl -w
[27556]	2
[27621]	3	# This program is meant to run the nightly diffcol
	4	# It is meant to be an equivalent for the existing task bash script
	5	# But it is intended to be expanded to work for Windows and GS3
	6	# For windows, need to REMEMBER to set the correct shebangs at the top
[27556]	7
[27663]	8
	9	# TODO:
	10	# Have a caveat mode and a stable mode (as in snapshot/task.pl)
	11	#
	12	#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat\|stable)" ) {
	13	# $major_version = 2;
	14	# $prefix="2t";
	15	# $rk="tk2"; # test kit
	16	#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat\|stable)" ) {
	17	# $major_version = 3;
	18	# $prefix="3t";
	19	# $rk="tk3"; # test kit
	20
[27628]	21	package diffcoltask;
[27556]	22
[27621]	23	use Cwd;
[29398]	24	#use Switch; # for switch(val) { case: ; ...}
[27621]	25	use File::Path; # for rmdir and mkdir type functions
	26	use File::Copy; # for recursive copying of folders but skipping .svn
	27	use File::Basename;
	28
[27628]	29	use strict;
	30	no strict 'subs'; # allow barewords (eg STDERR) as function arguments
[27621]	31
[27764]	32
[27621]	33	my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
[27764]	34	my $isMac = ($^O =~ m/macos\|darwin/i) ? 1 : 0;
[28571]	35
	36	my $osversion="";
	37	# Need to get the correct gnome-lib-minimal for the OS
	38	# darwin11* Lion, darwin12* Mountain Lion, darwin9* and darwin10* are Leopard and Snow Leopard
	39	if ($^O eq "darwin") {
	40	$osversion=`uname -r`; # e.g. 12.x.x
	41	#$osversion =~ s@\..*$@@; # e.g.12
[28634]	42	$osversion = ($osversion =~ m@^1[1-9](\.)?@i) ? "Lion-" : "";
[28571]	43	}
	44
	45
[27621]	46	my $sep = $isWin ? "\\" : "/";
	47	my $pathsep = $isWin ? ";" : ":";
	48	#my $script_ext = $isWin ? ".bat" : ".bash";
[28661]	49	my $setup_script = "setup"; # will become gs3-setup for GS3
[27687]	50	my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
[27621]	51
[28616]	52	my $install_type = "svn";
	53	my $install_version = "2";
	54
[28106]	55	my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
[28072]	56	my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
[27701]	57
[28072]	58	# if use_local_rebuild is on, use_static_model should be on
	59	if ($use_local_rebuild && !$use_static_model) {
	60	$use_static_model = 1;
	61	}
	62
[28172]	63	my $test_os = $isWin ? "windows" : ($isMac ? "darwin" : "linux");
	64	my $model_os = "linux"; # default
[28072]	65
[27621]	66	# TASK_HOME should be the toplevel diffcol folder
	67	$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
[27701]	68	if($isWin) {
	69	$ENV{'TASK_HOME'} =~ s@\/@\\@g;
	70	# need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
	71	# run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
	72	# At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
	73	# end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
	74	require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
	75	$ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
	76	}
[27621]	77	## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
	78
	79
[27678]	80	$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
	81
	82	# we'll be using BLAT to send mail attachments on Windows
[27687]	83	my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
	84	if($isWin && $use_blat && ! -e $blat) {
	85	print STDERR "\n***********************************\n";
	86	print STDERR "No blat.exe found in $blat.\n";
	87	print STDERR "Blat needed to send mail with attachments on Windows.\n";
	88	print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
	89	print STDERR "for your bit architecture and name the folder 'blat'\n";
	90	print STDERR "***********************************\n\n";
[27678]	91	$blat = 0;
	92	}
	93
	94
[27668]	95	$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
	96	$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
[27621]	97	$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
[27678]	98	$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
[27621]	99	##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
	100
[29448]	101	# When installing diffcol the first time, force the user to create the data-dir and upload-dir
	102	die "\n@@@@ data dir: ".$ENV{'DATA_DIR'}." does not exist\n" unless (-d $ENV{'DATA_DIR'});
	103	die "\n@@@@ reports/upload dir: ".$ENV{'UPLOAD_DIR'}." does not exist\n" unless (-d $ENV{'UPLOAD_DIR'});
	104
[27621]	105	# control if an existing compiled greenstone is used
	106	# or, if one should be checked out, which revision to checkout from svn
	107	$ENV{'SVN_OPT_REV'} = "-r head";
[27656]	108	#$ENV{'GSDLHOME'}=
	109	#$ENV{'GSDL3SRCHOME'}=
[27621]	110
[27628]	111
[27677]	112	# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
	113	my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
[27628]	114
	115	#parse arguments
	116	my $action = "all";
[28114]	117	my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
[28620]	118	my @collections = (); # list of collections that run_test should process
[28114]	119
[28620]	120	if(scalar(@ARGV) == 0) {
	121	$action="all";
	122	}
	123
	124	# process any arguments that are --gs2\|--gs3 and --bin\|--svn, and delete them from the array
	125	# if none provided, it's gs2 and svn by default.
	126	for (my $i = $#ARGV; $i >= 0; --$i) {
	127	if($ARGV[$i] =~ m/--(bin\|svn)/) {
	128	$install_type = $1;
	129	splice @ARGV, $i, 1; # remove the element from the argument array
	130	} elsif($ARGV[$i] =~ m/--gs(2\|3)/) {
	131	$install_version = $1;
[28661]	132	$setup_script = $install_version eq "3" ? "gs3-setup" : "setup"; # needs to become gs3-setup for GS3
[28620]	133	splice @ARGV, $i, 1; # remove the element from the argument array
	134	}
	135	}
	136
	137
[28172]	138	# run_test can take any number of args
[28620]	139	if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
[28172]	140	print STDERR "**** Wrong number of arguments\n";
[28114]	141	&printusage();
	142	exit -1;
[27628]	143	}
	144
[28620]	145	if(scalar(@ARGV) > 0) {
[29398]	146	#switch ($ARGV[0]) {
	147	#case qr/^(-h\|--?help\|help)$/i { &printusage; exit 0; }
	148	#case qr/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/ { $action=$ARGV[0]; }
	149	#else {
	150	#print STDERR "**** Bad subcommand.\n";
	151	#&printusage;
	152	#exit -1;
	153	#}
	154	#}
	155
	156
	157	my $switch = $ARGV[0];
	158	if($switch =~ m/^(-h\|--?help\|help)$/i) {
	159	&printusage;
	160	exit 0;
	161	} elsif ($switch =~ m/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/) {
	162	$action=$ARGV[0];
	163	} else {
	164	print STDERR "**** Bad subcommand.\n";
	165	&printusage;
	166	exit -1;
[27628]	167	}
[28616]	168
[28172]	169	# run_test action can take a subaction: nosvn\|justdiff. It can also take --modelOS (windows\|linux\|darwin)
[28114]	170	# nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
	171	# it out from the svn model-collect again.
	172	# justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
[28616]	173
[28620]	174	if($action eq "run_test" && scalar(@ARGV) >= 2) {
[28114]	175	push(@collections, @ARGV);
	176	shift @collections; # remove action from array
	177
[28172]	178	for (my $i=0; $i < scalar(@ARGV); $i++) {
	179	if($ARGV[$i] =~ m@^--@) {
	180	shift @collections; # remove subaction/flag from array
[28114]	181
[28172]	182	$subaction = $ARGV[$i];
[28114]	183	if($subaction eq "--justdiff") {
[28172]	184	$use_local_rebuild = $use_static_model = 1;
[28114]	185	} elsif ($subaction eq "--nosvn") {
[28172]	186	$use_static_model = 1;
	187	#} elsif ($subaction =~ m/\-\-testOS/i && defined $ARGV[$i+1]) {
	188	# $test_os = $ARGV[$i+1];
	189	# $i++;
	190	# shift @collections; # remove test_os value from array
	191	} elsif ($subaction =~ m/\-\-modelOS/i && defined $ARGV[$i+1] && $ARGV[$i+1] =~ m/windows\|linux\|darwin/i) {
	192	$model_os = $ARGV[$i+1];
	193	$i++;
	194	shift @collections; # remove model_os value from array
	195	#print STDERR "Model_os specified: $model_os\n";
[28114]	196	} else {
[28172]	197	print STDERR "**** Bad subaction/value: ".$ARGV[$i]."\n";
	198	&printusage;
	199	exit -1;
	200	}
	201	}
[28114]	202	}
	203
	204	# foreach my $col (@collections) {
	205	# print STDERR "Collection: $col\n";
	206	# }
	207	}
[27628]	208	}
	209
[28661]	210	print STDERR "Install type $install_type\n";
	211	print STDERR "Install version $install_version\n";
[28114]	212
[27621]	213	#check key environment vars are set
	214	if(!defined $ENV{'UPLOAD_DIR'}) {
[27629]	215	print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
[27621]	216	#return 1;
	217	}
	218	if(!defined $ENV{'DATA_DIR'}) {
[27629]	219	print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
[27621]	220	#return 1;
	221	}
	222	if(!defined $ENV{'MONITOR_EMAIL'}) {
[27629]	223	print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
[27621]	224	#return 1;
	225	}
	226
	227	if($ENV{'DATA_DIR'} eq "/") {
[27629]	228	print STDERR "DATA_DIR should not be the fs root\n";
[27621]	229	#return 1;
	230	}
	231
[27629]	232	print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
	233	print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
[27621]	234
	235	#create an id for this test
	236	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	237	$year += 1900;
	238	$mon += 1;
	239	$mon = "0$mon" if ($mon < 10);
	240	$mday = "0$mday" if ($mday < 10);
[27702]	241	my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
[27621]	242
[27629]	243	print STDERR "Starting test '$dateid'\n";
[27621]	244
	245
	246	# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
	247	$ENV{'CLASSPATH'} = "";
	248	my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
	249	my @files = <$jar_lib_path/.jar>; # /full/path/to/diffcol/lib/jar
[27628]	250	foreach my $file (@files) {
[27668]	251	$file =~ s@\/@\\@g if $isWin;
[27621]	252	$ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
	253	}
	254	##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
	255
	256
	257	#set the location of the full report
[28661]	258	my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$install_version-$dateid.xml");
[27621]	259	##print STDERR "XML: $xmlout\n";
	260
	261	# the toplevel folder of the greenstone installation being used
[28661]	262	my $greenstone3_home="";
	263	my $greenstone_home=""; # gs2build for gs3, toplevel install folder for gs2
[27621]	264	# gsdl is the checkout folder and can be greenstone2 or greenstone3
	265	my $gsdl="greenstone2";
	266
	267	# Check if using existing compiled-up greenstone installation
	268	# and set the greenstone_home location accordingly
	269
	270	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27629]	271	print STDERR "Found existing Greenstone home, will use that instead\n";
[28661]	272	$greenstone_home=$ENV{'GSDLHOME'} if defined $ENV{'GSDLHOME'};
	273
	274	if(defined $ENV{'GSDL3SRCHOME'}) {
	275	print STDERR "*** GSDL3SRCHOME set, changing to using installed GS3\n";
	276	$gsdl = "greenstone3";
	277	$install_version = "3";
	278	$greenstone3_home=$ENV{'GSDL3SRCHOME'};
	279	$greenstone_home=filename_concat($greenstone3_home, "gs2build") unless defined $ENV{'GSDLHOME'};
	280	}
[27621]	281	} else {
[28661]	282	if($install_version eq "3") {
	283	$gsdl = "greenstone3";
	284	$greenstone3_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	285	$greenstone_home=filename_concat($greenstone3_home, "gs2build");
	286	} else {
	287	$greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	288	}
[27621]	289	}
	290	##print STDERR "GSHOME: $greenstone_home\n";
	291
[27628]	292	#do the requested action
	293	if($action eq "setup_greenstone") {
	294	&setup_greenstone;
	295	}
	296	elsif ($action eq "run_test") {
	297	&run_test;
	298	}
	299	elsif ($action eq "summarise") {
	300	&summarise;
	301	}
	302	elsif ($action eq "upload") {
	303	&upload;
	304	&mail_with_report_attached;
	305	}
	306	elsif ($action eq "all") {
	307	&setup_greenstone;
	308	&run_test;
	309	&summarise;
	310	&upload;
	311	&mail_with_report_attached;
	312	}
[27621]	313
	314	##********************************
	315
[27628]	316	sub printusage
	317	{
[28172]	318	# print STDERR "Run as: $0 (help\|setup_greenstone\|run_test <--modelOS windows\|darwin\|linux> <--justdiff\|--nosvn> <col1 col2 ...> \|summarise\|upload\|all)\n";
	319	print STDERR "Run as: $0 (help\|setup_greenstone\|run_test\|summarise\|upload\|all)\n";
	320	print STDERR "where run_test can further take the following optional parameters:\n";
	321	print STDERR "\t--modelOS (windows\|darwin\|linux)\n";
	322	print STDERR "\t--justdiff\|--nosvn\n";
	323	print STDERR "\t<col1 col2 ...>\n";
[28616]	324	print STDERR "where setup_greenstone can further take the following optional parameters:\n";
	325	print STDERR "\t--gs2\|--gs3\n";
	326	print STDERR "\t--svn\|--bin\n";
[27628]	327	}
[27621]	328
	329	#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
	330
	331	sub setup_greenstone
	332	{
[28661]	333	if($install_version eq "3" && !defined $ENV{'JAVA_HOME'}) {
	334	print "*** JAVA_HOME not set. Set it and add its bin folder to the PATH\n";
	335	exit 0;
	336	}
	337
[27621]	338	#clean up from previous tests
[27628]	339	print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
[27621]	340	for my $i ( 1..5 ) {
	341	sleep 1; # 1 second
[27628]	342	print STDERR ".";
[27621]	343	}
[27628]	344	print STDERR "\n";
[27621]	345
	346	# http://perldoc.perl.org/File/Path.html
[27629]	347	print STDERR "cleaning up previous tests\n";
[28661]	348	#&File::Path::remove_tree($ENV{'DATA_DIR'});
	349	if(-d $greenstone_home && $install_version eq "2") {
	350	&File::Path::remove_tree($greenstone_home);
	351	}
	352	if(-d $greenstone3_home && $install_version eq "3") {
	353	&File::Path::remove_tree($greenstone3_home);
	354	}
	355	unlink glob "$ENV{'DATA_DIR'}$sep"."report-$install_version";
	356	unlink "$ENV{'DATA_DIR'}$sep"."compilation-errors";
[27629]	357	print STDERR "creating the data dir\n";
[28661]	358	#&File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
[27621]	359
	360	chdir($ENV{'DATA_DIR'});
	361
	362	# use existing compiled-up greenstone installation, if a GSDLHOME set
	363	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27710]	364	print STDERR "Found existing Greenstone home, will use that instead\n";
[27621]	365	return;
	366	}
	367
	368	# Else checkout a GS from svn into DATA_DIR
[28661]	369	if($install_type eq "svn") {
	370	&checkout_gs_from_svn();
	371	##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
[27621]	372
[28661]	373	if($install_version eq "2") {
	374	&compile_gs2_svn();
	375	} else {
	376	&compile_gs3_svn();
	377	}
	378	}
	379
	380	}
	381
	382	sub checkout_gs_from_svn {
	383
[27621]	384	#svn checkout of main gsdl directory
[27629]	385	print STDERR "checkout $gsdl:\n";
[28661]	386	my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone$install_version $gsdl";
[27621]	387	##print STDERR "Checkout CMD: $cmd\n";
	388
	389	# # unlike backticks operator, system() will print the output of the command to the screen as it executes
	390	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	391	my $status = system "$cmd"; #my $status = `$cmd`;
[28661]	392	if($status != 0) {
	393	print STDERR "@@@ SVN checkout of $gsdl failed\n";
	394	exit -1;
	395	}
[27629]	396	print STDERR "done\n";
[28661]	397	}
[27621]	398
[28661]	399	sub getImageMagickBins {
	400	my ($cmd, $status);
	401	my $imagickzip = "";
[27621]	402
[28661]	403	chdir ("$greenstone_home");
	404	if(!$isWin) { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
[27710]	405
	406	my $bit_arch=`uname -m`;
[28661]	407
	408	# imagemagick binary
[27710]	409	print STDERR "Getting imagemagick binary\n";
[28661]	410
[27724]	411	my $os = $isMac ? "darwin" : "linux";
[28661]	412	$imagickzip = "imagemagick-$os";
[27668]	413
[27710]	414	if($isMac) {
[28661]	415	# $imagickzip .= "-10.5.tar.gz";
[27962]	416	# at present, only the Imagemagick binaries created by Max for darwin work on the Macs
[28661]	417	# &File::Path::make_path("$greenstone_home$sep"."bin"."$sep$os"); # need to ensure gsdl/bin/darwin exists
[27962]	418	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
	419	$status = system($cmd);
[28661]	420	if($status != 0) {
	421	print STDERR "@@@ Unable to get imagemagick for darwin\n";
	422	}
	423
	424	# need ghostscript mac binary too for pdf to img conversions on mac
[27963]	425	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
	426	$status = system($cmd);
[28661]	427	if($status != 0) {
	428	print STDERR "@@@ Unable to get ghostscript for darwin\n";
	429	}
	430
	431	# the imagemagick and ghostscript binaries have been set to executable on svn trac now
	432	# system("chmod -R u+x $greenstone_home/bin/darwin/imagemagick/bin/*");
	433	# system("chmod -R u+x $greenstone_home/bin/darwin/ghostscript/bin/*");
[27710]	434	} else { # linux
	435	my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
[27724]	436	$imagickzip .= "$extension64.tar.gz";
[27962]	437
	438	# now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
	439	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
	440	$status = system ($cmd);
	441	system("cd ext && tar -xvzf $imagickzip");
[27724]	442	}
[28661]	443	}
	444	return $imagickzip;
	445	}
[27710]	446
[28661]	447	sub getGnomeLibExt
	448	{
[29455]	449	if($isWin) {
	450	return; # no gnome-lib for windows
	451	}
[28661]	452	my ($cmd, $status);
	453	my $bit_arch=`uname -m`;
	454	my $os = $isMac ? "darwin" : "linux";
	455
[27710]	456	# gnomelib binary
[27629]	457	print STDERR "setting up gnome-lib-minimal for compilation\n";
[27621]	458
	459	# To get gnome-lib, need to determine bit architecture of the linux/darwin
	460	# http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
	461	# $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
	462	# well on ActivePerl.
[28621]	463	# But since we know we're on a Linux/Darwin machine at this point, we can just run `uname -m` and other linux cmds
[27621]	464
[28571]	465	# osversion will be "Lion" or ""
	466	# and assuming all darwin is intel, not ppc!!
[28572]	467	my $gnome_lib_file = $isMac ? "darwin-".$osversion."intel" : "linux";
[27621]	468
[28571]	469	$gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/ && !$isMac); # linux only case
[27621]	470
	471	#svn checkout gnome-lib for this linux/darwin
[28661]	472	chdir("$greenstone_home$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
[27621]	473
	474	##print STDERR "**** gnomelib: $gnome_lib_file\n";
	475
	476	# checkout and unpack gnome-lib-minimal
	477
	478	#svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
	479	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
	480	system $cmd;
	481	system ("tar -xvzf gl.tar.gz");
	482
	483	##print STDERR "*** ARCH: $bit_arch\n";
[28661]	484	}
[27621]	485
[28661]	486
	487
	488	sub compile_gs3_svn() {
	489	my ($cmd, $status);
	490	chdir ("$greenstone3_home");
	491
	492	$cmd = "ant"; # creates the build.properties file from the .in template
	493	$status = system $cmd;
	494	if($status != 0) {
	495	print STDERR "Unable to run the ant command: $status\n";
	496	exit -1;
	497	}
	498
	499	$cmd = "ant -Dproperties.ok=y -Dcheckout.gnomelib.ext=true prepare"; # pass in confirmation to ant prepare step
	500	$status = system $cmd;
	501	if($status != 0) {
	502	print STDERR "Failed to run $cmd command: $status\n";
	503	exit -1;
	504	}
	505
	506	my $imagickzip = &getImageMagickBins();
	507	chdir ("$greenstone3_home");
	508
	509	$cmd = "ant -Dcheckout.gnomelib.ext=true install"; # Compile with gnome-lib.
	510	$status = system $cmd;
	511	if($status != 0) {
	512	print STDERR "Failed to run $cmd command: $status\n";
	513	exit -1;
	514	}
	515	}
	516
	517	sub compile_gs2_svn() {
	518	my $imagickzip = &getImageMagickBins();
	519	&getGnomeLibExt();
	520
	521	my ($cmd, $status);
	522	my $os = $isMac ? "darwin" : "linux";
	523
	524	chdir("$greenstone_home"); #chdir("$ENV{'DATA_DIR'}$sep$gsdl"); # goes into toplevel gs2 or gs3 folder
	525
	526	##print STDERR "@@@ OS: $^O.\|".$Config{'archname64'}."\|\n";
	527
	528	if($isWin) {
	529	print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
	530
	531	# we're now in the GS2 folder, call makegs2 with silent param
	532	$cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
	533	$status = system $cmd;
	534	if($status != 0) {
	535	print STDERR "Greenstone compilation on Windows failed\n";
	536	exit -1;
	537	}
	538
	539	} else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
	540
	541	chdir("ext$sep"."gnome-lib-minimal");
	542
[27621]	543	# need to run source devel.bash on gnome-lib followed by configure, make, make install
	544	# in one go, in order to preserve the compile environment set up by sourcing devel.bash
	545
	546	# http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
	547	# http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
	548
	549	# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
	550	$cmd = "bash -c \"";
	551
	552	$cmd .= "source ./devel.bash";
	553	$cmd .= " && cd ../..";
	554
	555	#configure
	556	# $cmd .= " && ./configure";
	557	$cmd .= " && echo 'configure $gsdl: ' ";
	558	$cmd .= " && echo '<configure>' >> $xmlout";
	559	$cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
	560	$cmd .= " && echo '</configure>' >> $xmlout";
	561	$cmd .= " && echo 'done'";
	562
	563	#make
	564	$cmd .= " && echo 'make $gsdl: '";
	565	$cmd .= " && echo '<make>' >> $xmlout";
	566	$cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
	567	$cmd .= " && echo '</make>' >> $xmlout";
	568	$cmd .= " && echo 'done'";
	569
	570	#make install
	571	$cmd .= " && echo 'make install $gsdl: '";
	572	$cmd .= " && echo '<make-install>' >> $xmlout";
	573	$cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
	574	$cmd .= " && echo '</make-install>' >> $xmlout";
	575	$cmd .= " && echo 'done'";
	576
	577	$cmd .= "\""; # close off cmd to bash and run it
[27668]	578	$status = system $cmd;
[27724]	579
[27962]	580	if(!$isMac) { # Linux
[28661]	581	# Moving imagemagick after instead of before compilation, since bin/darwin and bin/linux gets overwritten during compilation
	582	move("$greenstone_home$sep"."ext/imagemagick/$os", "$greenstone_home$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
	583	unlink "$greenstone_home$sep"."ext$sep$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
	584	&File::Path::remove_tree("$greenstone_home$sep"."ext$sep"."imagemagick"); # the untarred parent folder
[27962]	585	}
[27724]	586
[27621]	587	}
[27668]	588
[28661]	589	if($status != 0) {
	590	print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
	591	exit -1;
	592	}
	593
	594	&getIsisGdl("$greenstone_home"); #&getIsisGdl("$ENV{'DATA_DIR'}/$gsdl");
[27764]	595
[27621]	596	# set the path to the greenstone_home variable
[28661]	597	#$greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
[27764]	598	}
[27621]	599
[27764]	600	sub getPDFBox
	601	{
	602	# current revision is 27763, but using "head" works
	603	my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
	604	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
	605
	606	# now get the PDFBox extension for PDFBox tutorial
	607	print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
	608
	609	chdir($greenstone_home);
	610	my $cmd = "";
	611	if ($isWin) {
	612	$cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
	613
	614	} elsif ($isMac) { # need to use curl not wget
	615	$cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
	616	}
	617	else { # linux
[27765]	618	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
[27764]	619	}
	620	my $status = system $cmd;
	621	if($status != 0) {
	622	print STDERR "@@@ Failed to set up PDFBox\n";
[27765]	623	exit -1; # or proceed to testing other tutorials?
[27764]	624	}
[27621]	625	}
	626
[28615]	627	sub getOpenOfficeExt
	628	{
	629	# current revision is 27763, but using "head" works
	630	my $OOEXT_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/open-office/trunk/open-office-java"; # both for .zip and .tar.gz extension
	631	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/open-office/trunk/open-office-java";
	632
	633	# now get the OpenOffice extension for the AllDocTypes collection
	634	print STDERR "Getting open office ext from $OOEXT_TRAC_URL:\n";
	635
	636	chdir($greenstone_home);
	637	my $cmd = "";
	638	if ($isWin) {
	639	$cmd = "setup.bat && cd ext && wget $OOEXT_TRAC_URL.zip && unzip open-office-java.zip";
	640
	641	} elsif ($isMac) { # need to use curl not wget
	642	$cmd = "cd ext && curl $OOEXT_TRAC_URL.tar.gz > open-office-java.tar.gz && tar -xzf open-office-java.tar.gz";
	643	}
	644	else { # linux
	645	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $OOEXT_TRAC_URL.tar.gz && tar -xzf open-office-java.tar.gz\"";
	646	}
	647	my $status = system $cmd;
	648	if($status != 0) {
	649	print STDERR "@@@ Failed to set up the Open Office Extension\n";
	650	exit -1; # or proceed to testing other tutorials?
	651	}
	652	}
	653
[28605]	654	sub getIsisGdl {
[28606]	655	my $gsfolder = shift(@_);
[28605]	656
[28606]	657	if(!$isWin) {
[28661]	658	chdir($greenstone_home);
[28605]	659	my $bit_arch=`uname -m`;
	660	if ($bit_arch =~ m/64$/) {
[28606]	661	my $cmd = "";
	662	if($isMac) {
	663	$cmd = "cd $gsfolder/bin/darwin && curl http://www.greenstone.org/caveat-emptor/IsisGdl.macleopard > IsisGdl && chmod u+x IsisGdl";
	664	} else { # linux
	665	$cmd = "cd $gsfolder/bin/linux && wget http://www.greenstone.org/caveat-emptor/IsisGdl.bin32 && mv IsisGdl.bin32 IsisGdl && chmod u+x IsisGdl";
	666	}
[28605]	667	my $isis_status = system $cmd;
	668	if($isis_status != 0) {
[28606]	669	print STDERR "Unable to get IsisGdl from caveat page\n";
[28605]	670	}
	671	}
	672	}
	673	}
	674
[27621]	675	# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
	676	sub run_test
[28661]	677	{
	678	my $collect_parent = $greenstone_home;
	679	my $model_collect = "model-collect";
	680	my $build_options = "";
	681	if($install_version eq "3") {
	682	$collect_parent = &filename_concat($greenstone3_home,"web","sites","localsite");
	683	$model_collect = "gs3-model-collect";
	684	$build_options = " -site localsite ";
	685	chdir($greenstone3_home);
	686	} else {
	687	chdir($greenstone_home);
	688	}
	689
[28114]	690	my $num_cols = scalar(@collections); # remember the empty case
	691
	692	if($num_cols == 0) { # deal with all collections
	693	push (@collections, "");
[28615]	694	# putting the empty string in the array so that the "all collections" case
	695	# can be handled similar to how the case of user-specified collections is handled
[28114]	696
	697	} else { # deal with user specified set of collections
	698	# prefix the directory separator to each collection name
	699	@collections = map { $sep.$_ } @collections;
	700	}
	701
[27764]	702	my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
	703	if(!-d $pdfbox) {
	704	&getPDFBox();
	705	}
	706
[28615]	707	my $openofficeext = &filename_concat($greenstone_home, "ext", "open-office");
	708	if(!-d $openofficeext) {
	709	&getOpenOfficeExt();
	710	}
	711
	712	#&getIsisGdl("$greenstone_home");
[28605]	713
[27668]	714	open (my $xml_fh, '>'.$xmlout) \|\| die "Could not open xml file $xmlout for appending: $!\n";
[27621]	715
	716	# perform the requested subcommands, outputting xml information
	717	print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
[28072]	718
[28114]	719	my ($cmd, $status);
[27724]	720	# make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
[28072]	721
	722	if(!$use_local_rebuild) {
[28661]	723	print $xml_fh "Updating $ENV{'TASK_HOME'}/$model_collect:\n";
[28114]	724	for my $col (@collections) {
[28661]	725	$cmd = "svn up $ENV{'TASK_HOME'}/$model_collect$col"; #chdir("$ENV{'TASK_HOME'}/$model_collect");
[28114]	726	$status = system "$cmd";
	727	}
[28072]	728	}
[27724]	729
[28661]	730	# go to whichever collecthome parent we're using
	731	chdir($collect_parent);
[27621]	732
	733	# get svn info
[27629]	734	print STDERR "getting svn info: $xmlout\n";
[27621]	735	print $xml_fh "<svn-info>\n";
[27668]	736	&run_and_print_cmd("svn info", $xml_fh);
[27621]	737	print $xml_fh "</svn-info>\n";
[27629]	738	print STDERR "done\n";
[27621]	739
[28114]	740	if(!$use_local_rebuild) {
	741
	742	#make two copies of the model-collect directory in gsdl
	743	#one to be rebuilt and one as the basis for comparison
	744	#strip both of all .svn directories
	745
	746	#copy the model collections to the collect folder to be rebuilt
	747	print STDERR "installing test collections and model collections to new $gsdl installation... ";
[28072]	748	#clean up
	749	if(-d "collect") {
[28114]	750	for my $col (@collections) {
[28172]	751	if(-d "collect$col") {
	752	&File::Path::remove_tree("collect$col") \|\| die "Error could not delete collect: $!";
	753	}
[28114]	754	}
[28072]	755	}
[27621]	756
[28072]	757	if($use_static_model) {
[28114]	758	for my $col (@collections) {
	759	#copy to collect and strip .svn subfolders
	760	&File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
	761	&copy_recursively("model-collect$col", "collect$col", ".svn");
	762	}
[27621]	763
[28072]	764	} else { # the default situation: where we check out the model-collect from svn
[28114]	765	for my $col (@collections) {
	766	&File::Path::remove_tree("model-collect$col");
	767
	768	#copy to collect and strip .svn subfolders
	769	&File::Path::make_path("collect$col"); # create the folder and copy contents across
[28661]	770	&copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","$model_collect$col"), "collect$col", ".svn");
[28114]	771
	772	#make the model copy
	773	&File::Path::make_path("model-collect$col");
	774	&copy_recursively("collect$col", "model-collect$col"); # copy contents across
	775	}
[28072]	776	}
	777
	778	print STDERR "done\n";
	779	}
	780
[28114]	781	#for each collection, import, build and diff with its model counterpart
	782
	783	# if working with all collections, read the list of collections from the folders in collect
	784	if($num_cols == 0) {
	785	@collections = (); # get rid of the empty string put in the array to represent "all collections"
	786
[28661]	787	opendir my($collect_handle), "collect" or die "Could not open dir $collect_parent/collect: $!";
[28114]	788
	789	for my $collection (readdir $collect_handle) {
	790	next if ($collection eq "." \|\| $collection eq "..");
	791	next if ($collection eq "modelcol");
	792	push(@collections, $collection);
	793	}
	794	closedir $collect_handle; # close handle to collect dir
	795	}
	796
	797	for my $collection (@collections) {
[27621]	798
[28114]	799	# next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
	800	# next if ($collection !~ m/OAI\|METS\|DSpace\|MGPP\|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
[29449]	801	# next if ($collection !~ m/PDFBox/); ## TEMPORARY, FOR TESTING THIS SCRIPT
[28114]	802
[27621]	803	#escape the filename (in case of space)
	804	$collection =~ s@ @\\ @g;
	805	#getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
[28114]	806
	807	$collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
	808
	809	if (! -d "collect$sep$collection") {
	810	print STDERR "Collection $collection does not exist\n";
	811	next;
	812	}
	813
[27621]	814	print STDERR "*** Found collection $collection\n";
	815	print $xml_fh "<collection-test name=\"$collection\">\n";
	816
[28661]	817	# run the building scripts from the toplevel of the GS installation
	818	if($install_version eq "3") {
	819	chdir($greenstone3_home);
	820	} else {
	821	chdir($greenstone_home);
	822	}
	823
[28072]	824	if(!$use_local_rebuild) {
	825	#import
	826	# Ensure the OIDtype for importing is hash_on_full_filename
	827	# "to make document identifiers more stable across upgrades of the software,
	828	# although it means that duplicate documents contained in the collection are
	829	# no longer detected automatically."
	830	print STDERR "$collection - Importing:\n";
	831	print $xml_fh "<import>\n";
[28661]	832	&run_build_script("import.pl $build_options -removeold $collection"); #-OIDtype hash_on_full_filename
[28072]	833	print $xml_fh "</import>\n";
	834	print STDERR "done\n";
[27621]	835
[28072]	836	#build
	837	print STDERR "$collection - Building:\n";
	838	print $xml_fh "<build>\n";
[28661]	839	&run_build_script("buildcol.pl $build_options -removeold $collection");
[28072]	840	print $xml_fh "</build>\n";
	841	print STDERR "done\n";
	842
	843	#rename the intermediate 'building' directory 'index'
	844	print STDERR "$collection - Move \"building\" to \"index\"... ";
[28661]	845	my $index = &filename_concat($collect_parent, "collect", $collection, "index");
	846	my $building = &filename_concat($collect_parent, "collect", $collection, "building");
[28072]	847	&File::Path::remove_tree($index);
	848	# Renaming Directories, http://www.perlmonks.org/?node_id=177421
	849	move($building, $index) or die "copy failed: $!"; # File::Copy::move
	850	print STDERR "done\n";
	851	}
[27621]	852	#diffcol
[27629]	853	print STDERR "$collection - Diffing:\n";
[27621]	854	my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
[28172]	855
[28661]	856	# chdir($collect_parent); # this is actually where we are
[28172]	857	# help diffcol to know on what os the model cols were generated
	858	# and what os this test machine is (on which the test cols will be generated)
	859	$cmd = "diffcol.pl -testos $test_os -modelos $model_os -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
[27621]	860	&run_diff_script($cmd, $xml_fh, $diffcol_dir);
	861
[28661]	862	if($install_version eq "3") {
	863	chdir($greenstone3_home); # this is actually where we are
	864	} else {
	865	chdir($greenstone_home); # this is actually where we are
	866	}
[27629]	867	print STDERR "done\n";
[27621]	868	print $xml_fh "</collection-test>\n";
	869	}
	870
	871	print $xml_fh "</test>\n";
	872	close($xml_fh);
	873
[27629]	874	print STDERR "done\n";
[27621]	875	}
	876
	877	##***************************************************************
	878	# runs setup in greenstone_home before running the diff command
	879	sub run_diff_script {
	880	my ($cmd, $fh, $diffcol_dir) = @_;
	881
[28661]	882	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
	883
[27621]	884	# we're in greenstone_home now
	885	if(!$isWin) {
[28661]	886	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && cd $diffcol_dir && ./$cmd\"";
[27621]	887
	888	} else { # Need to prefix cmd -c/-k as necessary
[28661]	889	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
[27668]	890	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	891	}
	892
	893	return &run_and_print_cmd($cmd, $fh);
	894	}
	895
	896	# runs setup in greenstone_home before running the given build command
	897	sub run_build_script {
	898	my ($cmd, $fh) = @_;
[28661]	899	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
[27621]	900
	901	# chdir($greenstone_home);
[27687]	902	# we are in $greenstone_home already, can directly run the build cmd on the collection
[27621]	903	if(!$isWin) {
[28661]	904	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && $cmd\"";
[29493]	905	#$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& export PERL_HASH_SEED_DEBUG=1&& source $setup_script.$linux_ext && $cmd\"";
[27621]	906
	907	} else { # Need to prefix cmd -c/-k as necessary
[28661]	908	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
[27621]	909	}
[27687]	910	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	911
[27668]	912	return system($cmd);
[27629]	913	#return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
[27621]	914	}
	915
	916
	917	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
	918	# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
	919	sub run_and_print_cmd {
	920	my ($cmd, $fh) = @_;
	921
	922	open my $pin, "$cmd\|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-\|', 'powercfg -l') or die $!;
	923
	924	if(defined $fh) { # print cmd output both to the filehandle and to stdout
	925	while (my $line = <$pin>) {
	926	print $fh $line;
	927	# print STDOUT $line; # if also printing cmd output to STDOUT
	928	}
	929	}
	930	else { # no filehandle, so just need to print to stdout
	931
	932	# unlike backticks operator, system() will print the output of the command to the screen as it executes
	933	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	934
	935	my $status = system $cmd;
	936	if($status != 0) {
	937	print STDERR "ERROR ($status) running $cmd: $!\n";
	938	}
	939	}
	940	close($pin);
	941	}
	942
	943	sub filename_concat {
	944	my $first_file = shift(@_);
	945	my (@filenames) = @_;
	946
	947	# If first_file is not null or empty, then add it back into the list
	948	if (defined $first_file && $first_file =~ /\S/)
	949	{
	950	unshift(@filenames, $first_file);
	951	}
	952
	953	my $filename = join($sep, @filenames);
	954	$filename =~ s/[\\\/]$//; # remove trailing slashes if any
	955	return $filename;
	956	}
	957
	958
	959	# The following code is from
	960	# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
	961	# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
	962	sub copy_recursively {
	963	my ($from_dir, $to_dir, $regex) = @_;
	964	opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
	965
	966	# if(-d !$to_dir) {
	967	# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
	968	# }
	969
	970	for my $entry (readdir $dh) {
	971	next if ($entry eq "." \|\| $entry eq "..");
	972	next if (defined $regex && $entry =~ /$regex/);
	973	my $source = "$from_dir/$entry";
	974	my $destination = "$to_dir/$entry";
	975	if (-d $source) {
	976	mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
	977	copy_recursively($source, $destination, $regex);
	978	} else {
	979	copy($source, $destination) or die "copy failed: $!";
	980	}
	981	}
	982	closedir $dh;
	983	return;
	984	}
	985
	986	sub summarise {
	987
	988	# make a summarised Xml report
[27629]	989	print STDERR "Summarizing the xml report... ";
[28661]	990	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml";
[27621]	991	my $status = system($cmd);
[27629]	992	print STDERR "done\n";
[27621]	993
	994	# make a summarised HTMl report
[27629]	995	print STDERR "Creating an html summary report... ";
[28661]	996	$cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.html";
[29493]	997
	998	print STDERR "@@@ Running summarise command: $cmd\n\n";
[27621]	999	$status = system($cmd);
[29493]	1000
[27629]	1001	print STDERR "done\n";
[28005]	1002
	1003	# Print whether the tests passed or failed
	1004	print STDERR "*******************************************\n";
	1005	print STDERR "Checking if successful... \n";
	1006	$cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1007	$status = `$cmd`; #$status = system($cmd);
	1008	print STDERR "result: $status\n";
	1009	print STDERR "*******************************************\n";
[27621]	1010	}
	1011
	1012	sub upload {
[27657]	1013	# if the upload dir already existed, clear it of contents
	1014	if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
	1015	# don't want to keep previous days reports
	1016	# else we will have to manually clear them at some point
	1017	# just generate the set of reports for this run of task.pl upload
	1018	# and
	1019	&File::Path::remove_tree($ENV{'UPLOAD_DIR'});
	1020	}
	1021	# recreate the upload directory
	1022	&File::Path::make_path($ENV{'UPLOAD_DIR'});
[27621]	1023
	1024	# copy all .xml and .html files across to UPLOAD_DIR
	1025	opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
	1026	for my $entry (readdir $dh) {
[27687]	1027	next if ($entry !~ m/(\.xml\|\.html?)$/);
[28661]	1028	next if ($entry !~ m/(report-$install_version)/);
[27687]	1029
	1030	# copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
	1031	# html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
[27694]	1032	my $os_entry = $entry;
	1033	$os_entry =~ s@\[email protected]@ if $isWin;
[28571]	1034	if($isMac) {
[28634]	1035	$osversion = "Leopard-" if ($osversion eq "");
[28572]	1036	$os_entry = "diffcol-".$^O."-".$osversion."$os_entry"; # darwin-Lion for Lion/Mountain Lion
[28571]	1037	} else {
	1038	$os_entry = "diffcol-".$^O."-$os_entry";
	1039	}
[27687]	1040
[28621]	1041	# if the test failed, prefix "failed" to the report so that it shows up with an error icon on the caveat page
	1042	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1043	my $result = `$cmd`;
	1044	if($result ne "yes") {
	1045	$os_entry =~ s/diffcol-/diffcol-FAIL-/;
	1046	}
	1047
[27687]	1048	# get the absolute path to the original files before copying them over
	1049	$entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
	1050
	1051	# copy them over with their new names
	1052	## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
	1053	copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
[27621]	1054	}
	1055	closedir $dh;
	1056
[27637]	1057
	1058	# Upload the html file to puka
	1059	#default identity dir
	1060	if ( ! exists $ENV{'IDENTITY_DIR'} ) {
[27725]	1061	$ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
[27637]	1062	}
	1063	if (! exists $ENV{'SNAPSHOT_MODE'} ) {
	1064	$ENV{'SNAPSHOT_MODE'} = "caveat";
	1065	}
	1066
[27678]	1067	#use the correct key for uploading
[27637]	1068	$ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
	1069	if(-f $ENV{'IDENTITY_FILE'}) {
[27678]	1070	# if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
	1071
[28661]	1072	# the report we want to upload is actually just os-diffcol-report-$install_version-$dateid.html
[27687]	1073	my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c .htm \| "; #&& cat .html \| "; # && tar -c \|
[27637]	1074	$command .= ($^O eq "MSWin32" ? "plink" : "ssh");
	1075	$command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
	1076	#print "$command\n";
	1077	my $status = system("$command");
	1078	if($status != 0) {
[27678]	1079	print STDERR "*** Failed to upload test report to nzdl $status\n";
[27637]	1080	}
	1081	} else {
	1082	print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
	1083	}
	1084
[27629]	1085	print STDERR "Finished uploading\n";
[27621]	1086	}
	1087
	1088	# Sending emails with perl: http://learn.perl.org/examples/email.html
	1089	# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
[27628]	1090	# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
	1091	sub mail_with_report_attached
	1092	{
	1093	# email out with report attached, if the tests failed
[27629]	1094	print STDERR "Checking if successful... \n";
[27628]	1095	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1096	#my $result = system($cmd);
	1097	my $result = `$cmd`;
[27621]	1098
[27629]	1099	print STDERR "result: $result\n";
[27621]	1100
[27628]	1101	if($result ne "yes") {
	1102	my $msg = "$gsdl regression test for $dateid failed";
	1103	my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
[28661]	1104	my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$install_version-$dateid.html");
[27628]	1105
[27687]	1106	if($isWin) {
	1107	if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
[27678]	1108	# http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
	1109	#blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
[27628]	1110
[27678]	1111	# need to install blat on windows
	1112	$cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
	1113	$result = system($cmd);
	1114	}
	1115	else {
[27687]	1116	$result = 1; # status from running mail command is 0 if success, 1 if fail
[27678]	1117	print STDERR "********************************************\n";
[27687]	1118	if ($use_blat) {
	1119	print STDERR "Need blat and SMTP set to send mail attachment\n" ;
	1120	} else {
	1121	print STDERR "Not set up to send mail on Windows\n";
	1122	}
[27678]	1123	print STDERR "Inspect report at: $attach_file\n";
	1124	print STDERR "********************************************\n";
	1125	}
	1126	} else { # linux
[27628]	1127	my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
	1128
	1129	if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
	1130	$cmd="echo '$gsdl regression test for $dateid failed.' \| mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
	1131
	1132	print STDERR "********************************************\n";
	1133	print STDERR "No mutt installed, unable to mail attachment\n";
	1134	print STDERR "Inspect report at: $attach_file\n";
	1135	print STDERR "********************************************\n";
	1136	} else {
	1137	#$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
	1138	$cmd = "echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
	1139	}
[27678]	1140
	1141	# run the mail command
	1142	$result = system($cmd); #&run_and_print_cmd($cmd);
[27628]	1143	}
	1144
[27678]	1145
[27637]	1146	if($result != 0) {
	1147	print STDERR "*** Unable to send email: $?\n";
	1148	}
	1149	else {
	1150	print STDERR "Sent mail with report attached.\n";
	1151	}
[27628]	1152	} else {
	1153	print STDERR "********************************************\n";
[27629]	1154	print STDERR "Tests were successful. Not sending mail.\n";
[27628]	1155	print STDERR "********************************************\n";
	1156	}
	1157	}
	1158
[27621]	1159	# The old version of this program contained the following, consisting of 1 line of active code:
	1160
	1161	# Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
	1162	# Doing so will call this pl file and pass in "summarise" in ARGV
	1163	# This pl file will in turn call the task executable in this folder
	1164	# passing in "summarise" as a parameter.
	1165	#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
	1166
	1167	##system("/bin/bash -c \"./task @ARGV\"");
	1168	##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
[27694]	1169

Note: See TracBrowser for help on using the repository browser.

Download in other formats: