Context Navigation

source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 29504

Last change on this file since 29504 was 29496, checked in by ak19, 10 years ago
Forgot to update usage
Property svn:executable set to ``*
File size: 43.0 KB

Rev	Line
[27621]	1	#!/usr/bin/perl -w
[27556]	2
[27621]	3	# This program is meant to run the nightly diffcol
	4	# It is meant to be an equivalent for the existing task bash script
	5	# But it is intended to be expanded to work for Windows and GS3
	6	# For windows, need to REMEMBER to set the correct shebangs at the top
[27556]	7
[27663]	8
	9	# TODO:
	10	# Have a caveat mode and a stable mode (as in snapshot/task.pl)
	11	#
	12	#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat\|stable)" ) {
	13	# $major_version = 2;
	14	# $prefix="2t";
	15	# $rk="tk2"; # test kit
	16	#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat\|stable)" ) {
	17	# $major_version = 3;
	18	# $prefix="3t";
	19	# $rk="tk3"; # test kit
	20
[27628]	21	package diffcoltask;
[27556]	22
[27621]	23	use Cwd;
[29398]	24	#use Switch; # for switch(val) { case: ; ...}
[27621]	25	use File::Path; # for rmdir and mkdir type functions
	26	use File::Copy; # for recursive copying of folders but skipping .svn
	27	use File::Basename;
	28
[27628]	29	use strict;
	30	no strict 'subs'; # allow barewords (eg STDERR) as function arguments
[27621]	31
[27764]	32
[27621]	33	my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
[27764]	34	my $isMac = ($^O =~ m/macos\|darwin/i) ? 1 : 0;
[28571]	35
	36	my $osversion="";
	37	# Need to get the correct gnome-lib-minimal for the OS
	38	# darwin11* Lion, darwin12* Mountain Lion, darwin9* and darwin10* are Leopard and Snow Leopard
	39	if ($^O eq "darwin") {
	40	$osversion=`uname -r`; # e.g. 12.x.x
	41	#$osversion =~ s@\..*$@@; # e.g.12
[28634]	42	$osversion = ($osversion =~ m@^1[1-9](\.)?@i) ? "Lion-" : "";
[28571]	43	}
	44
	45
[27621]	46	my $sep = $isWin ? "\\" : "/";
	47	my $pathsep = $isWin ? ";" : ":";
	48	#my $script_ext = $isWin ? ".bat" : ".bash";
[28661]	49	my $setup_script = "setup"; # will become gs3-setup for GS3
[27687]	50	my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
[27621]	51
[28616]	52	my $install_type = "svn";
	53	my $install_version = "2";
	54
[28106]	55	my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
[28072]	56	my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
[27701]	57
[28072]	58	# if use_local_rebuild is on, use_static_model should be on
	59	if ($use_local_rebuild && !$use_static_model) {
	60	$use_static_model = 1;
	61	}
	62
[28172]	63	my $test_os = $isWin ? "windows" : ($isMac ? "darwin" : "linux");
	64	my $model_os = "linux"; # default
[28072]	65
[29494]	66	my $debugging = 0;
	67
[27621]	68	# TASK_HOME should be the toplevel diffcol folder
	69	$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
[27701]	70	if($isWin) {
	71	$ENV{'TASK_HOME'} =~ s@\/@\\@g;
	72	# need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
	73	# run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
	74	# At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
	75	# end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
	76	require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
	77	$ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
	78	}
[27621]	79	## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
	80
	81
[27678]	82	$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
	83
	84	# we'll be using BLAT to send mail attachments on Windows
[27687]	85	my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
	86	if($isWin && $use_blat && ! -e $blat) {
	87	print STDERR "\n***********************************\n";
	88	print STDERR "No blat.exe found in $blat.\n";
	89	print STDERR "Blat needed to send mail with attachments on Windows.\n";
	90	print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
	91	print STDERR "for your bit architecture and name the folder 'blat'\n";
	92	print STDERR "***********************************\n\n";
[27678]	93	$blat = 0;
	94	}
	95
	96
[27668]	97	$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
	98	$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
[27621]	99	$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
[27678]	100	$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
[27621]	101	##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
	102
[29448]	103	# When installing diffcol the first time, force the user to create the data-dir and upload-dir
	104	die "\n@@@@ data dir: ".$ENV{'DATA_DIR'}." does not exist\n" unless (-d $ENV{'DATA_DIR'});
	105	die "\n@@@@ reports/upload dir: ".$ENV{'UPLOAD_DIR'}." does not exist\n" unless (-d $ENV{'UPLOAD_DIR'});
	106
[27621]	107	# control if an existing compiled greenstone is used
	108	# or, if one should be checked out, which revision to checkout from svn
	109	$ENV{'SVN_OPT_REV'} = "-r head";
[27656]	110	#$ENV{'GSDLHOME'}=
	111	#$ENV{'GSDL3SRCHOME'}=
[27621]	112
[27628]	113
[27677]	114	# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
	115	my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
[27628]	116
	117	#parse arguments
	118	my $action = "all";
[28114]	119	my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
[28620]	120	my @collections = (); # list of collections that run_test should process
[28114]	121
[28620]	122	if(scalar(@ARGV) == 0) {
	123	$action="all";
	124	}
	125
	126	# process any arguments that are --gs2\|--gs3 and --bin\|--svn, and delete them from the array
	127	# if none provided, it's gs2 and svn by default.
	128	for (my $i = $#ARGV; $i >= 0; --$i) {
	129	if($ARGV[$i] =~ m/--(bin\|svn)/) {
	130	$install_type = $1;
	131	splice @ARGV, $i, 1; # remove the element from the argument array
	132	} elsif($ARGV[$i] =~ m/--gs(2\|3)/) {
	133	$install_version = $1;
[28661]	134	$setup_script = $install_version eq "3" ? "gs3-setup" : "setup"; # needs to become gs3-setup for GS3
[28620]	135	splice @ARGV, $i, 1; # remove the element from the argument array
	136	}
	137	}
	138
	139
[28172]	140	# run_test can take any number of args
[28620]	141	if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
[28172]	142	print STDERR "**** Wrong number of arguments\n";
[28114]	143	&printusage();
	144	exit -1;
[27628]	145	}
	146
[28620]	147	if(scalar(@ARGV) > 0) {
[29398]	148	#switch ($ARGV[0]) {
	149	#case qr/^(-h\|--?help\|help)$/i { &printusage; exit 0; }
	150	#case qr/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/ { $action=$ARGV[0]; }
	151	#else {
	152	#print STDERR "**** Bad subcommand.\n";
	153	#&printusage;
	154	#exit -1;
	155	#}
	156	#}
	157
	158
	159	my $switch = $ARGV[0];
	160	if($switch =~ m/^(-h\|--?help\|help)$/i) {
	161	&printusage;
	162	exit 0;
	163	} elsif ($switch =~ m/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/) {
	164	$action=$ARGV[0];
	165	} else {
	166	print STDERR "**** Bad subcommand.\n";
	167	&printusage;
	168	exit -1;
[27628]	169	}
[28616]	170
[28172]	171	# run_test action can take a subaction: nosvn\|justdiff. It can also take --modelOS (windows\|linux\|darwin)
[28114]	172	# nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
	173	# it out from the svn model-collect again.
	174	# justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
[28616]	175
[28620]	176	if($action eq "run_test" && scalar(@ARGV) >= 2) {
[28114]	177	push(@collections, @ARGV);
	178	shift @collections; # remove action from array
	179
[28172]	180	for (my $i=0; $i < scalar(@ARGV); $i++) {
	181	if($ARGV[$i] =~ m@^--@) {
	182	shift @collections; # remove subaction/flag from array
[28114]	183
[28172]	184	$subaction = $ARGV[$i];
[28114]	185	if($subaction eq "--justdiff") {
[28172]	186	$use_local_rebuild = $use_static_model = 1;
[28114]	187	} elsif ($subaction eq "--nosvn") {
[28172]	188	$use_static_model = 1;
	189	#} elsif ($subaction =~ m/\-\-testOS/i && defined $ARGV[$i+1]) {
	190	# $test_os = $ARGV[$i+1];
	191	# $i++;
	192	# shift @collections; # remove test_os value from array
	193	} elsif ($subaction =~ m/\-\-modelOS/i && defined $ARGV[$i+1] && $ARGV[$i+1] =~ m/windows\|linux\|darwin/i) {
	194	$model_os = $ARGV[$i+1];
	195	$i++;
	196	shift @collections; # remove model_os value from array
	197	#print STDERR "Model_os specified: $model_os\n";
[29494]	198	} elsif ($subaction =~ m/\-\-debug/i) {
	199	$debugging = 1;
[29496]	200	print STDERR "*** DEBUG MODE: debug output files will be in toplevel folder\n";
[28114]	201	} else {
[28172]	202	print STDERR "**** Bad subaction/value: ".$ARGV[$i]."\n";
	203	&printusage;
	204	exit -1;
	205	}
	206	}
[28114]	207	}
	208
	209	# foreach my $col (@collections) {
	210	# print STDERR "Collection: $col\n";
	211	# }
	212	}
[27628]	213	}
	214
[28661]	215	print STDERR "Install type $install_type\n";
	216	print STDERR "Install version $install_version\n";
[28114]	217
[27621]	218	#check key environment vars are set
	219	if(!defined $ENV{'UPLOAD_DIR'}) {
[27629]	220	print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
[27621]	221	#return 1;
	222	}
	223	if(!defined $ENV{'DATA_DIR'}) {
[27629]	224	print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
[27621]	225	#return 1;
	226	}
	227	if(!defined $ENV{'MONITOR_EMAIL'}) {
[27629]	228	print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
[27621]	229	#return 1;
	230	}
	231
	232	if($ENV{'DATA_DIR'} eq "/") {
[27629]	233	print STDERR "DATA_DIR should not be the fs root\n";
[27621]	234	#return 1;
	235	}
	236
[27629]	237	print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
	238	print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
[27621]	239
	240	#create an id for this test
	241	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	242	$year += 1900;
	243	$mon += 1;
	244	$mon = "0$mon" if ($mon < 10);
	245	$mday = "0$mday" if ($mday < 10);
[27702]	246	my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
[27621]	247
[27629]	248	print STDERR "Starting test '$dateid'\n";
[27621]	249
	250
	251	# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
	252	$ENV{'CLASSPATH'} = "";
	253	my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
	254	my @files = <$jar_lib_path/.jar>; # /full/path/to/diffcol/lib/jar
[27628]	255	foreach my $file (@files) {
[27668]	256	$file =~ s@\/@\\@g if $isWin;
[27621]	257	$ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
	258	}
	259	##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
	260
	261
	262	#set the location of the full report
[28661]	263	my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$install_version-$dateid.xml");
[27621]	264	##print STDERR "XML: $xmlout\n";
	265
	266	# the toplevel folder of the greenstone installation being used
[28661]	267	my $greenstone3_home="";
	268	my $greenstone_home=""; # gs2build for gs3, toplevel install folder for gs2
[27621]	269	# gsdl is the checkout folder and can be greenstone2 or greenstone3
	270	my $gsdl="greenstone2";
	271
	272	# Check if using existing compiled-up greenstone installation
	273	# and set the greenstone_home location accordingly
	274
	275	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27629]	276	print STDERR "Found existing Greenstone home, will use that instead\n";
[28661]	277	$greenstone_home=$ENV{'GSDLHOME'} if defined $ENV{'GSDLHOME'};
	278
	279	if(defined $ENV{'GSDL3SRCHOME'}) {
	280	print STDERR "*** GSDL3SRCHOME set, changing to using installed GS3\n";
	281	$gsdl = "greenstone3";
	282	$install_version = "3";
	283	$greenstone3_home=$ENV{'GSDL3SRCHOME'};
	284	$greenstone_home=filename_concat($greenstone3_home, "gs2build") unless defined $ENV{'GSDLHOME'};
	285	}
[27621]	286	} else {
[28661]	287	if($install_version eq "3") {
	288	$gsdl = "greenstone3";
	289	$greenstone3_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	290	$greenstone_home=filename_concat($greenstone3_home, "gs2build");
	291	} else {
	292	$greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	293	}
[27621]	294	}
	295	##print STDERR "GSHOME: $greenstone_home\n";
	296
[27628]	297	#do the requested action
	298	if($action eq "setup_greenstone") {
	299	&setup_greenstone;
	300	}
	301	elsif ($action eq "run_test") {
	302	&run_test;
	303	}
	304	elsif ($action eq "summarise") {
	305	&summarise;
	306	}
	307	elsif ($action eq "upload") {
	308	&upload;
	309	&mail_with_report_attached;
	310	}
	311	elsif ($action eq "all") {
	312	&setup_greenstone;
	313	&run_test;
	314	&summarise;
	315	&upload;
	316	&mail_with_report_attached;
	317	}
[27621]	318
	319	##********************************
	320
[27628]	321	sub printusage
	322	{
[28172]	323	# print STDERR "Run as: $0 (help\|setup_greenstone\|run_test <--modelOS windows\|darwin\|linux> <--justdiff\|--nosvn> <col1 col2 ...> \|summarise\|upload\|all)\n";
	324	print STDERR "Run as: $0 (help\|setup_greenstone\|run_test\|summarise\|upload\|all)\n";
	325	print STDERR "where run_test can further take the following optional parameters:\n";
	326	print STDERR "\t--modelOS (windows\|darwin\|linux)\n";
	327	print STDERR "\t--justdiff\|--nosvn\n";
[29496]	328	print STDERR "\t--debug\n";
[28172]	329	print STDERR "\t<col1 col2 ...>\n";
[28616]	330	print STDERR "where setup_greenstone can further take the following optional parameters:\n";
	331	print STDERR "\t--gs2\|--gs3\n";
	332	print STDERR "\t--svn\|--bin\n";
[27628]	333	}
[27621]	334
	335	#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
	336
	337	sub setup_greenstone
	338	{
[28661]	339	if($install_version eq "3" && !defined $ENV{'JAVA_HOME'}) {
	340	print "*** JAVA_HOME not set. Set it and add its bin folder to the PATH\n";
	341	exit 0;
	342	}
	343
[27621]	344	#clean up from previous tests
[27628]	345	print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
[27621]	346	for my $i ( 1..5 ) {
	347	sleep 1; # 1 second
[27628]	348	print STDERR ".";
[27621]	349	}
[27628]	350	print STDERR "\n";
[27621]	351
	352	# http://perldoc.perl.org/File/Path.html
[27629]	353	print STDERR "cleaning up previous tests\n";
[28661]	354	#&File::Path::remove_tree($ENV{'DATA_DIR'});
	355	if(-d $greenstone_home && $install_version eq "2") {
	356	&File::Path::remove_tree($greenstone_home);
	357	}
	358	if(-d $greenstone3_home && $install_version eq "3") {
	359	&File::Path::remove_tree($greenstone3_home);
	360	}
	361	unlink glob "$ENV{'DATA_DIR'}$sep"."report-$install_version";
	362	unlink "$ENV{'DATA_DIR'}$sep"."compilation-errors";
[27629]	363	print STDERR "creating the data dir\n";
[28661]	364	#&File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
[27621]	365
	366	chdir($ENV{'DATA_DIR'});
	367
	368	# use existing compiled-up greenstone installation, if a GSDLHOME set
	369	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27710]	370	print STDERR "Found existing Greenstone home, will use that instead\n";
[27621]	371	return;
	372	}
	373
	374	# Else checkout a GS from svn into DATA_DIR
[28661]	375	if($install_type eq "svn") {
	376	&checkout_gs_from_svn();
	377	##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
[27621]	378
[28661]	379	if($install_version eq "2") {
	380	&compile_gs2_svn();
	381	} else {
	382	&compile_gs3_svn();
	383	}
	384	}
	385
	386	}
	387
	388	sub checkout_gs_from_svn {
	389
[27621]	390	#svn checkout of main gsdl directory
[27629]	391	print STDERR "checkout $gsdl:\n";
[28661]	392	my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone$install_version $gsdl";
[27621]	393	##print STDERR "Checkout CMD: $cmd\n";
	394
	395	# # unlike backticks operator, system() will print the output of the command to the screen as it executes
	396	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	397	my $status = system "$cmd"; #my $status = `$cmd`;
[28661]	398	if($status != 0) {
	399	print STDERR "@@@ SVN checkout of $gsdl failed\n";
	400	exit -1;
	401	}
[27629]	402	print STDERR "done\n";
[28661]	403	}
[27621]	404
[28661]	405	sub getImageMagickBins {
	406	my ($cmd, $status);
	407	my $imagickzip = "";
[27621]	408
[28661]	409	chdir ("$greenstone_home");
	410	if(!$isWin) { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
[27710]	411
	412	my $bit_arch=`uname -m`;
[28661]	413
	414	# imagemagick binary
[27710]	415	print STDERR "Getting imagemagick binary\n";
[28661]	416
[27724]	417	my $os = $isMac ? "darwin" : "linux";
[28661]	418	$imagickzip = "imagemagick-$os";
[27668]	419
[27710]	420	if($isMac) {
[28661]	421	# $imagickzip .= "-10.5.tar.gz";
[27962]	422	# at present, only the Imagemagick binaries created by Max for darwin work on the Macs
[28661]	423	# &File::Path::make_path("$greenstone_home$sep"."bin"."$sep$os"); # need to ensure gsdl/bin/darwin exists
[27962]	424	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
	425	$status = system($cmd);
[28661]	426	if($status != 0) {
	427	print STDERR "@@@ Unable to get imagemagick for darwin\n";
	428	}
	429
	430	# need ghostscript mac binary too for pdf to img conversions on mac
[27963]	431	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
	432	$status = system($cmd);
[28661]	433	if($status != 0) {
	434	print STDERR "@@@ Unable to get ghostscript for darwin\n";
	435	}
	436
	437	# the imagemagick and ghostscript binaries have been set to executable on svn trac now
	438	# system("chmod -R u+x $greenstone_home/bin/darwin/imagemagick/bin/*");
	439	# system("chmod -R u+x $greenstone_home/bin/darwin/ghostscript/bin/*");
[27710]	440	} else { # linux
	441	my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
[27724]	442	$imagickzip .= "$extension64.tar.gz";
[27962]	443
	444	# now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
	445	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
	446	$status = system ($cmd);
	447	system("cd ext && tar -xvzf $imagickzip");
[27724]	448	}
[28661]	449	}
	450	return $imagickzip;
	451	}
[27710]	452
[28661]	453	sub getGnomeLibExt
	454	{
[29455]	455	if($isWin) {
	456	return; # no gnome-lib for windows
	457	}
[28661]	458	my ($cmd, $status);
	459	my $bit_arch=`uname -m`;
	460	my $os = $isMac ? "darwin" : "linux";
	461
[27710]	462	# gnomelib binary
[27629]	463	print STDERR "setting up gnome-lib-minimal for compilation\n";
[27621]	464
	465	# To get gnome-lib, need to determine bit architecture of the linux/darwin
	466	# http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
	467	# $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
	468	# well on ActivePerl.
[28621]	469	# But since we know we're on a Linux/Darwin machine at this point, we can just run `uname -m` and other linux cmds
[27621]	470
[28571]	471	# osversion will be "Lion" or ""
	472	# and assuming all darwin is intel, not ppc!!
[28572]	473	my $gnome_lib_file = $isMac ? "darwin-".$osversion."intel" : "linux";
[27621]	474
[28571]	475	$gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/ && !$isMac); # linux only case
[27621]	476
	477	#svn checkout gnome-lib for this linux/darwin
[28661]	478	chdir("$greenstone_home$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
[27621]	479
	480	##print STDERR "**** gnomelib: $gnome_lib_file\n";
	481
	482	# checkout and unpack gnome-lib-minimal
	483
	484	#svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
	485	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
	486	system $cmd;
	487	system ("tar -xvzf gl.tar.gz");
	488
	489	##print STDERR "*** ARCH: $bit_arch\n";
[28661]	490	}
[27621]	491
[28661]	492
	493
	494	sub compile_gs3_svn() {
	495	my ($cmd, $status);
	496	chdir ("$greenstone3_home");
	497
	498	$cmd = "ant"; # creates the build.properties file from the .in template
	499	$status = system $cmd;
	500	if($status != 0) {
	501	print STDERR "Unable to run the ant command: $status\n";
	502	exit -1;
	503	}
	504
	505	$cmd = "ant -Dproperties.ok=y -Dcheckout.gnomelib.ext=true prepare"; # pass in confirmation to ant prepare step
	506	$status = system $cmd;
	507	if($status != 0) {
	508	print STDERR "Failed to run $cmd command: $status\n";
	509	exit -1;
	510	}
	511
	512	my $imagickzip = &getImageMagickBins();
	513	chdir ("$greenstone3_home");
	514
	515	$cmd = "ant -Dcheckout.gnomelib.ext=true install"; # Compile with gnome-lib.
	516	$status = system $cmd;
	517	if($status != 0) {
	518	print STDERR "Failed to run $cmd command: $status\n";
	519	exit -1;
	520	}
	521	}
	522
	523	sub compile_gs2_svn() {
	524	my $imagickzip = &getImageMagickBins();
	525	&getGnomeLibExt();
	526
	527	my ($cmd, $status);
	528	my $os = $isMac ? "darwin" : "linux";
	529
	530	chdir("$greenstone_home"); #chdir("$ENV{'DATA_DIR'}$sep$gsdl"); # goes into toplevel gs2 or gs3 folder
	531
	532	##print STDERR "@@@ OS: $^O.\|".$Config{'archname64'}."\|\n";
	533
	534	if($isWin) {
	535	print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
	536
	537	# we're now in the GS2 folder, call makegs2 with silent param
	538	$cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
	539	$status = system $cmd;
	540	if($status != 0) {
	541	print STDERR "Greenstone compilation on Windows failed\n";
	542	exit -1;
	543	}
	544
	545	} else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
	546
	547	chdir("ext$sep"."gnome-lib-minimal");
	548
[27621]	549	# need to run source devel.bash on gnome-lib followed by configure, make, make install
	550	# in one go, in order to preserve the compile environment set up by sourcing devel.bash
	551
	552	# http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
	553	# http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
	554
	555	# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
	556	$cmd = "bash -c \"";
	557
	558	$cmd .= "source ./devel.bash";
	559	$cmd .= " && cd ../..";
	560
	561	#configure
	562	# $cmd .= " && ./configure";
	563	$cmd .= " && echo 'configure $gsdl: ' ";
	564	$cmd .= " && echo '<configure>' >> $xmlout";
	565	$cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
	566	$cmd .= " && echo '</configure>' >> $xmlout";
	567	$cmd .= " && echo 'done'";
	568
	569	#make
	570	$cmd .= " && echo 'make $gsdl: '";
	571	$cmd .= " && echo '<make>' >> $xmlout";
	572	$cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
	573	$cmd .= " && echo '</make>' >> $xmlout";
	574	$cmd .= " && echo 'done'";
	575
	576	#make install
	577	$cmd .= " && echo 'make install $gsdl: '";
	578	$cmd .= " && echo '<make-install>' >> $xmlout";
	579	$cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
	580	$cmd .= " && echo '</make-install>' >> $xmlout";
	581	$cmd .= " && echo 'done'";
	582
	583	$cmd .= "\""; # close off cmd to bash and run it
[27668]	584	$status = system $cmd;
[27724]	585
[27962]	586	if(!$isMac) { # Linux
[28661]	587	# Moving imagemagick after instead of before compilation, since bin/darwin and bin/linux gets overwritten during compilation
	588	move("$greenstone_home$sep"."ext/imagemagick/$os", "$greenstone_home$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
	589	unlink "$greenstone_home$sep"."ext$sep$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
	590	&File::Path::remove_tree("$greenstone_home$sep"."ext$sep"."imagemagick"); # the untarred parent folder
[27962]	591	}
[27724]	592
[27621]	593	}
[27668]	594
[28661]	595	if($status != 0) {
	596	print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
	597	exit -1;
	598	}
	599
	600	&getIsisGdl("$greenstone_home"); #&getIsisGdl("$ENV{'DATA_DIR'}/$gsdl");
[27764]	601
[27621]	602	# set the path to the greenstone_home variable
[28661]	603	#$greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
[27764]	604	}
[27621]	605
[27764]	606	sub getPDFBox
	607	{
	608	# current revision is 27763, but using "head" works
	609	my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
	610	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
	611
	612	# now get the PDFBox extension for PDFBox tutorial
	613	print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
	614
	615	chdir($greenstone_home);
	616	my $cmd = "";
	617	if ($isWin) {
	618	$cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
	619
	620	} elsif ($isMac) { # need to use curl not wget
	621	$cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
	622	}
	623	else { # linux
[27765]	624	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
[27764]	625	}
	626	my $status = system $cmd;
	627	if($status != 0) {
	628	print STDERR "@@@ Failed to set up PDFBox\n";
[27765]	629	exit -1; # or proceed to testing other tutorials?
[27764]	630	}
[27621]	631	}
	632
[28615]	633	sub getOpenOfficeExt
	634	{
	635	# current revision is 27763, but using "head" works
	636	my $OOEXT_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/open-office/trunk/open-office-java"; # both for .zip and .tar.gz extension
	637	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/open-office/trunk/open-office-java";
	638
	639	# now get the OpenOffice extension for the AllDocTypes collection
	640	print STDERR "Getting open office ext from $OOEXT_TRAC_URL:\n";
	641
	642	chdir($greenstone_home);
	643	my $cmd = "";
	644	if ($isWin) {
	645	$cmd = "setup.bat && cd ext && wget $OOEXT_TRAC_URL.zip && unzip open-office-java.zip";
	646
	647	} elsif ($isMac) { # need to use curl not wget
	648	$cmd = "cd ext && curl $OOEXT_TRAC_URL.tar.gz > open-office-java.tar.gz && tar -xzf open-office-java.tar.gz";
	649	}
	650	else { # linux
	651	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $OOEXT_TRAC_URL.tar.gz && tar -xzf open-office-java.tar.gz\"";
	652	}
	653	my $status = system $cmd;
	654	if($status != 0) {
	655	print STDERR "@@@ Failed to set up the Open Office Extension\n";
	656	exit -1; # or proceed to testing other tutorials?
	657	}
	658	}
	659
[28605]	660	sub getIsisGdl {
[28606]	661	my $gsfolder = shift(@_);
[28605]	662
[28606]	663	if(!$isWin) {
[28661]	664	chdir($greenstone_home);
[28605]	665	my $bit_arch=`uname -m`;
	666	if ($bit_arch =~ m/64$/) {
[28606]	667	my $cmd = "";
	668	if($isMac) {
	669	$cmd = "cd $gsfolder/bin/darwin && curl http://www.greenstone.org/caveat-emptor/IsisGdl.macleopard > IsisGdl && chmod u+x IsisGdl";
	670	} else { # linux
	671	$cmd = "cd $gsfolder/bin/linux && wget http://www.greenstone.org/caveat-emptor/IsisGdl.bin32 && mv IsisGdl.bin32 IsisGdl && chmod u+x IsisGdl";
	672	}
[28605]	673	my $isis_status = system $cmd;
	674	if($isis_status != 0) {
[28606]	675	print STDERR "Unable to get IsisGdl from caveat page\n";
[28605]	676	}
	677	}
	678	}
	679	}
	680
[27621]	681	# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
	682	sub run_test
[28661]	683	{
	684	my $collect_parent = $greenstone_home;
	685	my $model_collect = "model-collect";
	686	my $build_options = "";
	687	if($install_version eq "3") {
	688	$collect_parent = &filename_concat($greenstone3_home,"web","sites","localsite");
	689	$model_collect = "gs3-model-collect";
	690	$build_options = " -site localsite ";
	691	chdir($greenstone3_home);
	692	} else {
	693	chdir($greenstone_home);
	694	}
	695
[28114]	696	my $num_cols = scalar(@collections); # remember the empty case
	697
	698	if($num_cols == 0) { # deal with all collections
	699	push (@collections, "");
[28615]	700	# putting the empty string in the array so that the "all collections" case
	701	# can be handled similar to how the case of user-specified collections is handled
[28114]	702
	703	} else { # deal with user specified set of collections
	704	# prefix the directory separator to each collection name
	705	@collections = map { $sep.$_ } @collections;
	706	}
	707
[27764]	708	my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
	709	if(!-d $pdfbox) {
	710	&getPDFBox();
	711	}
	712
[28615]	713	my $openofficeext = &filename_concat($greenstone_home, "ext", "open-office");
	714	if(!-d $openofficeext) {
	715	&getOpenOfficeExt();
	716	}
	717
	718	#&getIsisGdl("$greenstone_home");
[28605]	719
[27668]	720	open (my $xml_fh, '>'.$xmlout) \|\| die "Could not open xml file $xmlout for appending: $!\n";
[27621]	721
	722	# perform the requested subcommands, outputting xml information
	723	print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
[28072]	724
[28114]	725	my ($cmd, $status);
[27724]	726	# make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
[28072]	727
	728	if(!$use_local_rebuild) {
[28661]	729	print $xml_fh "Updating $ENV{'TASK_HOME'}/$model_collect:\n";
[28114]	730	for my $col (@collections) {
[28661]	731	$cmd = "svn up $ENV{'TASK_HOME'}/$model_collect$col"; #chdir("$ENV{'TASK_HOME'}/$model_collect");
[28114]	732	$status = system "$cmd";
	733	}
[28072]	734	}
[27724]	735
[28661]	736	# go to whichever collecthome parent we're using
	737	chdir($collect_parent);
[27621]	738
	739	# get svn info
[27629]	740	print STDERR "getting svn info: $xmlout\n";
[27621]	741	print $xml_fh "<svn-info>\n";
[27668]	742	&run_and_print_cmd("svn info", $xml_fh);
[27621]	743	print $xml_fh "</svn-info>\n";
[27629]	744	print STDERR "done\n";
[27621]	745
[28114]	746	if(!$use_local_rebuild) {
	747
	748	#make two copies of the model-collect directory in gsdl
	749	#one to be rebuilt and one as the basis for comparison
	750	#strip both of all .svn directories
	751
	752	#copy the model collections to the collect folder to be rebuilt
	753	print STDERR "installing test collections and model collections to new $gsdl installation... ";
[28072]	754	#clean up
	755	if(-d "collect") {
[28114]	756	for my $col (@collections) {
[28172]	757	if(-d "collect$col") {
	758	&File::Path::remove_tree("collect$col") \|\| die "Error could not delete collect: $!";
	759	}
[28114]	760	}
[28072]	761	}
[27621]	762
[28072]	763	if($use_static_model) {
[28114]	764	for my $col (@collections) {
	765	#copy to collect and strip .svn subfolders
	766	&File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
	767	&copy_recursively("model-collect$col", "collect$col", ".svn");
	768	}
[27621]	769
[28072]	770	} else { # the default situation: where we check out the model-collect from svn
[28114]	771	for my $col (@collections) {
	772	&File::Path::remove_tree("model-collect$col");
	773
	774	#copy to collect and strip .svn subfolders
	775	&File::Path::make_path("collect$col"); # create the folder and copy contents across
[28661]	776	&copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","$model_collect$col"), "collect$col", ".svn");
[28114]	777
	778	#make the model copy
	779	&File::Path::make_path("model-collect$col");
	780	&copy_recursively("collect$col", "model-collect$col"); # copy contents across
	781	}
[28072]	782	}
	783
	784	print STDERR "done\n";
	785	}
	786
[28114]	787	#for each collection, import, build and diff with its model counterpart
	788
	789	# if working with all collections, read the list of collections from the folders in collect
	790	if($num_cols == 0) {
	791	@collections = (); # get rid of the empty string put in the array to represent "all collections"
	792
[28661]	793	opendir my($collect_handle), "collect" or die "Could not open dir $collect_parent/collect: $!";
[28114]	794
	795	for my $collection (readdir $collect_handle) {
	796	next if ($collection eq "." \|\| $collection eq "..");
	797	next if ($collection eq "modelcol");
	798	push(@collections, $collection);
	799	}
	800	closedir $collect_handle; # close handle to collect dir
	801	}
	802
	803	for my $collection (@collections) {
[27621]	804
[28114]	805	# next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
	806	# next if ($collection !~ m/OAI\|METS\|DSpace\|MGPP\|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
[29449]	807	# next if ($collection !~ m/PDFBox/); ## TEMPORARY, FOR TESTING THIS SCRIPT
[28114]	808
[27621]	809	#escape the filename (in case of space)
	810	$collection =~ s@ @\\ @g;
	811	#getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
[28114]	812
	813	$collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
	814
	815	if (! -d "collect$sep$collection") {
	816	print STDERR "Collection $collection does not exist\n";
	817	next;
	818	}
	819
[27621]	820	print STDERR "*** Found collection $collection\n";
	821	print $xml_fh "<collection-test name=\"$collection\">\n";
	822
[28661]	823	# run the building scripts from the toplevel of the GS installation
	824	if($install_version eq "3") {
	825	chdir($greenstone3_home);
	826	} else {
	827	chdir($greenstone_home);
	828	}
	829
[28072]	830	if(!$use_local_rebuild) {
	831	#import
	832	# Ensure the OIDtype for importing is hash_on_full_filename
	833	# "to make document identifiers more stable across upgrades of the software,
	834	# although it means that duplicate documents contained in the collection are
	835	# no longer detected automatically."
	836	print STDERR "$collection - Importing:\n";
	837	print $xml_fh "<import>\n";
[28661]	838	&run_build_script("import.pl $build_options -removeold $collection"); #-OIDtype hash_on_full_filename
[28072]	839	print $xml_fh "</import>\n";
	840	print STDERR "done\n";
[27621]	841
[28072]	842	#build
	843	print STDERR "$collection - Building:\n";
	844	print $xml_fh "<build>\n";
[28661]	845	&run_build_script("buildcol.pl $build_options -removeold $collection");
[28072]	846	print $xml_fh "</build>\n";
	847	print STDERR "done\n";
	848
	849	#rename the intermediate 'building' directory 'index'
	850	print STDERR "$collection - Move \"building\" to \"index\"... ";
[28661]	851	my $index = &filename_concat($collect_parent, "collect", $collection, "index");
	852	my $building = &filename_concat($collect_parent, "collect", $collection, "building");
[28072]	853	&File::Path::remove_tree($index);
	854	# Renaming Directories, http://www.perlmonks.org/?node_id=177421
	855	move($building, $index) or die "copy failed: $!"; # File::Copy::move
	856	print STDERR "done\n";
	857	}
[27621]	858	#diffcol
[27629]	859	print STDERR "$collection - Diffing:\n";
[27621]	860	my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
[28172]	861
[28661]	862	# chdir($collect_parent); # this is actually where we are
[28172]	863	# help diffcol to know on what os the model cols were generated
	864	# and what os this test machine is (on which the test cols will be generated)
[29494]	865	my $debug_state = $debugging ? "-debug" : "";
	866	$cmd = "diffcol.pl -testos $test_os -modelos $model_os -output xml -verbosity 10 $debug_state $collection"; # need to run with ./diffcol.pl if bash script
[29495]	867
	868	#print STDERR "Running diffcol cmd: $cmd\n";
[27621]	869	&run_diff_script($cmd, $xml_fh, $diffcol_dir);
	870
[28661]	871	if($install_version eq "3") {
	872	chdir($greenstone3_home); # this is actually where we are
	873	} else {
	874	chdir($greenstone_home); # this is actually where we are
	875	}
[27629]	876	print STDERR "done\n";
[27621]	877	print $xml_fh "</collection-test>\n";
	878	}
	879
	880	print $xml_fh "</test>\n";
	881	close($xml_fh);
	882
[27629]	883	print STDERR "done\n";
[27621]	884	}
	885
	886	##***************************************************************
	887	# runs setup in greenstone_home before running the diff command
	888	sub run_diff_script {
	889	my ($cmd, $fh, $diffcol_dir) = @_;
	890
[28661]	891	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
	892
[27621]	893	# we're in greenstone_home now
	894	if(!$isWin) {
[28661]	895	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && cd $diffcol_dir && ./$cmd\"";
[27621]	896
	897	} else { # Need to prefix cmd -c/-k as necessary
[28661]	898	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
[27668]	899	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	900	}
	901
	902	return &run_and_print_cmd($cmd, $fh);
	903	}
	904
	905	# runs setup in greenstone_home before running the given build command
	906	sub run_build_script {
	907	my ($cmd, $fh) = @_;
[28661]	908	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
[27621]	909
	910	# chdir($greenstone_home);
[27687]	911	# we are in $greenstone_home already, can directly run the build cmd on the collection
[27621]	912	if(!$isWin) {
[28661]	913	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && $cmd\"";
[29493]	914	#$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& export PERL_HASH_SEED_DEBUG=1&& source $setup_script.$linux_ext && $cmd\"";
[27621]	915
	916	} else { # Need to prefix cmd -c/-k as necessary
[28661]	917	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
[27621]	918	}
[27687]	919	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	920
[27668]	921	return system($cmd);
[27629]	922	#return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
[27621]	923	}
	924
	925
	926	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
	927	# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
	928	sub run_and_print_cmd {
	929	my ($cmd, $fh) = @_;
	930
	931	open my $pin, "$cmd\|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-\|', 'powercfg -l') or die $!;
	932
	933	if(defined $fh) { # print cmd output both to the filehandle and to stdout
	934	while (my $line = <$pin>) {
	935	print $fh $line;
	936	# print STDOUT $line; # if also printing cmd output to STDOUT
	937	}
	938	}
	939	else { # no filehandle, so just need to print to stdout
	940
	941	# unlike backticks operator, system() will print the output of the command to the screen as it executes
	942	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	943
	944	my $status = system $cmd;
	945	if($status != 0) {
	946	print STDERR "ERROR ($status) running $cmd: $!\n";
	947	}
	948	}
	949	close($pin);
	950	}
	951
	952	sub filename_concat {
	953	my $first_file = shift(@_);
	954	my (@filenames) = @_;
	955
	956	# If first_file is not null or empty, then add it back into the list
	957	if (defined $first_file && $first_file =~ /\S/)
	958	{
	959	unshift(@filenames, $first_file);
	960	}
	961
	962	my $filename = join($sep, @filenames);
	963	$filename =~ s/[\\\/]$//; # remove trailing slashes if any
	964	return $filename;
	965	}
	966
	967
	968	# The following code is from
	969	# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
	970	# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
	971	sub copy_recursively {
	972	my ($from_dir, $to_dir, $regex) = @_;
	973	opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
	974
	975	# if(-d !$to_dir) {
	976	# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
	977	# }
	978
	979	for my $entry (readdir $dh) {
	980	next if ($entry eq "." \|\| $entry eq "..");
	981	next if (defined $regex && $entry =~ /$regex/);
	982	my $source = "$from_dir/$entry";
	983	my $destination = "$to_dir/$entry";
	984	if (-d $source) {
	985	mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
	986	copy_recursively($source, $destination, $regex);
	987	} else {
	988	copy($source, $destination) or die "copy failed: $!";
	989	}
	990	}
	991	closedir $dh;
	992	return;
	993	}
	994
	995	sub summarise {
	996
	997	# make a summarised Xml report
[27629]	998	print STDERR "Summarizing the xml report... ";
[28661]	999	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml";
[27621]	1000	my $status = system($cmd);
[27629]	1001	print STDERR "done\n";
[27621]	1002
	1003	# make a summarised HTMl report
[27629]	1004	print STDERR "Creating an html summary report... ";
[28661]	1005	$cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.html";
[29493]	1006
	1007	print STDERR "@@@ Running summarise command: $cmd\n\n";
[27621]	1008	$status = system($cmd);
[29493]	1009
[27629]	1010	print STDERR "done\n";
[28005]	1011
	1012	# Print whether the tests passed or failed
	1013	print STDERR "*******************************************\n";
	1014	print STDERR "Checking if successful... \n";
	1015	$cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1016	$status = `$cmd`; #$status = system($cmd);
	1017	print STDERR "result: $status\n";
	1018	print STDERR "*******************************************\n";
[27621]	1019	}
	1020
	1021	sub upload {
[27657]	1022	# if the upload dir already existed, clear it of contents
	1023	if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
	1024	# don't want to keep previous days reports
	1025	# else we will have to manually clear them at some point
	1026	# just generate the set of reports for this run of task.pl upload
	1027	# and
	1028	&File::Path::remove_tree($ENV{'UPLOAD_DIR'});
	1029	}
	1030	# recreate the upload directory
	1031	&File::Path::make_path($ENV{'UPLOAD_DIR'});
[27621]	1032
	1033	# copy all .xml and .html files across to UPLOAD_DIR
	1034	opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
	1035	for my $entry (readdir $dh) {
[27687]	1036	next if ($entry !~ m/(\.xml\|\.html?)$/);
[28661]	1037	next if ($entry !~ m/(report-$install_version)/);
[27687]	1038
	1039	# copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
	1040	# html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
[27694]	1041	my $os_entry = $entry;
	1042	$os_entry =~ s@\[email protected]@ if $isWin;
[28571]	1043	if($isMac) {
[28634]	1044	$osversion = "Leopard-" if ($osversion eq "");
[28572]	1045	$os_entry = "diffcol-".$^O."-".$osversion."$os_entry"; # darwin-Lion for Lion/Mountain Lion
[28571]	1046	} else {
	1047	$os_entry = "diffcol-".$^O."-$os_entry";
	1048	}
[27687]	1049
[28621]	1050	# if the test failed, prefix "failed" to the report so that it shows up with an error icon on the caveat page
	1051	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1052	my $result = `$cmd`;
	1053	if($result ne "yes") {
	1054	$os_entry =~ s/diffcol-/diffcol-FAIL-/;
	1055	}
	1056
[27687]	1057	# get the absolute path to the original files before copying them over
	1058	$entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
	1059
	1060	# copy them over with their new names
	1061	## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
	1062	copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
[27621]	1063	}
	1064	closedir $dh;
	1065
[27637]	1066
	1067	# Upload the html file to puka
	1068	#default identity dir
	1069	if ( ! exists $ENV{'IDENTITY_DIR'} ) {
[27725]	1070	$ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
[27637]	1071	}
	1072	if (! exists $ENV{'SNAPSHOT_MODE'} ) {
	1073	$ENV{'SNAPSHOT_MODE'} = "caveat";
	1074	}
	1075
[27678]	1076	#use the correct key for uploading
[27637]	1077	$ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
	1078	if(-f $ENV{'IDENTITY_FILE'}) {
[27678]	1079	# if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
	1080
[28661]	1081	# the report we want to upload is actually just os-diffcol-report-$install_version-$dateid.html
[27687]	1082	my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c .htm \| "; #&& cat .html \| "; # && tar -c \|
[27637]	1083	$command .= ($^O eq "MSWin32" ? "plink" : "ssh");
	1084	$command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
	1085	#print "$command\n";
	1086	my $status = system("$command");
	1087	if($status != 0) {
[27678]	1088	print STDERR "*** Failed to upload test report to nzdl $status\n";
[27637]	1089	}
	1090	} else {
	1091	print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
	1092	}
	1093
[27629]	1094	print STDERR "Finished uploading\n";
[27621]	1095	}
	1096
	1097	# Sending emails with perl: http://learn.perl.org/examples/email.html
	1098	# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
[27628]	1099	# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
	1100	sub mail_with_report_attached
	1101	{
	1102	# email out with report attached, if the tests failed
[27629]	1103	print STDERR "Checking if successful... \n";
[27628]	1104	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1105	#my $result = system($cmd);
	1106	my $result = `$cmd`;
[27621]	1107
[27629]	1108	print STDERR "result: $result\n";
[27621]	1109
[27628]	1110	if($result ne "yes") {
	1111	my $msg = "$gsdl regression test for $dateid failed";
	1112	my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
[28661]	1113	my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$install_version-$dateid.html");
[27628]	1114
[27687]	1115	if($isWin) {
	1116	if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
[27678]	1117	# http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
	1118	#blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
[27628]	1119
[27678]	1120	# need to install blat on windows
	1121	$cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
	1122	$result = system($cmd);
	1123	}
	1124	else {
[27687]	1125	$result = 1; # status from running mail command is 0 if success, 1 if fail
[27678]	1126	print STDERR "********************************************\n";
[27687]	1127	if ($use_blat) {
	1128	print STDERR "Need blat and SMTP set to send mail attachment\n" ;
	1129	} else {
	1130	print STDERR "Not set up to send mail on Windows\n";
	1131	}
[27678]	1132	print STDERR "Inspect report at: $attach_file\n";
	1133	print STDERR "********************************************\n";
	1134	}
	1135	} else { # linux
[27628]	1136	my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
	1137
	1138	if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
	1139	$cmd="echo '$gsdl regression test for $dateid failed.' \| mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
	1140
	1141	print STDERR "********************************************\n";
	1142	print STDERR "No mutt installed, unable to mail attachment\n";
	1143	print STDERR "Inspect report at: $attach_file\n";
	1144	print STDERR "********************************************\n";
	1145	} else {
	1146	#$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
	1147	$cmd = "echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
	1148	}
[27678]	1149
	1150	# run the mail command
	1151	$result = system($cmd); #&run_and_print_cmd($cmd);
[27628]	1152	}
	1153
[27678]	1154
[27637]	1155	if($result != 0) {
	1156	print STDERR "*** Unable to send email: $?\n";
	1157	}
	1158	else {
	1159	print STDERR "Sent mail with report attached.\n";
	1160	}
[27628]	1161	} else {
	1162	print STDERR "********************************************\n";
[27629]	1163	print STDERR "Tests were successful. Not sending mail.\n";
[27628]	1164	print STDERR "********************************************\n";
	1165	}
	1166	}
	1167
[27621]	1168	# The old version of this program contained the following, consisting of 1 line of active code:
	1169
	1170	# Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
	1171	# Doing so will call this pl file and pass in "summarise" in ARGV
	1172	# This pl file will in turn call the task executable in this folder
	1173	# passing in "summarise" as a parameter.
	1174	#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
	1175
	1176	##system("/bin/bash -c \"./task @ARGV\"");
	1177	##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
[27694]	1178

Note: See TracBrowser for help on using the repository browser.

Download in other formats: