Context Navigation

source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 32047

Last change on this file since 32047 was 32047, checked in by ak19, 7 years ago
Uploading a copy of diffcol results to dev
Property svn:executable set to ``*
File size: 43.5 KB

Rev	Line
[27621]	1	#!/usr/bin/perl -w
[27556]	2
[27621]	3	# This program is meant to run the nightly diffcol
	4	# It is meant to be an equivalent for the existing task bash script
	5	# But it is intended to be expanded to work for Windows and GS3
	6	# For windows, need to REMEMBER to set the correct shebangs at the top
[27556]	7
[27663]	8
	9	# TODO:
	10	# Have a caveat mode and a stable mode (as in snapshot/task.pl)
	11	#
	12	#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat\|stable)" ) {
	13	# $major_version = 2;
	14	# $prefix="2t";
	15	# $rk="tk2"; # test kit
	16	#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat\|stable)" ) {
	17	# $major_version = 3;
	18	# $prefix="3t";
	19	# $rk="tk3"; # test kit
	20
[27628]	21	package diffcoltask;
[27556]	22
[27621]	23	use Cwd;
[29398]	24	#use Switch; # for switch(val) { case: ; ...}
[27621]	25	use File::Path; # for rmdir and mkdir type functions
	26	use File::Copy; # for recursive copying of folders but skipping .svn
	27	use File::Basename;
	28
[27628]	29	use strict;
	30	no strict 'subs'; # allow barewords (eg STDERR) as function arguments
[27621]	31
[27764]	32
[27621]	33	my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
[27764]	34	my $isMac = ($^O =~ m/macos\|darwin/i) ? 1 : 0;
[28571]	35
	36	my $osversion="";
	37	# Need to get the correct gnome-lib-minimal for the OS
	38	# darwin11* Lion, darwin12* Mountain Lion, darwin9* and darwin10* are Leopard and Snow Leopard
	39	if ($^O eq "darwin") {
	40	$osversion=`uname -r`; # e.g. 12.x.x
	41	#$osversion =~ s@\..*$@@; # e.g.12
[28634]	42	$osversion = ($osversion =~ m@^1[1-9](\.)?@i) ? "Lion-" : "";
[28571]	43	}
	44
	45
[27621]	46	my $sep = $isWin ? "\\" : "/";
	47	my $pathsep = $isWin ? ";" : ":";
	48	#my $script_ext = $isWin ? ".bat" : ".bash";
[28661]	49	my $setup_script = "setup"; # will become gs3-setup for GS3
[27687]	50	my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
[27621]	51
[28616]	52	my $install_type = "svn";
	53	my $install_version = "2";
	54
[28106]	55	my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
[28072]	56	my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
[27701]	57
[28072]	58	# if use_local_rebuild is on, use_static_model should be on
	59	if ($use_local_rebuild && !$use_static_model) {
	60	$use_static_model = 1;
	61	}
	62
[28172]	63	my $test_os = $isWin ? "windows" : ($isMac ? "darwin" : "linux");
	64	my $model_os = "linux"; # default
[28072]	65
[29494]	66	my $debugging = 0;
	67
[27621]	68	# TASK_HOME should be the toplevel diffcol folder
	69	$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
[27701]	70	if($isWin) {
	71	$ENV{'TASK_HOME'} =~ s@\/@\\@g;
	72	# need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
	73	# run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
	74	# At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
	75	# end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
	76	require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
	77	$ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
	78	}
[27621]	79	## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
	80
	81
[27678]	82	$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
	83
	84	# we'll be using BLAT to send mail attachments on Windows
[27687]	85	my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
	86	if($isWin && $use_blat && ! -e $blat) {
	87	print STDERR "\n***********************************\n";
	88	print STDERR "No blat.exe found in $blat.\n";
	89	print STDERR "Blat needed to send mail with attachments on Windows.\n";
	90	print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
	91	print STDERR "for your bit architecture and name the folder 'blat'\n";
	92	print STDERR "***********************************\n\n";
[27678]	93	$blat = 0;
	94	}
	95
	96
[27668]	97	$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
	98	$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
[27621]	99	$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
[27678]	100	$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
[27621]	101	##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
	102
[29448]	103	# When installing diffcol the first time, force the user to create the data-dir and upload-dir
	104	die "\n@@@@ data dir: ".$ENV{'DATA_DIR'}." does not exist\n" unless (-d $ENV{'DATA_DIR'});
	105	die "\n@@@@ reports/upload dir: ".$ENV{'UPLOAD_DIR'}." does not exist\n" unless (-d $ENV{'UPLOAD_DIR'});
	106
[27621]	107	# control if an existing compiled greenstone is used
	108	# or, if one should be checked out, which revision to checkout from svn
	109	$ENV{'SVN_OPT_REV'} = "-r head";
[27656]	110	#$ENV{'GSDLHOME'}=
	111	#$ENV{'GSDL3SRCHOME'}=
[27621]	112
[27628]	113
[27677]	114	# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
	115	my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
[27628]	116
	117	#parse arguments
	118	my $action = "all";
[28114]	119	my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
[28620]	120	my @collections = (); # list of collections that run_test should process
[28114]	121
[28620]	122	if(scalar(@ARGV) == 0) {
	123	$action="all";
	124	}
	125
	126	# process any arguments that are --gs2\|--gs3 and --bin\|--svn, and delete them from the array
	127	# if none provided, it's gs2 and svn by default.
	128	for (my $i = $#ARGV; $i >= 0; --$i) {
	129	if($ARGV[$i] =~ m/--(bin\|svn)/) {
	130	$install_type = $1;
	131	splice @ARGV, $i, 1; # remove the element from the argument array
	132	} elsif($ARGV[$i] =~ m/--gs(2\|3)/) {
	133	$install_version = $1;
[28661]	134	$setup_script = $install_version eq "3" ? "gs3-setup" : "setup"; # needs to become gs3-setup for GS3
[28620]	135	splice @ARGV, $i, 1; # remove the element from the argument array
	136	}
	137	}
	138
	139
[28172]	140	# run_test can take any number of args
[28620]	141	if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
[28172]	142	print STDERR "**** Wrong number of arguments\n";
[28114]	143	&printusage();
	144	exit -1;
[27628]	145	}
	146
[28620]	147	if(scalar(@ARGV) > 0) {
[29398]	148	#switch ($ARGV[0]) {
	149	#case qr/^(-h\|--?help\|help)$/i { &printusage; exit 0; }
	150	#case qr/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/ { $action=$ARGV[0]; }
	151	#else {
	152	#print STDERR "**** Bad subcommand.\n";
	153	#&printusage;
	154	#exit -1;
	155	#}
	156	#}
	157
	158
	159	my $switch = $ARGV[0];
	160	if($switch =~ m/^(-h\|--?help\|help)$/i) {
	161	&printusage;
	162	exit 0;
	163	} elsif ($switch =~ m/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/) {
	164	$action=$ARGV[0];
	165	} else {
	166	print STDERR "**** Bad subcommand.\n";
	167	&printusage;
	168	exit -1;
[27628]	169	}
[28616]	170
[28172]	171	# run_test action can take a subaction: nosvn\|justdiff. It can also take --modelOS (windows\|linux\|darwin)
[28114]	172	# nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
	173	# it out from the svn model-collect again.
	174	# justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
[28616]	175
[28620]	176	if($action eq "run_test" && scalar(@ARGV) >= 2) {
[28114]	177	push(@collections, @ARGV);
	178	shift @collections; # remove action from array
	179
[28172]	180	for (my $i=0; $i < scalar(@ARGV); $i++) {
	181	if($ARGV[$i] =~ m@^--@) {
	182	shift @collections; # remove subaction/flag from array
[28114]	183
[28172]	184	$subaction = $ARGV[$i];
[28114]	185	if($subaction eq "--justdiff") {
[28172]	186	$use_local_rebuild = $use_static_model = 1;
[28114]	187	} elsif ($subaction eq "--nosvn") {
[28172]	188	$use_static_model = 1;
	189	#} elsif ($subaction =~ m/\-\-testOS/i && defined $ARGV[$i+1]) {
	190	# $test_os = $ARGV[$i+1];
	191	# $i++;
	192	# shift @collections; # remove test_os value from array
	193	} elsif ($subaction =~ m/\-\-modelOS/i && defined $ARGV[$i+1] && $ARGV[$i+1] =~ m/windows\|linux\|darwin/i) {
	194	$model_os = $ARGV[$i+1];
	195	$i++;
	196	shift @collections; # remove model_os value from array
	197	#print STDERR "Model_os specified: $model_os\n";
[29494]	198	} elsif ($subaction =~ m/\-\-debug/i) {
	199	$debugging = 1;
[29496]	200	print STDERR "*** DEBUG MODE: debug output files will be in toplevel folder\n";
[28114]	201	} else {
[28172]	202	print STDERR "**** Bad subaction/value: ".$ARGV[$i]."\n";
	203	&printusage;
	204	exit -1;
	205	}
	206	}
[28114]	207	}
	208
	209	# foreach my $col (@collections) {
	210	# print STDERR "Collection: $col\n";
	211	# }
	212	}
[27628]	213	}
	214
[28661]	215	print STDERR "Install type $install_type\n";
	216	print STDERR "Install version $install_version\n";
[28114]	217
[27621]	218	#check key environment vars are set
	219	if(!defined $ENV{'UPLOAD_DIR'}) {
[27629]	220	print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
[27621]	221	#return 1;
	222	}
	223	if(!defined $ENV{'DATA_DIR'}) {
[27629]	224	print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
[27621]	225	#return 1;
	226	}
	227	if(!defined $ENV{'MONITOR_EMAIL'}) {
[27629]	228	print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
[27621]	229	#return 1;
	230	}
	231
	232	if($ENV{'DATA_DIR'} eq "/") {
[27629]	233	print STDERR "DATA_DIR should not be the fs root\n";
[27621]	234	#return 1;
	235	}
	236
[27629]	237	print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
	238	print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
[27621]	239
	240	#create an id for this test
	241	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	242	$year += 1900;
	243	$mon += 1;
	244	$mon = "0$mon" if ($mon < 10);
	245	$mday = "0$mday" if ($mday < 10);
[27702]	246	my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
[27621]	247
[27629]	248	print STDERR "Starting test '$dateid'\n";
[27621]	249
	250
	251	# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
	252	$ENV{'CLASSPATH'} = "";
	253	my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
	254	my @files = <$jar_lib_path/.jar>; # /full/path/to/diffcol/lib/jar
[27628]	255	foreach my $file (@files) {
[27668]	256	$file =~ s@\/@\\@g if $isWin;
[27621]	257	$ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
	258	}
	259	##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
	260
	261
	262	#set the location of the full report
[28661]	263	my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$install_version-$dateid.xml");
[27621]	264	##print STDERR "XML: $xmlout\n";
	265
	266	# the toplevel folder of the greenstone installation being used
[28661]	267	my $greenstone3_home="";
	268	my $greenstone_home=""; # gs2build for gs3, toplevel install folder for gs2
[27621]	269	# gsdl is the checkout folder and can be greenstone2 or greenstone3
	270	my $gsdl="greenstone2";
	271
	272	# Check if using existing compiled-up greenstone installation
	273	# and set the greenstone_home location accordingly
	274
	275	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27629]	276	print STDERR "Found existing Greenstone home, will use that instead\n";
[28661]	277	$greenstone_home=$ENV{'GSDLHOME'} if defined $ENV{'GSDLHOME'};
	278
	279	if(defined $ENV{'GSDL3SRCHOME'}) {
	280	print STDERR "*** GSDL3SRCHOME set, changing to using installed GS3\n";
	281	$gsdl = "greenstone3";
	282	$install_version = "3";
	283	$greenstone3_home=$ENV{'GSDL3SRCHOME'};
	284	$greenstone_home=filename_concat($greenstone3_home, "gs2build") unless defined $ENV{'GSDLHOME'};
	285	}
[27621]	286	} else {
[28661]	287	if($install_version eq "3") {
	288	$gsdl = "greenstone3";
	289	$greenstone3_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	290	$greenstone_home=filename_concat($greenstone3_home, "gs2build");
	291	} else {
	292	$greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	293	}
[27621]	294	}
	295	##print STDERR "GSHOME: $greenstone_home\n";
	296
[27628]	297	#do the requested action
	298	if($action eq "setup_greenstone") {
	299	&setup_greenstone;
	300	}
	301	elsif ($action eq "run_test") {
	302	&run_test;
	303	}
	304	elsif ($action eq "summarise") {
	305	&summarise;
	306	}
	307	elsif ($action eq "upload") {
	308	&upload;
	309	&mail_with_report_attached;
	310	}
	311	elsif ($action eq "all") {
	312	&setup_greenstone;
	313	&run_test;
	314	&summarise;
	315	&upload;
	316	&mail_with_report_attached;
	317	}
[27621]	318
	319	##********************************
	320
[27628]	321	sub printusage
	322	{
[28172]	323	# print STDERR "Run as: $0 (help\|setup_greenstone\|run_test <--modelOS windows\|darwin\|linux> <--justdiff\|--nosvn> <col1 col2 ...> \|summarise\|upload\|all)\n";
	324	print STDERR "Run as: $0 (help\|setup_greenstone\|run_test\|summarise\|upload\|all)\n";
	325	print STDERR "where run_test can further take the following optional parameters:\n";
	326	print STDERR "\t--modelOS (windows\|darwin\|linux)\n";
	327	print STDERR "\t--justdiff\|--nosvn\n";
[29496]	328	print STDERR "\t--debug\n";
[28172]	329	print STDERR "\t<col1 col2 ...>\n";
[28616]	330	print STDERR "where setup_greenstone can further take the following optional parameters:\n";
	331	print STDERR "\t--gs2\|--gs3\n";
	332	print STDERR "\t--svn\|--bin\n";
[27628]	333	}
[27621]	334
	335	#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
	336
	337	sub setup_greenstone
	338	{
[28661]	339	if($install_version eq "3" && !defined $ENV{'JAVA_HOME'}) {
	340	print "*** JAVA_HOME not set. Set it and add its bin folder to the PATH\n";
	341	exit 0;
	342	}
	343
[27621]	344	#clean up from previous tests
[27628]	345	print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
[27621]	346	for my $i ( 1..5 ) {
	347	sleep 1; # 1 second
[27628]	348	print STDERR ".";
[27621]	349	}
[27628]	350	print STDERR "\n";
[27621]	351
	352	# http://perldoc.perl.org/File/Path.html
[27629]	353	print STDERR "cleaning up previous tests\n";
[28661]	354	#&File::Path::remove_tree($ENV{'DATA_DIR'});
	355	if(-d $greenstone_home && $install_version eq "2") {
	356	&File::Path::remove_tree($greenstone_home);
	357	}
	358	if(-d $greenstone3_home && $install_version eq "3") {
	359	&File::Path::remove_tree($greenstone3_home);
	360	}
	361	unlink glob "$ENV{'DATA_DIR'}$sep"."report-$install_version";
	362	unlink "$ENV{'DATA_DIR'}$sep"."compilation-errors";
[27629]	363	print STDERR "creating the data dir\n";
[28661]	364	#&File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
[27621]	365
	366	chdir($ENV{'DATA_DIR'});
[32047]	367	#print STDERR "@@@@ current dir: ".`pwd`."\n";
	368	#print STDERR "@@@@ data dir: ".$ENV{'DATA_DIR'}."\n";
[27621]	369
	370	# use existing compiled-up greenstone installation, if a GSDLHOME set
	371	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27710]	372	print STDERR "Found existing Greenstone home, will use that instead\n";
[27621]	373	return;
	374	}
	375
	376	# Else checkout a GS from svn into DATA_DIR
[28661]	377	if($install_type eq "svn") {
	378	&checkout_gs_from_svn();
	379	##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
[27621]	380
[28661]	381	if($install_version eq "2") {
	382	&compile_gs2_svn();
	383	} else {
	384	&compile_gs3_svn();
	385	}
	386	}
	387
	388	}
	389
	390	sub checkout_gs_from_svn {
	391
[27621]	392	#svn checkout of main gsdl directory
[27629]	393	print STDERR "checkout $gsdl:\n";
[28661]	394	my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone$install_version $gsdl";
[27621]	395	##print STDERR "Checkout CMD: $cmd\n";
	396
	397	# # unlike backticks operator, system() will print the output of the command to the screen as it executes
	398	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	399	my $status = system "$cmd"; #my $status = `$cmd`;
[28661]	400	if($status != 0) {
	401	print STDERR "@@@ SVN checkout of $gsdl failed\n";
	402	exit -1;
	403	}
[27629]	404	print STDERR "done\n";
[28661]	405	}
[27621]	406
[28661]	407	sub getImageMagickBins {
	408	my ($cmd, $status);
	409	my $imagickzip = "";
[27621]	410
[28661]	411	chdir ("$greenstone_home");
	412	if(!$isWin) { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
[27710]	413
	414	my $bit_arch=`uname -m`;
[28661]	415
	416	# imagemagick binary
[27710]	417	print STDERR "Getting imagemagick binary\n";
[28661]	418
[27724]	419	my $os = $isMac ? "darwin" : "linux";
[28661]	420	$imagickzip = "imagemagick-$os";
[27668]	421
[27710]	422	if($isMac) {
[28661]	423	# $imagickzip .= "-10.5.tar.gz";
[27962]	424	# at present, only the Imagemagick binaries created by Max for darwin work on the Macs
[28661]	425	# &File::Path::make_path("$greenstone_home$sep"."bin"."$sep$os"); # need to ensure gsdl/bin/darwin exists
[27962]	426	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
	427	$status = system($cmd);
[28661]	428	if($status != 0) {
	429	print STDERR "@@@ Unable to get imagemagick for darwin\n";
	430	}
	431
	432	# need ghostscript mac binary too for pdf to img conversions on mac
[27963]	433	$cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
	434	$status = system($cmd);
[28661]	435	if($status != 0) {
	436	print STDERR "@@@ Unable to get ghostscript for darwin\n";
	437	}
	438
	439	# the imagemagick and ghostscript binaries have been set to executable on svn trac now
	440	# system("chmod -R u+x $greenstone_home/bin/darwin/imagemagick/bin/*");
	441	# system("chmod -R u+x $greenstone_home/bin/darwin/ghostscript/bin/*");
[27710]	442	} else { # linux
	443	my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
[27724]	444	$imagickzip .= "$extension64.tar.gz";
[27962]	445
	446	# now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
	447	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
	448	$status = system ($cmd);
	449	system("cd ext && tar -xvzf $imagickzip");
[27724]	450	}
[28661]	451	}
	452	return $imagickzip;
	453	}
[27710]	454
[28661]	455	sub getGnomeLibExt
	456	{
[29455]	457	if($isWin) {
	458	return; # no gnome-lib for windows
	459	}
[28661]	460	my ($cmd, $status);
	461	my $bit_arch=`uname -m`;
	462	my $os = $isMac ? "darwin" : "linux";
	463
[27710]	464	# gnomelib binary
[27629]	465	print STDERR "setting up gnome-lib-minimal for compilation\n";
[27621]	466
	467	# To get gnome-lib, need to determine bit architecture of the linux/darwin
	468	# http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
	469	# $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
	470	# well on ActivePerl.
[28621]	471	# But since we know we're on a Linux/Darwin machine at this point, we can just run `uname -m` and other linux cmds
[27621]	472
[28571]	473	# osversion will be "Lion" or ""
	474	# and assuming all darwin is intel, not ppc!!
[28572]	475	my $gnome_lib_file = $isMac ? "darwin-".$osversion."intel" : "linux";
[27621]	476
[28571]	477	$gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/ && !$isMac); # linux only case
[27621]	478
	479	#svn checkout gnome-lib for this linux/darwin
[28661]	480	chdir("$greenstone_home$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
[27621]	481
	482	##print STDERR "**** gnomelib: $gnome_lib_file\n";
	483
	484	# checkout and unpack gnome-lib-minimal
	485
	486	#svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
	487	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
	488	system $cmd;
	489	system ("tar -xvzf gl.tar.gz");
	490
	491	##print STDERR "*** ARCH: $bit_arch\n";
[28661]	492	}
[27621]	493
[28661]	494
	495
	496	sub compile_gs3_svn() {
	497	my ($cmd, $status);
	498	chdir ("$greenstone3_home");
	499
	500	$cmd = "ant"; # creates the build.properties file from the .in template
	501	$status = system $cmd;
	502	if($status != 0) {
	503	print STDERR "Unable to run the ant command: $status\n";
	504	exit -1;
	505	}
	506
	507	$cmd = "ant -Dproperties.ok=y -Dcheckout.gnomelib.ext=true prepare"; # pass in confirmation to ant prepare step
	508	$status = system $cmd;
	509	if($status != 0) {
	510	print STDERR "Failed to run $cmd command: $status\n";
	511	exit -1;
	512	}
	513
	514	my $imagickzip = &getImageMagickBins();
	515	chdir ("$greenstone3_home");
	516
	517	$cmd = "ant -Dcheckout.gnomelib.ext=true install"; # Compile with gnome-lib.
	518	$status = system $cmd;
	519	if($status != 0) {
	520	print STDERR "Failed to run $cmd command: $status\n";
	521	exit -1;
	522	}
	523	}
	524
	525	sub compile_gs2_svn() {
	526	my $imagickzip = &getImageMagickBins();
	527	&getGnomeLibExt();
	528
	529	my ($cmd, $status);
	530	my $os = $isMac ? "darwin" : "linux";
	531
	532	chdir("$greenstone_home"); #chdir("$ENV{'DATA_DIR'}$sep$gsdl"); # goes into toplevel gs2 or gs3 folder
	533
	534	##print STDERR "@@@ OS: $^O.\|".$Config{'archname64'}."\|\n";
	535
	536	if($isWin) {
	537	print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
	538
	539	# we're now in the GS2 folder, call makegs2 with silent param
	540	$cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
	541	$status = system $cmd;
	542	if($status != 0) {
	543	print STDERR "Greenstone compilation on Windows failed\n";
	544	exit -1;
	545	}
	546
	547	} else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
	548
	549	chdir("ext$sep"."gnome-lib-minimal");
	550
[27621]	551	# need to run source devel.bash on gnome-lib followed by configure, make, make install
	552	# in one go, in order to preserve the compile environment set up by sourcing devel.bash
	553
	554	# http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
	555	# http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
	556
	557	# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
	558	$cmd = "bash -c \"";
	559
	560	$cmd .= "source ./devel.bash";
	561	$cmd .= " && cd ../..";
	562
	563	#configure
	564	# $cmd .= " && ./configure";
	565	$cmd .= " && echo 'configure $gsdl: ' ";
	566	$cmd .= " && echo '<configure>' >> $xmlout";
	567	$cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
	568	$cmd .= " && echo '</configure>' >> $xmlout";
	569	$cmd .= " && echo 'done'";
	570
	571	#make
	572	$cmd .= " && echo 'make $gsdl: '";
	573	$cmd .= " && echo '<make>' >> $xmlout";
	574	$cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
	575	$cmd .= " && echo '</make>' >> $xmlout";
	576	$cmd .= " && echo 'done'";
	577
	578	#make install
	579	$cmd .= " && echo 'make install $gsdl: '";
	580	$cmd .= " && echo '<make-install>' >> $xmlout";
	581	$cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
	582	$cmd .= " && echo '</make-install>' >> $xmlout";
	583	$cmd .= " && echo 'done'";
	584
	585	$cmd .= "\""; # close off cmd to bash and run it
[27668]	586	$status = system $cmd;
[27724]	587
[27962]	588	if(!$isMac) { # Linux
[28661]	589	# Moving imagemagick after instead of before compilation, since bin/darwin and bin/linux gets overwritten during compilation
	590	move("$greenstone_home$sep"."ext/imagemagick/$os", "$greenstone_home$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
	591	unlink "$greenstone_home$sep"."ext$sep$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
	592	&File::Path::remove_tree("$greenstone_home$sep"."ext$sep"."imagemagick"); # the untarred parent folder
[27962]	593	}
[27724]	594
[27621]	595	}
[27668]	596
[28661]	597	if($status != 0) {
	598	print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
	599	exit -1;
	600	}
	601
	602	&getIsisGdl("$greenstone_home"); #&getIsisGdl("$ENV{'DATA_DIR'}/$gsdl");
[27764]	603
[27621]	604	# set the path to the greenstone_home variable
[28661]	605	#$greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
[27764]	606	}
[27621]	607
[27764]	608	sub getPDFBox
	609	{
	610	# current revision is 27763, but using "head" works
	611	my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
	612	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
	613
	614	# now get the PDFBox extension for PDFBox tutorial
	615	print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
	616
	617	chdir($greenstone_home);
	618	my $cmd = "";
	619	if ($isWin) {
	620	$cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
	621
	622	} elsif ($isMac) { # need to use curl not wget
	623	$cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
	624	}
	625	else { # linux
[27765]	626	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
[27764]	627	}
	628	my $status = system $cmd;
	629	if($status != 0) {
	630	print STDERR "@@@ Failed to set up PDFBox\n";
[27765]	631	exit -1; # or proceed to testing other tutorials?
[27764]	632	}
[27621]	633	}
	634
[28615]	635	sub getOpenOfficeExt
	636	{
	637	# current revision is 27763, but using "head" works
	638	my $OOEXT_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/open-office/trunk/open-office-java"; # both for .zip and .tar.gz extension
	639	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/open-office/trunk/open-office-java";
	640
	641	# now get the OpenOffice extension for the AllDocTypes collection
	642	print STDERR "Getting open office ext from $OOEXT_TRAC_URL:\n";
	643
	644	chdir($greenstone_home);
	645	my $cmd = "";
	646	if ($isWin) {
	647	$cmd = "setup.bat && cd ext && wget $OOEXT_TRAC_URL.zip && unzip open-office-java.zip";
	648
	649	} elsif ($isMac) { # need to use curl not wget
	650	$cmd = "cd ext && curl $OOEXT_TRAC_URL.tar.gz > open-office-java.tar.gz && tar -xzf open-office-java.tar.gz";
	651	}
	652	else { # linux
	653	$cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $OOEXT_TRAC_URL.tar.gz && tar -xzf open-office-java.tar.gz\"";
	654	}
	655	my $status = system $cmd;
	656	if($status != 0) {
	657	print STDERR "@@@ Failed to set up the Open Office Extension\n";
	658	exit -1; # or proceed to testing other tutorials?
	659	}
	660	}
	661
[28605]	662	sub getIsisGdl {
[28606]	663	my $gsfolder = shift(@_);
[28605]	664
[28606]	665	if(!$isWin) {
[28661]	666	chdir($greenstone_home);
[28605]	667	my $bit_arch=`uname -m`;
	668	if ($bit_arch =~ m/64$/) {
[28606]	669	my $cmd = "";
	670	if($isMac) {
	671	$cmd = "cd $gsfolder/bin/darwin && curl http://www.greenstone.org/caveat-emptor/IsisGdl.macleopard > IsisGdl && chmod u+x IsisGdl";
	672	} else { # linux
	673	$cmd = "cd $gsfolder/bin/linux && wget http://www.greenstone.org/caveat-emptor/IsisGdl.bin32 && mv IsisGdl.bin32 IsisGdl && chmod u+x IsisGdl";
	674	}
[28605]	675	my $isis_status = system $cmd;
	676	if($isis_status != 0) {
[28606]	677	print STDERR "Unable to get IsisGdl from caveat page\n";
[28605]	678	}
	679	}
	680	}
	681	}
	682
[27621]	683	# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
	684	sub run_test
[28661]	685	{
	686	my $collect_parent = $greenstone_home;
	687	my $model_collect = "model-collect";
	688	my $build_options = "";
	689	if($install_version eq "3") {
	690	$collect_parent = &filename_concat($greenstone3_home,"web","sites","localsite");
	691	$model_collect = "gs3-model-collect";
	692	$build_options = " -site localsite ";
	693	chdir($greenstone3_home);
	694	} else {
	695	chdir($greenstone_home);
	696	}
	697
[28114]	698	my $num_cols = scalar(@collections); # remember the empty case
	699
	700	if($num_cols == 0) { # deal with all collections
	701	push (@collections, "");
[28615]	702	# putting the empty string in the array so that the "all collections" case
	703	# can be handled similar to how the case of user-specified collections is handled
[28114]	704
	705	} else { # deal with user specified set of collections
	706	# prefix the directory separator to each collection name
	707	@collections = map { $sep.$_ } @collections;
	708	}
	709
[27764]	710	my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
	711	if(!-d $pdfbox) {
	712	&getPDFBox();
	713	}
	714
[28615]	715	my $openofficeext = &filename_concat($greenstone_home, "ext", "open-office");
	716	if(!-d $openofficeext) {
	717	&getOpenOfficeExt();
	718	}
	719
	720	#&getIsisGdl("$greenstone_home");
[28605]	721
[27668]	722	open (my $xml_fh, '>'.$xmlout) \|\| die "Could not open xml file $xmlout for appending: $!\n";
[27621]	723
	724	# perform the requested subcommands, outputting xml information
	725	print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
[28072]	726
[28114]	727	my ($cmd, $status);
[27724]	728	# make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
[28072]	729
	730	if(!$use_local_rebuild) {
[28661]	731	print $xml_fh "Updating $ENV{'TASK_HOME'}/$model_collect:\n";
[28114]	732	for my $col (@collections) {
[28661]	733	$cmd = "svn up $ENV{'TASK_HOME'}/$model_collect$col"; #chdir("$ENV{'TASK_HOME'}/$model_collect");
[28114]	734	$status = system "$cmd";
	735	}
[28072]	736	}
[27724]	737
[28661]	738	# go to whichever collecthome parent we're using
	739	chdir($collect_parent);
[27621]	740
	741	# get svn info
[27629]	742	print STDERR "getting svn info: $xmlout\n";
[27621]	743	print $xml_fh "<svn-info>\n";
[27668]	744	&run_and_print_cmd("svn info", $xml_fh);
[27621]	745	print $xml_fh "</svn-info>\n";
[27629]	746	print STDERR "done\n";
[27621]	747
[28114]	748	if(!$use_local_rebuild) {
	749
	750	#make two copies of the model-collect directory in gsdl
	751	#one to be rebuilt and one as the basis for comparison
	752	#strip both of all .svn directories
	753
	754	#copy the model collections to the collect folder to be rebuilt
	755	print STDERR "installing test collections and model collections to new $gsdl installation... ";
[28072]	756	#clean up
	757	if(-d "collect") {
[28114]	758	for my $col (@collections) {
[28172]	759	if(-d "collect$col") {
	760	&File::Path::remove_tree("collect$col") \|\| die "Error could not delete collect: $!";
	761	}
[28114]	762	}
[28072]	763	}
[27621]	764
[28072]	765	if($use_static_model) {
[28114]	766	for my $col (@collections) {
	767	#copy to collect and strip .svn subfolders
	768	&File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
	769	&copy_recursively("model-collect$col", "collect$col", ".svn");
	770	}
[27621]	771
[28072]	772	} else { # the default situation: where we check out the model-collect from svn
[28114]	773	for my $col (@collections) {
	774	&File::Path::remove_tree("model-collect$col");
	775
	776	#copy to collect and strip .svn subfolders
	777	&File::Path::make_path("collect$col"); # create the folder and copy contents across
[28661]	778	&copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","$model_collect$col"), "collect$col", ".svn");
[28114]	779
	780	#make the model copy
	781	&File::Path::make_path("model-collect$col");
	782	&copy_recursively("collect$col", "model-collect$col"); # copy contents across
	783	}
[28072]	784	}
	785
	786	print STDERR "done\n";
	787	}
	788
[28114]	789	#for each collection, import, build and diff with its model counterpart
	790
	791	# if working with all collections, read the list of collections from the folders in collect
	792	if($num_cols == 0) {
	793	@collections = (); # get rid of the empty string put in the array to represent "all collections"
	794
[28661]	795	opendir my($collect_handle), "collect" or die "Could not open dir $collect_parent/collect: $!";
[28114]	796
	797	for my $collection (readdir $collect_handle) {
	798	next if ($collection eq "." \|\| $collection eq "..");
	799	next if ($collection eq "modelcol");
	800	push(@collections, $collection);
	801	}
	802	closedir $collect_handle; # close handle to collect dir
	803	}
	804
	805	for my $collection (@collections) {
[27621]	806
[28114]	807	# next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
	808	# next if ($collection !~ m/OAI\|METS\|DSpace\|MGPP\|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
[29449]	809	# next if ($collection !~ m/PDFBox/); ## TEMPORARY, FOR TESTING THIS SCRIPT
[28114]	810
[27621]	811	#escape the filename (in case of space)
	812	$collection =~ s@ @\\ @g;
	813	#getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
[28114]	814
	815	$collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
	816
	817	if (! -d "collect$sep$collection") {
	818	print STDERR "Collection $collection does not exist\n";
	819	next;
	820	}
	821
[27621]	822	print STDERR "*** Found collection $collection\n";
	823	print $xml_fh "<collection-test name=\"$collection\">\n";
	824
[28661]	825	# run the building scripts from the toplevel of the GS installation
	826	if($install_version eq "3") {
	827	chdir($greenstone3_home);
	828	} else {
	829	chdir($greenstone_home);
	830	}
	831
[28072]	832	if(!$use_local_rebuild) {
	833	#import
	834	# Ensure the OIDtype for importing is hash_on_full_filename
	835	# "to make document identifiers more stable across upgrades of the software,
	836	# although it means that duplicate documents contained in the collection are
	837	# no longer detected automatically."
	838	print STDERR "$collection - Importing:\n";
	839	print $xml_fh "<import>\n";
[28661]	840	&run_build_script("import.pl $build_options -removeold $collection"); #-OIDtype hash_on_full_filename
[28072]	841	print $xml_fh "</import>\n";
	842	print STDERR "done\n";
[27621]	843
[28072]	844	#build
	845	print STDERR "$collection - Building:\n";
	846	print $xml_fh "<build>\n";
[28661]	847	&run_build_script("buildcol.pl $build_options -removeold $collection");
[28072]	848	print $xml_fh "</build>\n";
	849	print STDERR "done\n";
	850
	851	#rename the intermediate 'building' directory 'index'
	852	print STDERR "$collection - Move \"building\" to \"index\"... ";
[28661]	853	my $index = &filename_concat($collect_parent, "collect", $collection, "index");
	854	my $building = &filename_concat($collect_parent, "collect", $collection, "building");
[28072]	855	&File::Path::remove_tree($index);
	856	# Renaming Directories, http://www.perlmonks.org/?node_id=177421
	857	move($building, $index) or die "copy failed: $!"; # File::Copy::move
	858	print STDERR "done\n";
	859	}
[27621]	860	#diffcol
[27629]	861	print STDERR "$collection - Diffing:\n";
[27621]	862	my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
[28172]	863
[28661]	864	# chdir($collect_parent); # this is actually where we are
[28172]	865	# help diffcol to know on what os the model cols were generated
	866	# and what os this test machine is (on which the test cols will be generated)
[29494]	867	my $debug_state = $debugging ? "-debug" : "";
	868	$cmd = "diffcol.pl -testos $test_os -modelos $model_os -output xml -verbosity 10 $debug_state $collection"; # need to run with ./diffcol.pl if bash script
[29495]	869
	870	#print STDERR "Running diffcol cmd: $cmd\n";
[27621]	871	&run_diff_script($cmd, $xml_fh, $diffcol_dir);
	872
[28661]	873	if($install_version eq "3") {
	874	chdir($greenstone3_home); # this is actually where we are
	875	} else {
	876	chdir($greenstone_home); # this is actually where we are
	877	}
[27629]	878	print STDERR "done\n";
[27621]	879	print $xml_fh "</collection-test>\n";
	880	}
	881
	882	print $xml_fh "</test>\n";
	883	close($xml_fh);
	884
[27629]	885	print STDERR "done\n";
[27621]	886	}
	887
	888	##***************************************************************
	889	# runs setup in greenstone_home before running the diff command
	890	sub run_diff_script {
	891	my ($cmd, $fh, $diffcol_dir) = @_;
	892
[28661]	893	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
	894
[27621]	895	# we're in greenstone_home now
	896	if(!$isWin) {
[28661]	897	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && cd $diffcol_dir && ./$cmd\"";
[27621]	898
	899	} else { # Need to prefix cmd -c/-k as necessary
[28661]	900	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
[27668]	901	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	902	}
	903
	904	return &run_and_print_cmd($cmd, $fh);
	905	}
	906
	907	# runs setup in greenstone_home before running the given build command
	908	sub run_build_script {
	909	my ($cmd, $fh) = @_;
[28661]	910	my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
[27621]	911
	912	# chdir($greenstone_home);
[27687]	913	# we are in $greenstone_home already, can directly run the build cmd on the collection
[27621]	914	if(!$isWin) {
[28661]	915	$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && $cmd\"";
[29493]	916	#$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& export PERL_HASH_SEED_DEBUG=1&& source $setup_script.$linux_ext && $cmd\"";
[27621]	917
	918	} else { # Need to prefix cmd -c/-k as necessary
[28661]	919	$cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
[27621]	920	}
[27687]	921	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	922
[27668]	923	return system($cmd);
[27629]	924	#return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
[27621]	925	}
	926
	927
	928	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
	929	# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
	930	sub run_and_print_cmd {
	931	my ($cmd, $fh) = @_;
	932
	933	open my $pin, "$cmd\|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-\|', 'powercfg -l') or die $!;
	934
	935	if(defined $fh) { # print cmd output both to the filehandle and to stdout
	936	while (my $line = <$pin>) {
	937	print $fh $line;
	938	# print STDOUT $line; # if also printing cmd output to STDOUT
	939	}
	940	}
	941	else { # no filehandle, so just need to print to stdout
	942
	943	# unlike backticks operator, system() will print the output of the command to the screen as it executes
	944	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	945
	946	my $status = system $cmd;
	947	if($status != 0) {
	948	print STDERR "ERROR ($status) running $cmd: $!\n";
	949	}
	950	}
	951	close($pin);
	952	}
	953
	954	sub filename_concat {
	955	my $first_file = shift(@_);
	956	my (@filenames) = @_;
	957
	958	# If first_file is not null or empty, then add it back into the list
	959	if (defined $first_file && $first_file =~ /\S/)
	960	{
	961	unshift(@filenames, $first_file);
	962	}
	963
	964	my $filename = join($sep, @filenames);
	965	$filename =~ s/[\\\/]$//; # remove trailing slashes if any
	966	return $filename;
	967	}
	968
	969
	970	# The following code is from
	971	# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
	972	# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
	973	sub copy_recursively {
	974	my ($from_dir, $to_dir, $regex) = @_;
	975	opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
	976
	977	# if(-d !$to_dir) {
	978	# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
	979	# }
	980
	981	for my $entry (readdir $dh) {
	982	next if ($entry eq "." \|\| $entry eq "..");
	983	next if (defined $regex && $entry =~ /$regex/);
	984	my $source = "$from_dir/$entry";
	985	my $destination = "$to_dir/$entry";
	986	if (-d $source) {
	987	mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
	988	copy_recursively($source, $destination, $regex);
	989	} else {
	990	copy($source, $destination) or die "copy failed: $!";
	991	}
	992	}
	993	closedir $dh;
	994	return;
	995	}
	996
	997	sub summarise {
	998
	999	# make a summarised Xml report
[27629]	1000	print STDERR "Summarizing the xml report... ";
[28661]	1001	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml";
[27621]	1002	my $status = system($cmd);
[27629]	1003	print STDERR "done\n";
[27621]	1004
	1005	# make a summarised HTMl report
[27629]	1006	print STDERR "Creating an html summary report... ";
[28661]	1007	$cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.html";
[29493]	1008
	1009	print STDERR "@@@ Running summarise command: $cmd\n\n";
[27621]	1010	$status = system($cmd);
[29493]	1011
[27629]	1012	print STDERR "done\n";
[28005]	1013
	1014	# Print whether the tests passed or failed
	1015	print STDERR "*******************************************\n";
	1016	print STDERR "Checking if successful... \n";
	1017	$cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1018	$status = `$cmd`; #$status = system($cmd);
	1019	print STDERR "result: $status\n";
	1020	print STDERR "*******************************************\n";
[27621]	1021	}
	1022
	1023	sub upload {
[27657]	1024	# if the upload dir already existed, clear it of contents
	1025	if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
	1026	# don't want to keep previous days reports
	1027	# else we will have to manually clear them at some point
	1028	# just generate the set of reports for this run of task.pl upload
	1029	# and
	1030	&File::Path::remove_tree($ENV{'UPLOAD_DIR'});
	1031	}
	1032	# recreate the upload directory
	1033	&File::Path::make_path($ENV{'UPLOAD_DIR'});
[27621]	1034
	1035	# copy all .xml and .html files across to UPLOAD_DIR
	1036	opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
	1037	for my $entry (readdir $dh) {
[27687]	1038	next if ($entry !~ m/(\.xml\|\.html?)$/);
[28661]	1039	next if ($entry !~ m/(report-$install_version)/);
[27687]	1040
	1041	# copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
	1042	# html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
[27694]	1043	my $os_entry = $entry;
	1044	$os_entry =~ s@\[email protected]@ if $isWin;
[28571]	1045	if($isMac) {
[28634]	1046	$osversion = "Leopard-" if ($osversion eq "");
[28572]	1047	$os_entry = "diffcol-".$^O."-".$osversion."$os_entry"; # darwin-Lion for Lion/Mountain Lion
[28571]	1048	} else {
	1049	$os_entry = "diffcol-".$^O."-$os_entry";
	1050	}
[27687]	1051
[28621]	1052	# if the test failed, prefix "failed" to the report so that it shows up with an error icon on the caveat page
	1053	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1054	my $result = `$cmd`;
	1055	if($result ne "yes") {
	1056	$os_entry =~ s/diffcol-/diffcol-FAIL-/;
	1057	}
	1058
[27687]	1059	# get the absolute path to the original files before copying them over
	1060	$entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
	1061
	1062	# copy them over with their new names
	1063	## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
	1064	copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
[27621]	1065	}
	1066	closedir $dh;
	1067
[27637]	1068
	1069	# Upload the html file to puka
	1070	#default identity dir
	1071	if ( ! exists $ENV{'IDENTITY_DIR'} ) {
[27725]	1072	$ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
[27637]	1073	}
	1074	if (! exists $ENV{'SNAPSHOT_MODE'} ) {
	1075	$ENV{'SNAPSHOT_MODE'} = "caveat";
	1076	}
	1077
[27678]	1078	#use the correct key for uploading
[27637]	1079	$ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
	1080	if(-f $ENV{'IDENTITY_FILE'}) {
[27678]	1081	# if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
	1082
[28661]	1083	# the report we want to upload is actually just os-diffcol-report-$install_version-$dateid.html
[27687]	1084	my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c .htm \| "; #&& cat .html \| "; # && tar -c \|
[27637]	1085	$command .= ($^O eq "MSWin32" ? "plink" : "ssh");
	1086	$command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
	1087	#print "$command\n";
	1088	my $status = system("$command");
	1089	if($status != 0) {
[32047]	1090	print STDERR "*** Failed to upload test report to nzdl $status\n";
[27637]	1091	}
[32047]	1092
	1093	# for now, upload a copy to the new machine, later to replace puka
	1094	$command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c .htm \| "; #&& cat .html \| "; # && tar -c \|
	1095	$command .= ($^O eq "MSWin32" ? "plink" : "ssh");
	1096	$command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl-gsorg\@wwwdev.greenstone.org";
	1097	#print "$command\n";
	1098	$status = system("$command");
	1099	if($status != 0) {
	1100	print STDERR "*** Failed to upload test report to nzdl $status\n";
	1101	}
	1102
[27637]	1103	} else {
	1104	print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
	1105	}
	1106
[27629]	1107	print STDERR "Finished uploading\n";
[27621]	1108	}
	1109
	1110	# Sending emails with perl: http://learn.perl.org/examples/email.html
	1111	# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
[27628]	1112	# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
	1113	sub mail_with_report_attached
	1114	{
	1115	# email out with report attached, if the tests failed
[27629]	1116	print STDERR "Checking if successful... \n";
[27628]	1117	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	1118	#my $result = system($cmd);
	1119	my $result = `$cmd`;
[27621]	1120
[27629]	1121	print STDERR "result: $result\n";
[27621]	1122
[27628]	1123	if($result ne "yes") {
	1124	my $msg = "$gsdl regression test for $dateid failed";
	1125	my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
[28661]	1126	my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$install_version-$dateid.html");
[27628]	1127
[27687]	1128	if($isWin) {
	1129	if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
[27678]	1130	# http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
	1131	#blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
[27628]	1132
[27678]	1133	# need to install blat on windows
	1134	$cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
	1135	$result = system($cmd);
	1136	}
	1137	else {
[27687]	1138	$result = 1; # status from running mail command is 0 if success, 1 if fail
[27678]	1139	print STDERR "********************************************\n";
[27687]	1140	if ($use_blat) {
	1141	print STDERR "Need blat and SMTP set to send mail attachment\n" ;
	1142	} else {
	1143	print STDERR "Not set up to send mail on Windows\n";
	1144	}
[27678]	1145	print STDERR "Inspect report at: $attach_file\n";
	1146	print STDERR "********************************************\n";
	1147	}
	1148	} else { # linux
[27628]	1149	my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
	1150
	1151	if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
	1152	$cmd="echo '$gsdl regression test for $dateid failed.' \| mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
	1153
	1154	print STDERR "********************************************\n";
	1155	print STDERR "No mutt installed, unable to mail attachment\n";
	1156	print STDERR "Inspect report at: $attach_file\n";
	1157	print STDERR "********************************************\n";
	1158	} else {
	1159	#$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
	1160	$cmd = "echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
	1161	}
[27678]	1162
	1163	# run the mail command
	1164	$result = system($cmd); #&run_and_print_cmd($cmd);
[27628]	1165	}
	1166
[27678]	1167
[27637]	1168	if($result != 0) {
	1169	print STDERR "*** Unable to send email: $?\n";
	1170	}
	1171	else {
	1172	print STDERR "Sent mail with report attached.\n";
	1173	}
[27628]	1174	} else {
	1175	print STDERR "********************************************\n";
[27629]	1176	print STDERR "Tests were successful. Not sending mail.\n";
[27628]	1177	print STDERR "********************************************\n";
	1178	}
	1179	}
	1180
[27621]	1181	# The old version of this program contained the following, consisting of 1 line of active code:
	1182
	1183	# Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
	1184	# Doing so will call this pl file and pass in "summarise" in ARGV
	1185	# This pl file will in turn call the task executable in this folder
	1186	# passing in "summarise" as a parameter.
	1187	#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
	1188
	1189	##system("/bin/bash -c \"./task @ARGV\"");
	1190	##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
[27694]	1191

Note: See TracBrowser for help on using the repository browser.

Download in other formats: