Context Navigation

source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 27764

Last change on this file since 27764 was 27764, checked in by ak19, 11 years ago
Now it gets the PDFBox binary from svn and unzips it into ext for testing the PDFBox tutorial.
Property svn:executable set to ``*
File size: 28.1 KB

Rev	Line
[27621]	1	#!/usr/bin/perl -w
[27556]	2
[27621]	3	# This program is meant to run the nightly diffcol
	4	# It is meant to be an equivalent for the existing task bash script
	5	# But it is intended to be expanded to work for Windows and GS3
	6	# For windows, need to REMEMBER to set the correct shebangs at the top
[27556]	7
[27663]	8
	9	# TODO:
	10	# Have a caveat mode and a stable mode (as in snapshot/task.pl)
	11	#
	12	#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat\|stable)" ) {
	13	# $major_version = 2;
	14	# $prefix="2t";
	15	# $rk="tk2"; # test kit
	16	#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat\|stable)" ) {
	17	# $major_version = 3;
	18	# $prefix="3t";
	19	# $rk="tk3"; # test kit
	20
[27628]	21	package diffcoltask;
[27556]	22
[27621]	23	use Cwd;
[27628]	24	use Switch; # for switch(val) { case: ; ...}
[27621]	25	use File::Path; # for rmdir and mkdir type functions
	26	use File::Copy; # for recursive copying of folders but skipping .svn
	27	use File::Basename;
	28
[27628]	29	use strict;
	30	no strict 'subs'; # allow barewords (eg STDERR) as function arguments
[27621]	31
[27764]	32
[27621]	33	my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
[27764]	34	my $isMac = ($^O =~ m/macos\|darwin/i) ? 1 : 0;
[27621]	35	my $sep = $isWin ? "\\" : "/";
	36	my $pathsep = $isWin ? ";" : ":";
	37	#my $script_ext = $isWin ? ".bat" : ".bash";
	38	my $setup_script = "setup"; # needs to become gs3-setup for GS3
[27687]	39	my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
[27621]	40
[27701]	41
[27621]	42	# TASK_HOME should be the toplevel diffcol folder
	43	$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
[27701]	44	if($isWin) {
	45	$ENV{'TASK_HOME'} =~ s@\/@\\@g;
	46	# need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
	47	# run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
	48	# At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
	49	# end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
	50	require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
	51	$ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
	52	}
[27621]	53	## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
	54
	55
[27678]	56	$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
	57
	58	# we'll be using BLAT to send mail attachments on Windows
[27687]	59	my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
	60	if($isWin && $use_blat && ! -e $blat) {
	61	print STDERR "\n***********************************\n";
	62	print STDERR "No blat.exe found in $blat.\n";
	63	print STDERR "Blat needed to send mail with attachments on Windows.\n";
	64	print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
	65	print STDERR "for your bit architecture and name the folder 'blat'\n";
	66	print STDERR "***********************************\n\n";
[27678]	67	$blat = 0;
	68	}
	69
	70
[27668]	71	$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
	72	$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
[27621]	73	$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
[27678]	74	$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
[27621]	75	##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
	76
	77	# control if an existing compiled greenstone is used
	78	# or, if one should be checked out, which revision to checkout from svn
	79	$ENV{'SVN_OPT_REV'} = "-r head";
[27656]	80	#$ENV{'GSDLHOME'}=
	81	#$ENV{'GSDL3SRCHOME'}=
[27621]	82
[27628]	83
[27677]	84	# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
	85	my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
[27628]	86
	87	#parse arguments
	88	my $action = "all";
	89	if(scalar(@ARGV) > 1) {
	90	&printusage();
	91	exit 0;
	92	}
	93
	94	if(scalar(@ARGV) == 0) {
	95	$action="all";
	96	}
	97	else {
	98	switch ($ARGV[0]) {
	99	case qr/^(-h\|-help\|help)$/i { &printusage; exit 0; }
	100	case qr/^(setup_greenstone\|run_test\|summarise\|upload\|all)$/ { $action=$ARGV[0]; }
	101	else {
	102	print STDERR "Bad subcommand.\n";
	103	&printusage;
	104	exit -1;
	105	}
	106	}
	107	}
	108
[27621]	109	#check key environment vars are set
	110	if(!defined $ENV{'UPLOAD_DIR'}) {
[27629]	111	print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
[27621]	112	#return 1;
	113	}
	114	if(!defined $ENV{'DATA_DIR'}) {
[27629]	115	print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
[27621]	116	#return 1;
	117	}
	118	if(!defined $ENV{'MONITOR_EMAIL'}) {
[27629]	119	print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
[27621]	120	#return 1;
	121	}
	122
	123	if($ENV{'DATA_DIR'} eq "/") {
[27629]	124	print STDERR "DATA_DIR should not be the fs root\n";
[27621]	125	#return 1;
	126	}
	127
[27629]	128	print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
	129	print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
[27621]	130
	131	#create an id for this test
	132	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	133	$year += 1900;
	134	$mon += 1;
	135	$mon = "0$mon" if ($mon < 10);
	136	$mday = "0$mday" if ($mday < 10);
[27702]	137	my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
[27621]	138
[27629]	139	print STDERR "Starting test '$dateid'\n";
[27621]	140
	141
	142	# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
	143	$ENV{'CLASSPATH'} = "";
	144	my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
	145	my @files = <$jar_lib_path/.jar>; # /full/path/to/diffcol/lib/jar
[27628]	146	foreach my $file (@files) {
[27668]	147	$file =~ s@\/@\\@g if $isWin;
[27621]	148	$ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
	149	}
	150	##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
	151
	152
	153	#set the location of the full report
	154	my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$dateid.xml");
	155	##print STDERR "XML: $xmlout\n";
	156
	157	# the toplevel folder of the greenstone installation being used
	158	my $greenstone_home="";
	159	# gsdl is the checkout folder and can be greenstone2 or greenstone3
	160	my $gsdl="greenstone2";
	161
	162
	163	# Check if using existing compiled-up greenstone installation
	164	# and set the greenstone_home location accordingly
	165
	166	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27629]	167	print STDERR "Found existing Greenstone home, will use that instead\n";
[27621]	168	$greenstone_home=$ENV{'GSDLHOME'};
	169	} else {
	170	$greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
	171	}
	172	##print STDERR "GSHOME: $greenstone_home\n";
	173
[27628]	174	#do the requested action
	175	if($action eq "setup_greenstone") {
	176	&setup_greenstone;
	177	}
	178	elsif ($action eq "run_test") {
	179	&run_test;
	180	}
	181	elsif ($action eq "summarise") {
	182	&summarise;
	183	}
	184	elsif ($action eq "upload") {
	185	&upload;
	186	&mail_with_report_attached;
	187	}
	188	elsif ($action eq "all") {
	189	&setup_greenstone;
	190	&run_test;
	191	&summarise;
	192	&upload;
	193	&mail_with_report_attached;
	194	}
[27621]	195
	196	##********************************
	197
[27628]	198	sub printusage
	199	{
	200	print STDERR "Run as: $0 (help\|setup_greenstone\|run_test\|summarise\|upload\|all)\n";
	201	}
[27621]	202
	203	#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
	204
	205	sub setup_greenstone
	206	{
	207	#clean up from previous tests
[27628]	208	print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
[27621]	209	for my $i ( 1..5 ) {
	210	sleep 1; # 1 second
[27628]	211	print STDERR ".";
[27621]	212	}
[27628]	213	print STDERR "\n";
[27621]	214
	215	# http://perldoc.perl.org/File/Path.html
[27629]	216	print STDERR "cleaning up previous tests\n";
[27621]	217	&File::Path::remove_tree($ENV{'DATA_DIR'});
	218
[27629]	219	print STDERR "creating the data dir\n";
[27621]	220	&File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
	221
	222	chdir($ENV{'DATA_DIR'});
	223
	224	# use existing compiled-up greenstone installation, if a GSDLHOME set
	225	if(defined $ENV{'GSDL3SRCHOME'} \|\| defined $ENV{'GSDLHOME'}) {
[27710]	226	print STDERR "Found existing Greenstone home, will use that instead\n";
[27621]	227	return;
	228	}
	229
	230	# Else checkout a GS from svn into DATA_DIR
	231
	232	#svn checkout of main gsdl directory
[27629]	233	print STDERR "checkout $gsdl:\n";
[27621]	234	my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone2 $gsdl";
	235	##print STDERR "Checkout CMD: $cmd\n";
	236
	237	# # unlike backticks operator, system() will print the output of the command to the screen as it executes
	238	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	239	my $status = system "$cmd"; #my $status = `$cmd`;
[27668]	240	if($status != 0) {
	241	print STDERR "@@@ SVN checkout of $gsdl failed\n";
	242	exit -1;
	243	}
[27629]	244	print STDERR "done\n";
[27621]	245
	246	##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
	247
	248	chdir("$ENV{'DATA_DIR'}$sep$gsdl");
	249
	250	##print STDERR "@@@ OS: $^O.\|".$Config{'archname64'}."\|\n";
[27764]	251
[27668]	252	if($isWin) {
	253	print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
	254
	255	# we're now in the GS2 folder, call makegs2 with silent param
	256	$cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
	257	$status = system $cmd;
	258
[27710]	259	} else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
	260
	261	my $bit_arch=`uname -m`;
	262
	263	# imagmagick binary
	264	print STDERR "Getting imagemagick binary\n";
[27724]	265
	266	my $os = $isMac ? "darwin" : "linux";
	267	my $imagickzip = "imagemagick-$os";
[27668]	268
[27710]	269	if($isMac) {
[27724]	270	$imagickzip .= "-10.5.tar.gz";
	271	# &File::Path::make_path($ENV{'DATA_DIR'}."$sep$gsdl$sep$os"); # need to ensure gsdl/bin/darwin exists
	272	# the following mac img bin contains binaries that points to libraries containing fixed paths to /home/max
	273	# $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
	274	# $status = system($cmd);
[27710]	275	} else { # linux
	276	my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
[27724]	277	$imagickzip .= "$extension64.tar.gz";
	278	}
[27710]	279
[27724]	280	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
	281	$status = system ($cmd);
	282	system("cd ext && tar -xvzf $imagickzip");
[27710]	283
	284	# gnomelib binary
[27629]	285	print STDERR "setting up gnome-lib-minimal for compilation\n";
[27621]	286
	287	# To get gnome-lib, need to determine bit architecture of the linux/darwin
	288	# http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
	289	# $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
	290	# well on ActivePerl.
	291	# But since we know we're on a Linux/Darwin machine at this point, wecan just run `uname -m` and other linux cmds
	292
[27710]	293	my $gnome_lib_file = $isMac ? "darwin-intel" : "linux"; # assuming all darwin is intel, not ppc!!
[27621]	294
	295	$gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/);
	296
	297	#svn checkout gnome-lib for this linux/darwin
	298	chdir("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
	299
	300	##print STDERR "**** gnomelib: $gnome_lib_file\n";
	301
	302	# checkout and unpack gnome-lib-minimal
	303
	304	#svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
	305	$cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
	306	system $cmd;
	307	system ("tar -xvzf gl.tar.gz");
	308
	309	chdir("gnome-lib-minimal");
	310	##print STDERR "*** ARCH: $bit_arch\n";
	311
	312	# need to run source devel.bash on gnome-lib followed by configure, make, make install
	313	# in one go, in order to preserve the compile environment set up by sourcing devel.bash
	314
	315	# http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
	316	# http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
	317
	318	# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
	319	$cmd = "bash -c \"";
	320
	321	$cmd .= "source ./devel.bash";
	322	$cmd .= " && cd ../..";
	323
	324	#configure
	325	# $cmd .= " && ./configure";
	326	$cmd .= " && echo 'configure $gsdl: ' ";
	327	$cmd .= " && echo '<configure>' >> $xmlout";
	328	$cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
	329	$cmd .= " && echo '</configure>' >> $xmlout";
	330	$cmd .= " && echo 'done'";
	331
	332	#make
	333	$cmd .= " && echo 'make $gsdl: '";
	334	$cmd .= " && echo '<make>' >> $xmlout";
	335	$cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
	336	$cmd .= " && echo '</make>' >> $xmlout";
	337	$cmd .= " && echo 'done'";
	338
	339	#make install
	340	$cmd .= " && echo 'make install $gsdl: '";
	341	$cmd .= " && echo '<make-install>' >> $xmlout";
	342	$cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
	343	$cmd .= " && echo '</make-install>' >> $xmlout";
	344	$cmd .= " && echo 'done'";
	345
	346	$cmd .= "\""; # close off cmd to bash and run it
[27668]	347	$status = system $cmd;
[27724]	348
	349
	350	# Moving imagemagick after instead of before compilation, since bin/darwin gets overwritten during compilation
	351	move("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick/$os", "$ENV{'DATA_DIR'}$sep$gsdl$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
	352	unlink "$ENV{'DATA_DIR'}/$gsdl"."/ext/$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
	353	&File::Path::remove_tree("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick"); # the untarred parent folder
	354
[27621]	355	}
[27668]	356
	357	if($status != 0) {
	358	print STDERR "@@@ Compile failed\n";
	359	exit -1;
	360	}
[27764]	361
[27621]	362	# set the path to the greenstone_home variable
	363	$greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
[27764]	364
	365	}
[27621]	366
[27764]	367	sub getPDFBox
	368	{
	369	# current revision is 27763, but using "head" works
	370	my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
	371	#"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
	372
	373	# now get the PDFBox extension for PDFBox tutorial
	374	print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
	375
	376	chdir($greenstone_home);
	377	my $cmd = "";
	378	if ($isWin) {
	379	$cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
	380
	381	} elsif ($isMac) { # need to use curl not wget
	382	$cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
	383	}
	384	else { # linux
	385	$cmd = "source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz";
	386	}
	387	my $status = system $cmd;
	388	if($status != 0) {
	389	print STDERR "@@@ Failed to set up PDFBox\n";
	390	#exit -1; # proceed testing other tutorials?
	391	}
[27621]	392	}
	393
	394	# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
	395	sub run_test
	396	{
[27764]	397	my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
	398	if(!-d $pdfbox) {
	399	&getPDFBox();
	400	}
	401
[27668]	402	open (my $xml_fh, '>'.$xmlout) \|\| die "Could not open xml file $xmlout for appending: $!\n";
[27621]	403
	404	# perform the requested subcommands, outputting xml information
	405	print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
	406
[27724]	407	# make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
	408
[27621]	409	print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
	410	my $cmd = "svn up $ENV{'TASK_HOME'}/model-collect"; #chdir("$ENV{'TASK_HOME'}/model-collect");
	411	my $status = system "$cmd";
	412
	413	# go to whichever greenstone_home we're using
	414	chdir($greenstone_home);
	415
	416	# get svn info
[27629]	417	print STDERR "getting svn info: $xmlout\n";
[27621]	418	print $xml_fh "<svn-info>\n";
[27668]	419	&run_and_print_cmd("svn info", $xml_fh);
[27621]	420	print $xml_fh "</svn-info>\n";
[27629]	421	print STDERR "done\n";
[27621]	422
	423	#make two copies of the model-collect directory in gsdl
	424	#one to be rebuilt and one as the basis for comparison
	425	#strip both of all .svn directories
	426
	427	#copy the model collections to the collect folder to be rebuilt
[27629]	428	print STDERR "installing test collections and model collections to new $gsdl installation... ";
[27621]	429
	430	#clean up
	431	if(-d "collect") {
	432	&File::Path::remove_tree("collect") \|\| die "Error could not delete collect: $!";
	433	}
	434	&File::Path::remove_tree("model-collect");
	435
	436	#copy to collect and strip .svn subfolders
	437	&File::Path::make_path("collect"); # create the folder and copy contents across
	438	&copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect"), "collect", ".svn");
	439
	440	#make the model copy
	441	&File::Path::make_path("model-collect");
	442	&copy_recursively("collect", "model-collect"); # copy contents across
	443
[27629]	444	print STDERR "done\n";
[27621]	445
	446
	447	#for each collection, import, build and diff with its model counterpart
	448	opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
	449	for my $collection (readdir $collect_handle) {
	450	next if ($collection eq "." \|\| $collection eq "..");
[27743]	451	# next if ($collection ne "Word-PDF-Basic"); ## TEMPORARY, FOR TESTING THIS SCRIPT
[27621]	452
	453	#escape the filename (in case of space)
	454	$collection =~ s@ @\\ @g;
	455	#getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
	456
	457	print STDERR "*** Found collection $collection\n";
	458	print $xml_fh "<collection-test name=\"$collection\">\n";
	459
	460	#import
	461	# Ensure the OIDtype for importing is hash_on_full_filename
	462	# "to make document identifiers more stable across upgrades of the software,
	463	# although it means that duplicate documents contained in the collection are
	464	# no longer detected automatically."
[27629]	465	print STDERR "$collection - Importing:\n";
[27621]	466	print $xml_fh "<import>\n";
[27701]	467	&run_build_script("import.pl -removeold $collection"); #-OIDtype hash_on_full_filename
[27621]	468	print $xml_fh "</import>\n";
[27629]	469	print STDERR "done\n";
[27621]	470
	471	#build
[27629]	472	print STDERR "$collection - Building:\n";
[27621]	473	print $xml_fh "<build>\n";
[27701]	474	&run_build_script("buildcol.pl -removeold $collection");
[27621]	475	print $xml_fh "</build>\n";
[27629]	476	print STDERR "done\n";
[27621]	477
	478	#rename the intermediate 'building' directory 'index'
[27629]	479	print STDERR "$collection - Move \"building\" to \"index\"... ";
[27621]	480	my $index = &filename_concat("collect", $collection, "index");
	481	my $building = &filename_concat("collect", $collection, "building");
	482	&File::Path::remove_tree($index);
	483	# Renaming Directories, http://www.perlmonks.org/?node_id=177421
	484	move($building, $index) or die "copy failed: $!"; # File::Copy::move
[27629]	485	print STDERR "done\n";
[27621]	486
	487	#diffcol
[27629]	488	print STDERR "$collection - Diffing:\n";
[27621]	489	my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
[27667]	490	$cmd = "diffcol.pl -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
[27621]	491	&run_diff_script($cmd, $xml_fh, $diffcol_dir);
	492
	493	chdir($greenstone_home); # this is actually where we are
[27629]	494	print STDERR "done\n";
[27621]	495	print $xml_fh "</collection-test>\n";
	496	}
	497	closedir $collect_handle; # close handle to collect dir
	498
	499	print $xml_fh "</test>\n";
	500	close($xml_fh);
	501
[27629]	502	print STDERR "done\n";
[27621]	503	}
	504
	505	##***************************************************************
	506	# runs setup in greenstone_home before running the diff command
	507	sub run_diff_script {
	508	my ($cmd, $fh, $diffcol_dir) = @_;
	509
	510	# we're in greenstone_home now
	511	if(!$isWin) {
[27667]	512	$cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && cd $diffcol_dir && ./$cmd\"";
[27621]	513
	514	} else { # Need to prefix cmd -c/-k as necessary
[27668]	515	$cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
	516	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	517	}
	518
	519	return &run_and_print_cmd($cmd, $fh);
	520	}
	521
	522	# runs setup in greenstone_home before running the given build command
	523	sub run_build_script {
	524	my ($cmd, $fh) = @_;
	525
	526	# chdir($greenstone_home);
[27687]	527	# we are in $greenstone_home already, can directly run the build cmd on the collection
[27621]	528	if(!$isWin) {
	529	$cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && $cmd\"";
	530
	531	} else { # Need to prefix cmd -c/-k as necessary
[27668]	532	$cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
[27621]	533	}
[27687]	534	## print STDERR "@@@@ Going to call command: $cmd\n";
[27621]	535
[27668]	536	return system($cmd);
[27629]	537	#return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
[27621]	538	}
	539
	540
	541	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
	542	# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
	543	sub run_and_print_cmd {
	544	my ($cmd, $fh) = @_;
	545
	546	open my $pin, "$cmd\|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-\|', 'powercfg -l') or die $!;
	547
	548	if(defined $fh) { # print cmd output both to the filehandle and to stdout
	549	while (my $line = <$pin>) {
	550	print $fh $line;
	551	# print STDOUT $line; # if also printing cmd output to STDOUT
	552	}
	553	}
	554	else { # no filehandle, so just need to print to stdout
	555
	556	# unlike backticks operator, system() will print the output of the command to the screen as it executes
	557	# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
	558
	559	my $status = system $cmd;
	560	if($status != 0) {
	561	print STDERR "ERROR ($status) running $cmd: $!\n";
	562	}
	563	}
	564	close($pin);
	565	}
	566
	567	sub filename_concat {
	568	my $first_file = shift(@_);
	569	my (@filenames) = @_;
	570
	571	# If first_file is not null or empty, then add it back into the list
	572	if (defined $first_file && $first_file =~ /\S/)
	573	{
	574	unshift(@filenames, $first_file);
	575	}
	576
	577	my $filename = join($sep, @filenames);
	578	$filename =~ s/[\\\/]$//; # remove trailing slashes if any
	579	return $filename;
	580	}
	581
	582
	583	# The following code is from
	584	# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
	585	# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
	586	sub copy_recursively {
	587	my ($from_dir, $to_dir, $regex) = @_;
	588	opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
	589
	590	# if(-d !$to_dir) {
	591	# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
	592	# }
	593
	594	for my $entry (readdir $dh) {
	595	next if ($entry eq "." \|\| $entry eq "..");
	596	next if (defined $regex && $entry =~ /$regex/);
	597	my $source = "$from_dir/$entry";
	598	my $destination = "$to_dir/$entry";
	599	if (-d $source) {
	600	mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
	601	copy_recursively($source, $destination, $regex);
	602	} else {
	603	copy($source, $destination) or die "copy failed: $!";
	604	}
	605	}
	606	closedir $dh;
	607	return;
	608	}
	609
	610	sub summarise {
	611
	612	# make a summarised Xml report
[27629]	613	print STDERR "Summarizing the xml report... ";
[27621]	614	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.xml";
	615	my $status = system($cmd);
[27629]	616	print STDERR "done\n";
[27621]	617
	618	# make a summarised HTMl report
[27629]	619	print STDERR "Creating an html summary report... ";
[27621]	620	$cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.html";
	621	$status = system($cmd);
[27629]	622	print STDERR "done\n";
[27621]	623	}
	624
	625	sub upload {
[27657]	626	# if the upload dir already existed, clear it of contents
	627	if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
	628	# don't want to keep previous days reports
	629	# else we will have to manually clear them at some point
	630	# just generate the set of reports for this run of task.pl upload
	631	# and
	632	&File::Path::remove_tree($ENV{'UPLOAD_DIR'});
	633	}
	634	# recreate the upload directory
	635	&File::Path::make_path($ENV{'UPLOAD_DIR'});
[27621]	636
	637	# copy all .xml and .html files across to UPLOAD_DIR
	638	opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
	639	for my $entry (readdir $dh) {
[27687]	640	next if ($entry !~ m/(\.xml\|\.html?)$/);
	641
	642	# copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
	643	# html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
[27694]	644	my $os_entry = $entry;
	645	$os_entry =~ s@\[email protected]@ if $isWin;
[27702]	646	$os_entry = $^O."-diffcol-$os_entry";
[27687]	647
	648	# get the absolute path to the original files before copying them over
	649	$entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
	650
	651	# copy them over with their new names
	652	## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
	653	copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
[27621]	654	}
	655	closedir $dh;
	656
[27637]	657
	658	# Upload the html file to puka
	659	#default identity dir
	660	if ( ! exists $ENV{'IDENTITY_DIR'} ) {
[27725]	661	$ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
[27637]	662	}
	663	if (! exists $ENV{'SNAPSHOT_MODE'} ) {
	664	$ENV{'SNAPSHOT_MODE'} = "caveat";
	665	}
	666
[27678]	667	#use the correct key for uploading
[27637]	668	$ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
	669	if(-f $ENV{'IDENTITY_FILE'}) {
[27678]	670	# if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
	671
[27694]	672	# the report we want to upload is actually just os-diffcol-report-$dateid.html
[27687]	673	my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c .htm \| "; #&& cat .html \| "; # && tar -c \|
[27637]	674	$command .= ($^O eq "MSWin32" ? "plink" : "ssh");
	675	$command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
	676	#print "$command\n";
	677	my $status = system("$command");
	678	if($status != 0) {
[27678]	679	print STDERR "*** Failed to upload test report to nzdl $status\n";
[27637]	680	}
	681	} else {
	682	print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
	683	}
	684
[27629]	685	print STDERR "Finished uploading\n";
[27621]	686	}
	687
	688	# Sending emails with perl: http://learn.perl.org/examples/email.html
	689	# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
[27628]	690	# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
	691	sub mail_with_report_attached
	692	{
	693	# email out with report attached, if the tests failed
[27629]	694	print STDERR "Checking if successful... \n";
[27628]	695	my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
	696	#my $result = system($cmd);
	697	my $result = `$cmd`;
[27621]	698
[27629]	699	print STDERR "result: $result\n";
[27621]	700
[27628]	701	if($result ne "yes") {
	702	my $msg = "$gsdl regression test for $dateid failed";
	703	my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
	704	my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$dateid.html");
	705
[27687]	706	if($isWin) {
	707	if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
[27678]	708	# http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
	709	#blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
[27628]	710
[27678]	711	# need to install blat on windows
	712	$cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
	713	$result = system($cmd);
	714	}
	715	else {
[27687]	716	$result = 1; # status from running mail command is 0 if success, 1 if fail
[27678]	717	print STDERR "********************************************\n";
[27687]	718	if ($use_blat) {
	719	print STDERR "Need blat and SMTP set to send mail attachment\n" ;
	720	} else {
	721	print STDERR "Not set up to send mail on Windows\n";
	722	}
[27678]	723	print STDERR "Inspect report at: $attach_file\n";
	724	print STDERR "********************************************\n";
	725	}
	726	} else { # linux
[27628]	727	my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
	728
	729	if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
	730	$cmd="echo '$gsdl regression test for $dateid failed.' \| mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
	731
	732	print STDERR "********************************************\n";
	733	print STDERR "No mutt installed, unable to mail attachment\n";
	734	print STDERR "Inspect report at: $attach_file\n";
	735	print STDERR "********************************************\n";
	736	} else {
	737	#$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
	738	$cmd = "echo '$gsdl regression test for $dateid failed' \| mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
	739	}
[27678]	740
	741	# run the mail command
	742	$result = system($cmd); #&run_and_print_cmd($cmd);
[27628]	743	}
	744
[27678]	745
[27637]	746	if($result != 0) {
	747	print STDERR "*** Unable to send email: $?\n";
	748	}
	749	else {
	750	print STDERR "Sent mail with report attached.\n";
	751	}
[27628]	752	} else {
	753	print STDERR "********************************************\n";
[27629]	754	print STDERR "Tests were successful. Not sending mail.\n";
[27628]	755	print STDERR "********************************************\n";
	756	}
	757	}
	758
[27621]	759	# The old version of this program contained the following, consisting of 1 line of active code:
	760
	761	# Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
	762	# Doing so will call this pl file and pass in "summarise" in ARGV
	763	# This pl file will in turn call the task executable in this folder
	764	# passing in "summarise" as a parameter.
	765	#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
	766
	767	##system("/bin/bash -c \"./task @ARGV\"");
	768	##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
[27694]	769

Note: See TracBrowser for help on using the repository browser.

Download in other formats: