source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 28106

Last change on this file since 28106 was 28106, checked in by ak19, 8 years ago

Stop diffcol task if compile fails.

  • Property svn:executable set to *
File size: 30.5 KB
Line 
1#!/usr/bin/perl -w
2
3# This program is meant to run the nightly diffcol
4# It is meant to be an equivalent for the existing task bash script
5# But it is intended to be expanded to work for Windows and GS3
6# For windows, need to REMEMBER to set the correct shebangs at the top
7
8
9# TODO:
10# Have a caveat mode and a stable mode (as in snapshot/task.pl)
11#
12#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat|stable)" ) {
13# $major_version = 2;
14# $prefix="2t";
15# $rk="tk2"; # test kit
16#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat|stable)" ) {
17# $major_version = 3;
18# $prefix="3t";
19# $rk="tk3"; # test kit
20
21package diffcoltask;
22
23use Cwd;
24use Switch; # for switch(val) { case: ; ...}
25use File::Path; # for rmdir and mkdir type functions
26use File::Copy; # for recursive copying of folders but skipping .svn
27use File::Basename;
28
29use strict;
30no strict 'subs'; # allow barewords (eg STDERR) as function arguments
31
32
33my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
34my $isMac = ($^O =~ m/macos|darwin/i) ? 1 : 0;
35my $sep = $isWin ? "\\" : "/";
36my $pathsep = $isWin ? ";" : ":";
37#my $script_ext = $isWin ? ".bat" : ".bash";
38my $setup_script = "setup"; # needs to become gs3-setup for GS3
39my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
40
41my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
42my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
43
44# if use_local_rebuild is on, use_static_model should be on
45if ($use_local_rebuild && !$use_static_model) {
46 $use_static_model = 1;
47}
48
49
50# TASK_HOME should be the toplevel diffcol folder
51$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
52if($isWin) {
53 $ENV{'TASK_HOME'} =~ s@\/@\\@g;
54 # need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
55 # run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
56 # At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
57 # end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
58 require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
59 $ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
60}
61## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
62
63
64$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
65
66# we'll be using BLAT to send mail attachments on Windows
67my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
68if($isWin && $use_blat && ! -e $blat) {
69 print STDERR "\n***********************************\n";
70 print STDERR "No blat.exe found in $blat.\n";
71 print STDERR "Blat needed to send mail with attachments on Windows.\n";
72 print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
73 print STDERR "for your bit architecture and name the folder 'blat'\n";
74 print STDERR "***********************************\n\n";
75 $blat = 0;
76}
77
78
79$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
80$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
81$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
82$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
83##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
84
85# control if an existing compiled greenstone is used
86# or, if one should be checked out, which revision to checkout from svn
87$ENV{'SVN_OPT_REV'} = "-r head";
88#$ENV{'GSDLHOME'}=
89#$ENV{'GSDL3SRCHOME'}=
90
91
92# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
93my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
94
95#parse arguments
96my $action = "all";
97if(scalar(@ARGV) > 1) {
98 &printusage();
99 exit 0;
100}
101
102if(scalar(@ARGV) == 0) {
103 $action="all";
104}
105else {
106 switch ($ARGV[0]) {
107 case qr/^(-h|-help|help)$/i { &printusage; exit 0; }
108 case qr/^(setup_greenstone|run_test|summarise|upload|all)$/ { $action=$ARGV[0]; }
109 else {
110 print STDERR "Bad subcommand.\n";
111 &printusage;
112 exit -1;
113 }
114 }
115}
116
117#check key environment vars are set
118if(!defined $ENV{'UPLOAD_DIR'}) {
119 print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
120 #return 1;
121}
122if(!defined $ENV{'DATA_DIR'}) {
123 print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
124 #return 1;
125}
126if(!defined $ENV{'MONITOR_EMAIL'}) {
127 print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
128 #return 1;
129}
130
131if($ENV{'DATA_DIR'} eq "/") {
132 print STDERR "DATA_DIR should not be the fs root\n";
133 #return 1;
134}
135
136print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
137print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
138
139#create an id for this test
140my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
141$year += 1900;
142$mon += 1;
143$mon = "0$mon" if ($mon < 10);
144$mday = "0$mday" if ($mday < 10);
145my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
146
147print STDERR "Starting test '$dateid'\n";
148
149
150# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
151$ENV{'CLASSPATH'} = "";
152my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
153my @files = <$jar_lib_path/*.jar>; # /full/path/to/diffcol/lib/*jar
154foreach my $file (@files) {
155 $file =~ s@\/@\\@g if $isWin;
156 $ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
157}
158##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
159
160
161#set the location of the full report
162my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$dateid.xml");
163##print STDERR "XML: $xmlout\n";
164
165# the toplevel folder of the greenstone installation being used
166my $greenstone_home="";
167# gsdl is the checkout folder and can be greenstone2 or greenstone3
168my $gsdl="greenstone2";
169
170
171# Check if using existing compiled-up greenstone installation
172# and set the greenstone_home location accordingly
173
174if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
175 print STDERR "Found existing Greenstone home, will use that instead\n";
176 $greenstone_home=$ENV{'GSDLHOME'};
177} else {
178 $greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
179}
180##print STDERR "GSHOME: $greenstone_home\n";
181
182#do the requested action
183if($action eq "setup_greenstone") {
184 &setup_greenstone;
185}
186elsif ($action eq "run_test") {
187 &run_test;
188}
189elsif ($action eq "summarise") {
190 &summarise;
191}
192elsif ($action eq "upload") {
193 &upload;
194 &mail_with_report_attached;
195}
196elsif ($action eq "all") {
197 &setup_greenstone;
198 &run_test;
199 &summarise;
200 &upload;
201 &mail_with_report_attached;
202}
203
204##********************************
205
206sub printusage
207{
208 print STDERR "Run as: $0 (help|setup_greenstone|run_test|summarise|upload|all)\n";
209}
210
211#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
212
213sub setup_greenstone
214{
215 #clean up from previous tests
216 print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
217 for my $i ( 1..5 ) {
218 sleep 1; # 1 second
219 print STDERR ".";
220 }
221 print STDERR "\n";
222
223 # http://perldoc.perl.org/File/Path.html
224 print STDERR "cleaning up previous tests\n";
225 &File::Path::remove_tree($ENV{'DATA_DIR'});
226
227 print STDERR "creating the data dir\n";
228 &File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
229
230 chdir($ENV{'DATA_DIR'});
231
232 # use existing compiled-up greenstone installation, if a GSDLHOME set
233 if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
234 print STDERR "Found existing Greenstone home, will use that instead\n";
235 return;
236 }
237
238 # Else checkout a GS from svn into DATA_DIR
239
240 #svn checkout of main gsdl directory
241 print STDERR "checkout $gsdl:\n";
242 my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone2 $gsdl";
243 ##print STDERR "Checkout CMD: $cmd\n";
244
245 # # unlike backticks operator, system() will print the output of the command to the screen as it executes
246 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
247 my $status = system "$cmd"; #my $status = `$cmd`;
248 if($status != 0) {
249 print STDERR "@@@ SVN checkout of $gsdl failed\n";
250 exit -1;
251 }
252 print STDERR "done\n";
253
254 ##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
255
256 chdir("$ENV{'DATA_DIR'}$sep$gsdl");
257
258 ##print STDERR "@@@ OS: $^O.|".$Config{'archname64'}."|\n";
259
260 if($isWin) {
261 print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
262
263 # we're now in the GS2 folder, call makegs2 with silent param
264 $cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
265 $status = system $cmd;
266 if($status != 0) {
267 print STDERR "Greenstone compilation on Windows failed\n";
268 exit -1;
269 }
270
271 } else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
272
273 my $bit_arch=`uname -m`;
274
275 # imagmagick binary
276 print STDERR "Getting imagemagick binary\n";
277
278 my $os = $isMac ? "darwin" : "linux";
279 my $imagickzip = "imagemagick-$os";
280
281 if($isMac) {
282# $imagickzip .= "-10.5.tar.gz";
283 # at present, only the Imagemagick binaries created by Max for darwin work on the Macs
284 &File::Path::make_path($ENV{'DATA_DIR'}."$sep$gsdl$sep$os"); # need to ensure gsdl/bin/darwin exists
285 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
286 $status = system($cmd);
287 if($status != 0) {
288 print STDERR "@@@ Unable to get imagemagick for darwin\n";
289 }
290
291 # need ghostscript mac binary too for pdf to img conversions on mac
292 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
293 $status = system($cmd);
294 if($status != 0) {
295 print STDERR "@@@ Unable to get ghostscript for darwin\n";
296 }
297
298 # the imagemagick and ghostscript binaries have been set to executable on svn trac now
299# system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/imagemagick/bin/*");
300# system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/ghostscript/bin/*");
301 } else { # linux
302 my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
303 $imagickzip .= "$extension64.tar.gz";
304
305 # now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
306 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
307 $status = system ($cmd);
308 system("cd ext && tar -xvzf $imagickzip");
309 }
310
311 # gnomelib binary
312 print STDERR "setting up gnome-lib-minimal for compilation\n";
313
314 # To get gnome-lib, need to determine bit architecture of the linux/darwin
315 # http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
316 # $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
317 # well on ActivePerl.
318 # But since we know we're on a Linux/Darwin machine at this point, wecan just run `uname -m` and other linux cmds
319
320 my $gnome_lib_file = $isMac ? "darwin-intel" : "linux"; # assuming all darwin is intel, not ppc!!
321
322 $gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/);
323
324 #svn checkout gnome-lib for this linux/darwin
325 chdir("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
326
327 ##print STDERR "**** gnomelib: $gnome_lib_file\n";
328
329 # checkout and unpack gnome-lib-minimal
330
331 #svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
332 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
333 system $cmd;
334 system ("tar -xvzf gl.tar.gz");
335
336 chdir("gnome-lib-minimal");
337 ##print STDERR "*** ARCH: $bit_arch\n";
338
339 # need to run source devel.bash on gnome-lib followed by configure, make, make install
340 # in one go, in order to preserve the compile environment set up by sourcing devel.bash
341
342 # http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
343 # http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
344
345# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
346 $cmd = "bash -c \"";
347
348 $cmd .= "source ./devel.bash";
349 $cmd .= " && cd ../..";
350
351 #configure
352 # $cmd .= " && ./configure";
353 $cmd .= " && echo 'configure $gsdl: ' ";
354 $cmd .= " && echo '<configure>' >> $xmlout";
355 $cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
356 $cmd .= " && echo '</configure>' >> $xmlout";
357 $cmd .= " && echo 'done'";
358
359 #make
360 $cmd .= " && echo 'make $gsdl: '";
361 $cmd .= " && echo '<make>' >> $xmlout";
362 $cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
363 $cmd .= " && echo '</make>' >> $xmlout";
364 $cmd .= " && echo 'done'";
365
366 #make install
367 $cmd .= " && echo 'make install $gsdl: '";
368 $cmd .= " && echo '<make-install>' >> $xmlout";
369 $cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
370 $cmd .= " && echo '</make-install>' >> $xmlout";
371 $cmd .= " && echo 'done'";
372
373 $cmd .= "\""; # close off cmd to bash and run it
374 $status = system $cmd;
375
376 if(!$isMac) { # Linux
377 # Moving imagemagick after instead of before compilation, since bin/darwin gets overwritten during compilation
378 move("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick/$os", "$ENV{'DATA_DIR'}$sep$gsdl$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
379 unlink "$ENV{'DATA_DIR'}/$gsdl"."/ext/$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
380 &File::Path::remove_tree("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick"); # the untarred parent folder
381 }
382
383 }
384
385 if($status != 0) {
386 print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
387 exit -1;
388 }
389
390 # set the path to the greenstone_home variable
391 $greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
392
393}
394
395sub getPDFBox
396{
397 # current revision is 27763, but using "head" works
398 my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
399 #"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
400
401 # now get the PDFBox extension for PDFBox tutorial
402 print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
403
404 chdir($greenstone_home);
405 my $cmd = "";
406 if ($isWin) {
407 $cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
408
409 } elsif ($isMac) { # need to use curl not wget
410 $cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
411 }
412 else { # linux
413 $cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
414 }
415 my $status = system $cmd;
416 if($status != 0) {
417 print STDERR "@@@ Failed to set up PDFBox\n";
418 exit -1; # or proceed to testing other tutorials?
419 }
420}
421
422# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
423sub run_test
424{
425 my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
426 if(!-d $pdfbox) {
427 &getPDFBox();
428 }
429
430 open (my $xml_fh, '>'.$xmlout) || die "Could not open xml file $xmlout for appending: $!\n";
431
432 # perform the requested subcommands, outputting xml information
433 print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
434
435 my $cmd;
436 # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
437
438 if(!$use_local_rebuild) {
439 print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
440 $cmd = "svn up $ENV{'TASK_HOME'}/model-collect"; #chdir("$ENV{'TASK_HOME'}/model-collect");
441 my $status = system "$cmd";
442 }
443
444 # go to whichever greenstone_home we're using
445 chdir($greenstone_home);
446
447 # get svn info
448 print STDERR "getting svn info: $xmlout\n";
449 print $xml_fh "<svn-info>\n";
450 &run_and_print_cmd("svn info", $xml_fh);
451 print $xml_fh "</svn-info>\n";
452 print STDERR "done\n";
453
454 #make two copies of the model-collect directory in gsdl
455 #one to be rebuilt and one as the basis for comparison
456 #strip both of all .svn directories
457
458 #copy the model collections to the collect folder to be rebuilt
459 print STDERR "installing test collections and model collections to new $gsdl installation... ";
460
461 if(!$use_local_rebuild) {
462 #clean up
463 if(-d "collect") {
464 &File::Path::remove_tree("collect") || die "Error could not delete collect: $!";
465 }
466
467 if($use_static_model) {
468 #copy to collect and strip .svn subfolders
469 &File::Path::make_path("collect"); # create the collect folder and copy contents from static model-collection across
470 &copy_recursively("model-collect", "collect", ".svn");
471
472 } else { # the default situation: where we check out the model-collect from svn
473 &File::Path::remove_tree("model-collect");
474
475 #copy to collect and strip .svn subfolders
476 &File::Path::make_path("collect"); # create the folder and copy contents across
477 &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect"), "collect", ".svn");
478
479 #make the model copy
480 &File::Path::make_path("model-collect");
481 &copy_recursively("collect", "model-collect"); # copy contents across
482 }
483
484 print STDERR "done\n";
485 }
486
487 #for each collection, import, build and diff with its model counterpart
488 opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
489 for my $collection (readdir $collect_handle) {
490 next if ($collection eq "." || $collection eq "..");
491 next if ($collection eq "modelcol");
492# next if ($collection ne "Section-Tagging"); ## TEMPORARY, FOR TESTING THIS SCRIPT
493# next if ($collection !~ m/MARC/); ## TEMPORARY, FOR TESTING THIS SCRIPT
494
495 #escape the filename (in case of space)
496 $collection =~ s@ @\\ @g;
497 #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
498
499 print STDERR "*** Found collection $collection\n";
500 print $xml_fh "<collection-test name=\"$collection\">\n";
501
502 if(!$use_local_rebuild) {
503 #import
504 # Ensure the OIDtype for importing is hash_on_full_filename
505 # "to make document identifiers more stable across upgrades of the software,
506 # although it means that duplicate documents contained in the collection are
507 # no longer detected automatically."
508 print STDERR "$collection - Importing:\n";
509 print $xml_fh "<import>\n";
510 &run_build_script("import.pl -removeold $collection"); #-OIDtype hash_on_full_filename
511 print $xml_fh "</import>\n";
512 print STDERR "done\n";
513
514 #build
515 print STDERR "$collection - Building:\n";
516 print $xml_fh "<build>\n";
517 &run_build_script("buildcol.pl -removeold $collection");
518 print $xml_fh "</build>\n";
519 print STDERR "done\n";
520
521 #rename the intermediate 'building' directory 'index'
522 print STDERR "$collection - Move \"building\" to \"index\"... ";
523 my $index = &filename_concat("collect", $collection, "index");
524 my $building = &filename_concat("collect", $collection, "building");
525 &File::Path::remove_tree($index);
526 # Renaming Directories, http://www.perlmonks.org/?node_id=177421
527 move($building, $index) or die "copy failed: $!"; # File::Copy::move
528 print STDERR "done\n";
529 }
530 #diffcol
531 print STDERR "$collection - Diffing:\n";
532 my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
533 $cmd = "diffcol.pl -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
534 &run_diff_script($cmd, $xml_fh, $diffcol_dir);
535
536 chdir($greenstone_home); # this is actually where we are
537 print STDERR "done\n";
538 print $xml_fh "</collection-test>\n";
539 }
540 closedir $collect_handle; # close handle to collect dir
541
542 print $xml_fh "</test>\n";
543 close($xml_fh);
544
545 print STDERR "done\n";
546}
547
548##***************************************************************
549# runs setup in greenstone_home before running the diff command
550sub run_diff_script {
551 my ($cmd, $fh, $diffcol_dir) = @_;
552
553 # we're in greenstone_home now
554 if(!$isWin) {
555 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && cd $diffcol_dir && ./$cmd\"";
556
557 } else { # Need to prefix cmd -c/-k as necessary
558 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
559## print STDERR "@@@@ Going to call command: $cmd\n";
560 }
561
562 return &run_and_print_cmd($cmd, $fh);
563}
564
565# runs setup in greenstone_home before running the given build command
566sub run_build_script {
567 my ($cmd, $fh) = @_;
568
569# chdir($greenstone_home);
570 # we are in $greenstone_home already, can directly run the build cmd on the collection
571 if(!$isWin) {
572 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && $cmd\"";
573
574 } else { # Need to prefix cmd -c/-k as necessary
575 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
576 }
577## print STDERR "@@@@ Going to call command: $cmd\n";
578
579 return system($cmd);
580 #return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
581}
582
583
584# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
585# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
586sub run_and_print_cmd {
587 my ($cmd, $fh) = @_;
588
589 open my $pin, "$cmd|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-|', 'powercfg -l') or die $!;
590
591 if(defined $fh) { # print cmd output both to the filehandle and to stdout
592 while (my $line = <$pin>) {
593 print $fh $line;
594# print STDOUT $line; # if also printing cmd output to STDOUT
595 }
596 }
597 else { # no filehandle, so just need to print to stdout
598
599 # unlike backticks operator, system() will print the output of the command to the screen as it executes
600 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
601
602 my $status = system $cmd;
603 if($status != 0) {
604 print STDERR "ERROR ($status) running $cmd: $!\n";
605 }
606 }
607 close($pin);
608}
609
610sub filename_concat {
611 my $first_file = shift(@_);
612 my (@filenames) = @_;
613
614 # If first_file is not null or empty, then add it back into the list
615 if (defined $first_file && $first_file =~ /\S/)
616 {
617 unshift(@filenames, $first_file);
618 }
619
620 my $filename = join($sep, @filenames);
621 $filename =~ s/[\\\/]$//; # remove trailing slashes if any
622 return $filename;
623}
624
625
626# The following code is from
627# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
628# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
629sub copy_recursively {
630 my ($from_dir, $to_dir, $regex) = @_;
631 opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
632
633# if(-d !$to_dir) {
634# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
635# }
636
637 for my $entry (readdir $dh) {
638 next if ($entry eq "." || $entry eq "..");
639 next if (defined $regex && $entry =~ /$regex/);
640 my $source = "$from_dir/$entry";
641 my $destination = "$to_dir/$entry";
642 if (-d $source) {
643 mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
644 copy_recursively($source, $destination, $regex);
645 } else {
646 copy($source, $destination) or die "copy failed: $!";
647 }
648 }
649 closedir $dh;
650 return;
651}
652
653sub summarise {
654
655 # make a summarised Xml report
656 print STDERR "Summarizing the xml report... ";
657 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.xml";
658 my $status = system($cmd);
659 print STDERR "done\n";
660
661 # make a summarised HTMl report
662 print STDERR "Creating an html summary report... ";
663 $cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.html";
664 $status = system($cmd);
665 print STDERR "done\n";
666
667 # Print whether the tests passed or failed
668 print STDERR "*******************************************\n";
669 print STDERR "Checking if successful... \n";
670 $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
671 $status = `$cmd`; #$status = system($cmd);
672 print STDERR "result: $status\n";
673 print STDERR "*******************************************\n";
674}
675
676sub upload {
677 # if the upload dir already existed, clear it of contents
678 if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
679 # don't want to keep previous days reports
680 # else we will have to manually clear them at some point
681 # just generate the set of reports for this run of task.pl upload
682 # and
683 &File::Path::remove_tree($ENV{'UPLOAD_DIR'});
684 }
685 # recreate the upload directory
686 &File::Path::make_path($ENV{'UPLOAD_DIR'});
687
688 # copy all *.xml and *.html files across to UPLOAD_DIR
689 opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
690 for my $entry (readdir $dh) {
691 next if ($entry !~ m/(\.xml|\.html?)$/);
692
693 # copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
694 # html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
695 my $os_entry = $entry;
696 $os_entry =~ s@\.html$@.htm@ if $isWin;
697 $os_entry = "diffcol-".$^O."-$os_entry";
698
699 # get the absolute path to the original files before copying them over
700 $entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
701
702 # copy them over with their new names
703## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
704 copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
705 }
706 closedir $dh;
707
708
709 # Upload the html file to puka
710 #default identity dir
711 if ( ! exists $ENV{'IDENTITY_DIR'} ) {
712 $ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
713 }
714 if (! exists $ENV{'SNAPSHOT_MODE'} ) {
715 $ENV{'SNAPSHOT_MODE'} = "caveat";
716 }
717
718 #use the correct key for uploading
719 $ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
720 if(-f $ENV{'IDENTITY_FILE'}) {
721 # if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
722
723 # the report we want to upload is actually just os-diffcol-report-$dateid.html
724 my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c *.htm* | "; #&& cat *.html | "; # && tar -c * |
725 $command .= ($^O eq "MSWin32" ? "plink" : "ssh");
726 $command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
727 #print "$command\n";
728 my $status = system("$command");
729 if($status != 0) {
730 print STDERR "*** Failed to upload test report to nzdl $status\n";
731 }
732 } else {
733 print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
734 }
735
736 print STDERR "Finished uploading\n";
737}
738
739# Sending emails with perl: http://learn.perl.org/examples/email.html
740# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
741# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
742sub mail_with_report_attached
743{
744 # email out with report attached, if the tests failed
745 print STDERR "Checking if successful... \n";
746 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
747 #my $result = system($cmd);
748 my $result = `$cmd`;
749
750 print STDERR "result: $result\n";
751
752 if($result ne "yes") {
753 my $msg = "$gsdl regression test for $dateid failed";
754 my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
755 my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$dateid.html");
756
757 if($isWin) {
758 if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
759 # http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
760 #blat -to user@example.com -server smtp.example.com -f batch_script@example.com -subject "subject" -body "body"
761
762 # need to install blat on windows
763 $cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
764 $result = system($cmd);
765 }
766 else {
767 $result = 1; # status from running mail command is 0 if success, 1 if fail
768 print STDERR "********************************************\n";
769 if ($use_blat) {
770 print STDERR "Need blat and SMTP set to send mail attachment\n" ;
771 } else {
772 print STDERR "Not set up to send mail on Windows\n";
773 }
774 print STDERR "Inspect report at: $attach_file\n";
775 print STDERR "********************************************\n";
776 }
777 } else { # linux
778 my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
779
780 if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
781 $cmd="echo '$gsdl regression test for $dateid failed.' | mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
782
783 print STDERR "********************************************\n";
784 print STDERR "No mutt installed, unable to mail attachment\n";
785 print STDERR "Inspect report at: $attach_file\n";
786 print STDERR "********************************************\n";
787 } else {
788 #$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
789 $cmd = "echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
790 }
791
792 # run the mail command
793 $result = system($cmd); #&run_and_print_cmd($cmd);
794 }
795
796
797 if($result != 0) {
798 print STDERR "*** Unable to send email: $?\n";
799 }
800 else {
801 print STDERR "Sent mail with report attached.\n";
802 }
803 } else {
804 print STDERR "********************************************\n";
805 print STDERR "Tests were successful. Not sending mail.\n";
806 print STDERR "********************************************\n";
807 }
808}
809
810# The old version of this program contained the following, consisting of 1 line of active code:
811
812 # Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
813 # Doing so will call this pl file and pass in "summarise" in ARGV
814 # This pl file will in turn call the task executable in this folder
815 # passing in "summarise" as a parameter.
816#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
817
818 ##system("/bin/bash -c \"./task @ARGV\"");
819 ##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
820
Note: See TracBrowser for help on using the repository browser.