source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 27963

Last change on this file since 27963 was 27963, checked in by ak19, 11 years ago

Need Max's ghostscript binary with his imagemagick for darwin to convert pdfs to images on macs.

  • Property svn:executable set to *
File size: 28.9 KB
Line 
1#!/usr/bin/perl -w
2
3# This program is meant to run the nightly diffcol
4# It is meant to be an equivalent for the existing task bash script
5# But it is intended to be expanded to work for Windows and GS3
6# For windows, need to REMEMBER to set the correct shebangs at the top
7
8
9# TODO:
10# Have a caveat mode and a stable mode (as in snapshot/task.pl)
11#
12#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat|stable)" ) {
13# $major_version = 2;
14# $prefix="2t";
15# $rk="tk2"; # test kit
16#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat|stable)" ) {
17# $major_version = 3;
18# $prefix="3t";
19# $rk="tk3"; # test kit
20
21package diffcoltask;
22
23use Cwd;
24use Switch; # for switch(val) { case: ; ...}
25use File::Path; # for rmdir and mkdir type functions
26use File::Copy; # for recursive copying of folders but skipping .svn
27use File::Basename;
28
29use strict;
30no strict 'subs'; # allow barewords (eg STDERR) as function arguments
31
32
33my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
34my $isMac = ($^O =~ m/macos|darwin/i) ? 1 : 0;
35my $sep = $isWin ? "\\" : "/";
36my $pathsep = $isWin ? ";" : ":";
37#my $script_ext = $isWin ? ".bat" : ".bash";
38my $setup_script = "setup"; # needs to become gs3-setup for GS3
39my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
40
41
42# TASK_HOME should be the toplevel diffcol folder
43$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
44if($isWin) {
45 $ENV{'TASK_HOME'} =~ s@\/@\\@g;
46 # need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
47 # run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
48 # At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
49 # end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
50 require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
51 $ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
52}
53## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
54
55
56$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
57
58# we'll be using BLAT to send mail attachments on Windows
59my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
60if($isWin && $use_blat && ! -e $blat) {
61 print STDERR "\n***********************************\n";
62 print STDERR "No blat.exe found in $blat.\n";
63 print STDERR "Blat needed to send mail with attachments on Windows.\n";
64 print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
65 print STDERR "for your bit architecture and name the folder 'blat'\n";
66 print STDERR "***********************************\n\n";
67 $blat = 0;
68}
69
70
71$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
72$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
73$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
74$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
75##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
76
77# control if an existing compiled greenstone is used
78# or, if one should be checked out, which revision to checkout from svn
79$ENV{'SVN_OPT_REV'} = "-r head";
80#$ENV{'GSDLHOME'}=
81#$ENV{'GSDL3SRCHOME'}=
82
83
84# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
85my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
86
87#parse arguments
88my $action = "all";
89if(scalar(@ARGV) > 1) {
90 &printusage();
91 exit 0;
92}
93
94if(scalar(@ARGV) == 0) {
95 $action="all";
96}
97else {
98 switch ($ARGV[0]) {
99 case qr/^(-h|-help|help)$/i { &printusage; exit 0; }
100 case qr/^(setup_greenstone|run_test|summarise|upload|all)$/ { $action=$ARGV[0]; }
101 else {
102 print STDERR "Bad subcommand.\n";
103 &printusage;
104 exit -1;
105 }
106 }
107}
108
109#check key environment vars are set
110if(!defined $ENV{'UPLOAD_DIR'}) {
111 print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
112 #return 1;
113}
114if(!defined $ENV{'DATA_DIR'}) {
115 print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
116 #return 1;
117}
118if(!defined $ENV{'MONITOR_EMAIL'}) {
119 print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
120 #return 1;
121}
122
123if($ENV{'DATA_DIR'} eq "/") {
124 print STDERR "DATA_DIR should not be the fs root\n";
125 #return 1;
126}
127
128print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
129print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
130
131#create an id for this test
132my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
133$year += 1900;
134$mon += 1;
135$mon = "0$mon" if ($mon < 10);
136$mday = "0$mday" if ($mday < 10);
137my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
138
139print STDERR "Starting test '$dateid'\n";
140
141
142# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
143$ENV{'CLASSPATH'} = "";
144my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
145my @files = <$jar_lib_path/*.jar>; # /full/path/to/diffcol/lib/*jar
146foreach my $file (@files) {
147 $file =~ s@\/@\\@g if $isWin;
148 $ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
149}
150##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
151
152
153#set the location of the full report
154my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$dateid.xml");
155##print STDERR "XML: $xmlout\n";
156
157# the toplevel folder of the greenstone installation being used
158my $greenstone_home="";
159# gsdl is the checkout folder and can be greenstone2 or greenstone3
160my $gsdl="greenstone2";
161
162
163# Check if using existing compiled-up greenstone installation
164# and set the greenstone_home location accordingly
165
166if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
167 print STDERR "Found existing Greenstone home, will use that instead\n";
168 $greenstone_home=$ENV{'GSDLHOME'};
169} else {
170 $greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
171}
172##print STDERR "GSHOME: $greenstone_home\n";
173
174#do the requested action
175if($action eq "setup_greenstone") {
176 &setup_greenstone;
177}
178elsif ($action eq "run_test") {
179 &run_test;
180}
181elsif ($action eq "summarise") {
182 &summarise;
183}
184elsif ($action eq "upload") {
185 &upload;
186 &mail_with_report_attached;
187}
188elsif ($action eq "all") {
189 &setup_greenstone;
190 &run_test;
191 &summarise;
192 &upload;
193 &mail_with_report_attached;
194}
195
196##********************************
197
198sub printusage
199{
200 print STDERR "Run as: $0 (help|setup_greenstone|run_test|summarise|upload|all)\n";
201}
202
203#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
204
205sub setup_greenstone
206{
207 #clean up from previous tests
208 print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
209 for my $i ( 1..5 ) {
210 sleep 1; # 1 second
211 print STDERR ".";
212 }
213 print STDERR "\n";
214
215 # http://perldoc.perl.org/File/Path.html
216 print STDERR "cleaning up previous tests\n";
217 &File::Path::remove_tree($ENV{'DATA_DIR'});
218
219 print STDERR "creating the data dir\n";
220 &File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
221
222 chdir($ENV{'DATA_DIR'});
223
224 # use existing compiled-up greenstone installation, if a GSDLHOME set
225 if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
226 print STDERR "Found existing Greenstone home, will use that instead\n";
227 return;
228 }
229
230 # Else checkout a GS from svn into DATA_DIR
231
232 #svn checkout of main gsdl directory
233 print STDERR "checkout $gsdl:\n";
234 my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone2 $gsdl";
235 ##print STDERR "Checkout CMD: $cmd\n";
236
237 # # unlike backticks operator, system() will print the output of the command to the screen as it executes
238 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
239 my $status = system "$cmd"; #my $status = `$cmd`;
240 if($status != 0) {
241 print STDERR "@@@ SVN checkout of $gsdl failed\n";
242 exit -1;
243 }
244 print STDERR "done\n";
245
246 ##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
247
248 chdir("$ENV{'DATA_DIR'}$sep$gsdl");
249
250 ##print STDERR "@@@ OS: $^O.|".$Config{'archname64'}."|\n";
251
252 if($isWin) {
253 print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
254
255 # we're now in the GS2 folder, call makegs2 with silent param
256 $cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
257 $status = system $cmd;
258
259 } else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
260
261 my $bit_arch=`uname -m`;
262
263 # imagmagick binary
264 print STDERR "Getting imagemagick binary\n";
265
266 my $os = $isMac ? "darwin" : "linux";
267 my $imagickzip = "imagemagick-$os";
268
269 if($isMac) {
270# $imagickzip .= "-10.5.tar.gz";
271 # at present, only the Imagemagick binaries created by Max for darwin work on the Macs
272 &File::Path::make_path($ENV{'DATA_DIR'}."$sep$gsdl$sep$os"); # need to ensure gsdl/bin/darwin exists
273 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
274 $status = system($cmd);
275 if($status != 0) {
276 print STDERR "@@@ Unable to get imagemagick for darwin\n";
277 }
278
279 # need ghostscript mac binary too for pdf to img conversions on mac
280 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
281 $status = system($cmd);
282 if($status != 0) {
283 print STDERR "@@@ Unable to get ghostscript for darwin\n";
284 }
285
286 # need to set imagemagick and ghostscript binaries to executable
287 system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/imagemagick/bin/*");
288 system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/ghostscript/bin/*");
289 } else { # linux
290 my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
291 $imagickzip .= "$extension64.tar.gz";
292
293 # now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
294 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
295 $status = system ($cmd);
296 system("cd ext && tar -xvzf $imagickzip");
297 }
298
299 # gnomelib binary
300 print STDERR "setting up gnome-lib-minimal for compilation\n";
301
302 # To get gnome-lib, need to determine bit architecture of the linux/darwin
303 # http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
304 # $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
305 # well on ActivePerl.
306 # But since we know we're on a Linux/Darwin machine at this point, wecan just run `uname -m` and other linux cmds
307
308 my $gnome_lib_file = $isMac ? "darwin-intel" : "linux"; # assuming all darwin is intel, not ppc!!
309
310 $gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/);
311
312 #svn checkout gnome-lib for this linux/darwin
313 chdir("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
314
315 ##print STDERR "**** gnomelib: $gnome_lib_file\n";
316
317 # checkout and unpack gnome-lib-minimal
318
319 #svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
320 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
321 system $cmd;
322 system ("tar -xvzf gl.tar.gz");
323
324 chdir("gnome-lib-minimal");
325 ##print STDERR "*** ARCH: $bit_arch\n";
326
327 # need to run source devel.bash on gnome-lib followed by configure, make, make install
328 # in one go, in order to preserve the compile environment set up by sourcing devel.bash
329
330 # http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
331 # http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
332
333# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
334 $cmd = "bash -c \"";
335
336 $cmd .= "source ./devel.bash";
337 $cmd .= " && cd ../..";
338
339 #configure
340 # $cmd .= " && ./configure";
341 $cmd .= " && echo 'configure $gsdl: ' ";
342 $cmd .= " && echo '<configure>' >> $xmlout";
343 $cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
344 $cmd .= " && echo '</configure>' >> $xmlout";
345 $cmd .= " && echo 'done'";
346
347 #make
348 $cmd .= " && echo 'make $gsdl: '";
349 $cmd .= " && echo '<make>' >> $xmlout";
350 $cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
351 $cmd .= " && echo '</make>' >> $xmlout";
352 $cmd .= " && echo 'done'";
353
354 #make install
355 $cmd .= " && echo 'make install $gsdl: '";
356 $cmd .= " && echo '<make-install>' >> $xmlout";
357 $cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
358 $cmd .= " && echo '</make-install>' >> $xmlout";
359 $cmd .= " && echo 'done'";
360
361 $cmd .= "\""; # close off cmd to bash and run it
362 $status = system $cmd;
363
364 if(!$isMac) { # Linux
365 # Moving imagemagick after instead of before compilation, since bin/darwin gets overwritten during compilation
366 move("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick/$os", "$ENV{'DATA_DIR'}$sep$gsdl$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
367 unlink "$ENV{'DATA_DIR'}/$gsdl"."/ext/$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
368 &File::Path::remove_tree("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick"); # the untarred parent folder
369 }
370
371 }
372
373 if($status != 0) {
374 print STDERR "@@@ Compile failed\n";
375 exit -1;
376 }
377
378 # set the path to the greenstone_home variable
379 $greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
380
381}
382
383sub getPDFBox
384{
385 # current revision is 27763, but using "head" works
386 my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
387 #"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
388
389 # now get the PDFBox extension for PDFBox tutorial
390 print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
391
392 chdir($greenstone_home);
393 my $cmd = "";
394 if ($isWin) {
395 $cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
396
397 } elsif ($isMac) { # need to use curl not wget
398 $cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
399 }
400 else { # linux
401 $cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
402 }
403 my $status = system $cmd;
404 if($status != 0) {
405 print STDERR "@@@ Failed to set up PDFBox\n";
406 exit -1; # or proceed to testing other tutorials?
407 }
408}
409
410# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
411sub run_test
412{
413 my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
414 if(!-d $pdfbox) {
415 &getPDFBox();
416 }
417
418 open (my $xml_fh, '>'.$xmlout) || die "Could not open xml file $xmlout for appending: $!\n";
419
420 # perform the requested subcommands, outputting xml information
421 print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
422
423 # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
424
425 print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
426 my $cmd = "svn up $ENV{'TASK_HOME'}/model-collect"; #chdir("$ENV{'TASK_HOME'}/model-collect");
427 my $status = system "$cmd";
428
429 # go to whichever greenstone_home we're using
430 chdir($greenstone_home);
431
432 # get svn info
433 print STDERR "getting svn info: $xmlout\n";
434 print $xml_fh "<svn-info>\n";
435 &run_and_print_cmd("svn info", $xml_fh);
436 print $xml_fh "</svn-info>\n";
437 print STDERR "done\n";
438
439 #make two copies of the model-collect directory in gsdl
440 #one to be rebuilt and one as the basis for comparison
441 #strip both of all .svn directories
442
443 #copy the model collections to the collect folder to be rebuilt
444 print STDERR "installing test collections and model collections to new $gsdl installation... ";
445
446 #clean up
447 if(-d "collect") {
448 &File::Path::remove_tree("collect") || die "Error could not delete collect: $!";
449 }
450 &File::Path::remove_tree("model-collect");
451
452 #copy to collect and strip .svn subfolders
453 &File::Path::make_path("collect"); # create the folder and copy contents across
454 &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect"), "collect", ".svn");
455
456 #make the model copy
457 &File::Path::make_path("model-collect");
458 &copy_recursively("collect", "model-collect"); # copy contents across
459
460 print STDERR "done\n";
461
462
463 #for each collection, import, build and diff with its model counterpart
464 opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
465 for my $collection (readdir $collect_handle) {
466 next if ($collection eq "." || $collection eq "..");
467# next if ($collection ne "Word-PDF-Basic"); ## TEMPORARY, FOR TESTING THIS SCRIPT
468
469 #escape the filename (in case of space)
470 $collection =~ s@ @\\ @g;
471 #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
472
473 print STDERR "*** Found collection $collection\n";
474 print $xml_fh "<collection-test name=\"$collection\">\n";
475
476 #import
477# Ensure the OIDtype for importing is hash_on_full_filename
478# "to make document identifiers more stable across upgrades of the software,
479# although it means that duplicate documents contained in the collection are
480# no longer detected automatically."
481 print STDERR "$collection - Importing:\n";
482 print $xml_fh "<import>\n";
483 &run_build_script("import.pl -removeold $collection"); #-OIDtype hash_on_full_filename
484 print $xml_fh "</import>\n";
485 print STDERR "done\n";
486
487 #build
488 print STDERR "$collection - Building:\n";
489 print $xml_fh "<build>\n";
490 &run_build_script("buildcol.pl -removeold $collection");
491 print $xml_fh "</build>\n";
492 print STDERR "done\n";
493
494 #rename the intermediate 'building' directory 'index'
495 print STDERR "$collection - Move \"building\" to \"index\"... ";
496 my $index = &filename_concat("collect", $collection, "index");
497 my $building = &filename_concat("collect", $collection, "building");
498 &File::Path::remove_tree($index);
499 # Renaming Directories, http://www.perlmonks.org/?node_id=177421
500 move($building, $index) or die "copy failed: $!"; # File::Copy::move
501 print STDERR "done\n";
502
503 #diffcol
504 print STDERR "$collection - Diffing:\n";
505 my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
506 $cmd = "diffcol.pl -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
507 &run_diff_script($cmd, $xml_fh, $diffcol_dir);
508
509 chdir($greenstone_home); # this is actually where we are
510 print STDERR "done\n";
511 print $xml_fh "</collection-test>\n";
512 }
513 closedir $collect_handle; # close handle to collect dir
514
515 print $xml_fh "</test>\n";
516 close($xml_fh);
517
518 print STDERR "done\n";
519}
520
521##***************************************************************
522# runs setup in greenstone_home before running the diff command
523sub run_diff_script {
524 my ($cmd, $fh, $diffcol_dir) = @_;
525
526 # we're in greenstone_home now
527 if(!$isWin) {
528 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && cd $diffcol_dir && ./$cmd\"";
529
530 } else { # Need to prefix cmd -c/-k as necessary
531 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
532## print STDERR "@@@@ Going to call command: $cmd\n";
533 }
534
535 return &run_and_print_cmd($cmd, $fh);
536}
537
538# runs setup in greenstone_home before running the given build command
539sub run_build_script {
540 my ($cmd, $fh) = @_;
541
542# chdir($greenstone_home);
543 # we are in $greenstone_home already, can directly run the build cmd on the collection
544 if(!$isWin) {
545 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && $cmd\"";
546
547 } else { # Need to prefix cmd -c/-k as necessary
548 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
549 }
550## print STDERR "@@@@ Going to call command: $cmd\n";
551
552 return system($cmd);
553 #return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
554}
555
556
557# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
558# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
559sub run_and_print_cmd {
560 my ($cmd, $fh) = @_;
561
562 open my $pin, "$cmd|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-|', 'powercfg -l') or die $!;
563
564 if(defined $fh) { # print cmd output both to the filehandle and to stdout
565 while (my $line = <$pin>) {
566 print $fh $line;
567# print STDOUT $line; # if also printing cmd output to STDOUT
568 }
569 }
570 else { # no filehandle, so just need to print to stdout
571
572 # unlike backticks operator, system() will print the output of the command to the screen as it executes
573 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
574
575 my $status = system $cmd;
576 if($status != 0) {
577 print STDERR "ERROR ($status) running $cmd: $!\n";
578 }
579 }
580 close($pin);
581}
582
583sub filename_concat {
584 my $first_file = shift(@_);
585 my (@filenames) = @_;
586
587 # If first_file is not null or empty, then add it back into the list
588 if (defined $first_file && $first_file =~ /\S/)
589 {
590 unshift(@filenames, $first_file);
591 }
592
593 my $filename = join($sep, @filenames);
594 $filename =~ s/[\\\/]$//; # remove trailing slashes if any
595 return $filename;
596}
597
598
599# The following code is from
600# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
601# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
602sub copy_recursively {
603 my ($from_dir, $to_dir, $regex) = @_;
604 opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
605
606# if(-d !$to_dir) {
607# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
608# }
609
610 for my $entry (readdir $dh) {
611 next if ($entry eq "." || $entry eq "..");
612 next if (defined $regex && $entry =~ /$regex/);
613 my $source = "$from_dir/$entry";
614 my $destination = "$to_dir/$entry";
615 if (-d $source) {
616 mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
617 copy_recursively($source, $destination, $regex);
618 } else {
619 copy($source, $destination) or die "copy failed: $!";
620 }
621 }
622 closedir $dh;
623 return;
624}
625
626sub summarise {
627
628 # make a summarised Xml report
629 print STDERR "Summarizing the xml report... ";
630 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.xml";
631 my $status = system($cmd);
632 print STDERR "done\n";
633
634 # make a summarised HTMl report
635 print STDERR "Creating an html summary report... ";
636 $cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.html";
637 $status = system($cmd);
638 print STDERR "done\n";
639}
640
641sub upload {
642 # if the upload dir already existed, clear it of contents
643 if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
644 # don't want to keep previous days reports
645 # else we will have to manually clear them at some point
646 # just generate the set of reports for this run of task.pl upload
647 # and
648 &File::Path::remove_tree($ENV{'UPLOAD_DIR'});
649 }
650 # recreate the upload directory
651 &File::Path::make_path($ENV{'UPLOAD_DIR'});
652
653 # copy all *.xml and *.html files across to UPLOAD_DIR
654 opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
655 for my $entry (readdir $dh) {
656 next if ($entry !~ m/(\.xml|\.html?)$/);
657
658 # copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
659 # html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
660 my $os_entry = $entry;
661 $os_entry =~ s@\[email protected]@ if $isWin;
662 $os_entry = $^O."-diffcol-$os_entry";
663
664 # get the absolute path to the original files before copying them over
665 $entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
666
667 # copy them over with their new names
668## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
669 copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
670 }
671 closedir $dh;
672
673
674 # Upload the html file to puka
675 #default identity dir
676 if ( ! exists $ENV{'IDENTITY_DIR'} ) {
677 $ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
678 }
679 if (! exists $ENV{'SNAPSHOT_MODE'} ) {
680 $ENV{'SNAPSHOT_MODE'} = "caveat";
681 }
682
683 #use the correct key for uploading
684 $ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
685 if(-f $ENV{'IDENTITY_FILE'}) {
686 # if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
687
688 # the report we want to upload is actually just os-diffcol-report-$dateid.html
689 my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c *.htm* | "; #&& cat *.html | "; # && tar -c * |
690 $command .= ($^O eq "MSWin32" ? "plink" : "ssh");
691 $command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
692 #print "$command\n";
693 my $status = system("$command");
694 if($status != 0) {
695 print STDERR "*** Failed to upload test report to nzdl $status\n";
696 }
697 } else {
698 print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
699 }
700
701 print STDERR "Finished uploading\n";
702}
703
704# Sending emails with perl: http://learn.perl.org/examples/email.html
705# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
706# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
707sub mail_with_report_attached
708{
709 # email out with report attached, if the tests failed
710 print STDERR "Checking if successful... \n";
711 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
712 #my $result = system($cmd);
713 my $result = `$cmd`;
714
715 print STDERR "result: $result\n";
716
717 if($result ne "yes") {
718 my $msg = "$gsdl regression test for $dateid failed";
719 my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
720 my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$dateid.html");
721
722 if($isWin) {
723 if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
724 # http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
725 #blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
726
727 # need to install blat on windows
728 $cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
729 $result = system($cmd);
730 }
731 else {
732 $result = 1; # status from running mail command is 0 if success, 1 if fail
733 print STDERR "********************************************\n";
734 if ($use_blat) {
735 print STDERR "Need blat and SMTP set to send mail attachment\n" ;
736 } else {
737 print STDERR "Not set up to send mail on Windows\n";
738 }
739 print STDERR "Inspect report at: $attach_file\n";
740 print STDERR "********************************************\n";
741 }
742 } else { # linux
743 my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
744
745 if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
746 $cmd="echo '$gsdl regression test for $dateid failed.' | mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
747
748 print STDERR "********************************************\n";
749 print STDERR "No mutt installed, unable to mail attachment\n";
750 print STDERR "Inspect report at: $attach_file\n";
751 print STDERR "********************************************\n";
752 } else {
753 #$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
754 $cmd = "echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
755 }
756
757 # run the mail command
758 $result = system($cmd); #&run_and_print_cmd($cmd);
759 }
760
761
762 if($result != 0) {
763 print STDERR "*** Unable to send email: $?\n";
764 }
765 else {
766 print STDERR "Sent mail with report attached.\n";
767 }
768 } else {
769 print STDERR "********************************************\n";
770 print STDERR "Tests were successful. Not sending mail.\n";
771 print STDERR "********************************************\n";
772 }
773}
774
775# The old version of this program contained the following, consisting of 1 line of active code:
776
777 # Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
778 # Doing so will call this pl file and pass in "summarise" in ARGV
779 # This pl file will in turn call the task executable in this folder
780 # passing in "summarise" as a parameter.
781#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
782
783 ##system("/bin/bash -c \"./task @ARGV\"");
784 ##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
785
Note: See TracBrowser for help on using the repository browser.