source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 28172

Last change on this file since 28172 was 28172, checked in by ak19, 11 years ago

task.pl now takes a model_os param as argument to the run_test command and passes in the test_OS and model_OS variables to diffcol.pl. These specify on what Operating System the model collections were built and what the current OS is, on which the test collections will be built when the tests are run and diffed. This will provide a more foolproof way for diffcol to work out on what OS the model collection was generated and what the current OS is (or what OS task.pl chooses to specify as the current OS for test collections).
  • Property svn:executable set to *
File size: 34.2 KB
Line 
1#!/usr/bin/perl -w
2
3# This program is meant to run the nightly diffcol
4# It is meant to be an equivalent for the existing task bash script
5# But it is intended to be expanded to work for Windows and GS3
6# For windows, need to REMEMBER to set the correct shebangs at the top
7
8
9# TODO:
10# Have a caveat mode and a stable mode (as in snapshot/task.pl)
11#
12#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat|stable)" ) {
13# $major_version = 2;
14# $prefix="2t";
15# $rk="tk2"; # test kit
16#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat|stable)" ) {
17# $major_version = 3;
18# $prefix="3t";
19# $rk="tk3"; # test kit
20
21package diffcoltask;
22
23use Cwd;
24use Switch; # for switch(val) { case: ; ...}
25use File::Path; # for rmdir and mkdir type functions
26use File::Copy; # for recursive copying of folders but skipping .svn
27use File::Basename;
28
29use strict;
30no strict 'subs'; # allow barewords (eg STDERR) as function arguments
31
32
33my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
34my $isMac = ($^O =~ m/macos|darwin/i) ? 1 : 0;
35my $sep = $isWin ? "\\" : "/";
36my $pathsep = $isWin ? ";" : ":";
37#my $script_ext = $isWin ? ".bat" : ".bash";
38my $setup_script = "setup"; # needs to become gs3-setup for GS3
39my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
40
41my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
42my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
43
44# if use_local_rebuild is on, use_static_model should be on
45if ($use_local_rebuild && !$use_static_model) {
46 $use_static_model = 1;
47}
48
49my $test_os = $isWin ? "windows" : ($isMac ? "darwin" : "linux");
50my $model_os = "linux"; # default
51
52# TASK_HOME should be the toplevel diffcol folder
53$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
54if($isWin) {
55 $ENV{'TASK_HOME'} =~ s@\/@\\@g;
56 # need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
57 # run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
58 # At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
59 # end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
60 require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
61 $ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
62}
63## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
64
65
66$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
67
68# we'll be using BLAT to send mail attachments on Windows
69my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
70if($isWin && $use_blat && ! -e $blat) {
71 print STDERR "\n***********************************\n";
72 print STDERR "No blat.exe found in $blat.\n";
73 print STDERR "Blat needed to send mail with attachments on Windows.\n";
74 print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
75 print STDERR "for your bit architecture and name the folder 'blat'\n";
76 print STDERR "***********************************\n\n";
77 $blat = 0;
78}
79
80
81$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
82$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
83$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
84$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
85##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
86
87# control if an existing compiled greenstone is used
88# or, if one should be checked out, which revision to checkout from svn
89$ENV{'SVN_OPT_REV'} = "-r head";
90#$ENV{'GSDLHOME'}=
91#$ENV{'GSDL3SRCHOME'}=
92
93
94# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
95my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
96
97#parse arguments
98my $action = "all";
99my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
100my @collections = ();
101
102# run_test can take any number of args
103if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
104 print STDERR "**** Wrong number of arguments\n";
105 &printusage();
106 exit -1;
107}
108
109if(scalar(@ARGV) == 0) {
110 $action="all";
111}
112else {
113 switch ($ARGV[0]) {
114 case qr/^(-h|--?help|help)$/i { &printusage; exit 0; }
115 case qr/^(setup_greenstone|run_test|summarise|upload|all)$/ { $action=$ARGV[0]; }
116 else {
117 print STDERR "**** Bad subcommand.\n";
118 &printusage;
119 exit -1;
120 }
121 }
122
123 # run_test action can take a subaction: nosvn|justdiff. It can also take --modelOS (windows|linux|darwin)
124 # nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
125 # it out from the svn model-collect again.
126 # justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
127 if($action eq "run_test" && scalar(@ARGV) >= 2) {
128 push(@collections, @ARGV);
129 shift @collections; # remove action from array
130
131 for (my $i=0; $i < scalar(@ARGV); $i++) {
132 if($ARGV[$i] =~ m@^--@) {
133 shift @collections; # remove subaction/flag from array
134
135 $subaction = $ARGV[$i];
136 if($subaction eq "--justdiff") {
137 $use_local_rebuild = $use_static_model = 1;
138 } elsif ($subaction eq "--nosvn") {
139 $use_static_model = 1;
140 #} elsif ($subaction =~ m/\-\-testOS/i && defined $ARGV[$i+1]) {
141 # $test_os = $ARGV[$i+1];
142 # $i++;
143 # shift @collections; # remove test_os value from array
144 } elsif ($subaction =~ m/\-\-modelOS/i && defined $ARGV[$i+1] && $ARGV[$i+1] =~ m/windows|linux|darwin/i) {
145 $model_os = $ARGV[$i+1];
146 $i++;
147 shift @collections; # remove model_os value from array
148 #print STDERR "Model_os specified: $model_os\n";
149 } else {
150 print STDERR "**** Bad subaction/value: ".$ARGV[$i]."\n";
151 &printusage;
152 exit -1;
153 }
154 }
155 }
156
157# foreach my $col (@collections) {
158# print STDERR "Collection: $col\n";
159# }
160 }
161}
162
163
164#check key environment vars are set
165if(!defined $ENV{'UPLOAD_DIR'}) {
166 print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
167 #return 1;
168}
169if(!defined $ENV{'DATA_DIR'}) {
170 print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
171 #return 1;
172}
173if(!defined $ENV{'MONITOR_EMAIL'}) {
174 print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
175 #return 1;
176}
177
178if($ENV{'DATA_DIR'} eq "/") {
179 print STDERR "DATA_DIR should not be the fs root\n";
180 #return 1;
181}
182
183print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
184print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
185
186#create an id for this test
187my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
188$year += 1900;
189$mon += 1;
190$mon = "0$mon" if ($mon < 10);
191$mday = "0$mday" if ($mday < 10);
192my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
193
194print STDERR "Starting test '$dateid'\n";
195
196
197# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
198$ENV{'CLASSPATH'} = "";
199my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
200my @files = <$jar_lib_path/*.jar>; # /full/path/to/diffcol/lib/*jar
201foreach my $file (@files) {
202 $file =~ s@\/@\\@g if $isWin;
203 $ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
204}
205##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
206
207
208#set the location of the full report
209my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$dateid.xml");
210##print STDERR "XML: $xmlout\n";
211
212# the toplevel folder of the greenstone installation being used
213my $greenstone_home="";
214# gsdl is the checkout folder and can be greenstone2 or greenstone3
215my $gsdl="greenstone2";
216
217
218# Check if using existing compiled-up greenstone installation
219# and set the greenstone_home location accordingly
220
221if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
222 print STDERR "Found existing Greenstone home, will use that instead\n";
223 $greenstone_home=$ENV{'GSDLHOME'};
224} else {
225 $greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
226}
227##print STDERR "GSHOME: $greenstone_home\n";
228
229#do the requested action
230if($action eq "setup_greenstone") {
231 &setup_greenstone;
232}
233elsif ($action eq "run_test") {
234 &run_test;
235}
236elsif ($action eq "summarise") {
237 &summarise;
238}
239elsif ($action eq "upload") {
240 &upload;
241 &mail_with_report_attached;
242}
243elsif ($action eq "all") {
244 &setup_greenstone;
245 &run_test;
246 &summarise;
247 &upload;
248 &mail_with_report_attached;
249}
250
251##********************************
252
253sub printusage
254{
255# print STDERR "Run as: $0 (help|setup_greenstone|run_test <--modelOS windows|darwin|linux> <--justdiff|--nosvn> <col1 col2 ...> |summarise|upload|all)\n";
256 print STDERR "Run as: $0 (help|setup_greenstone|run_test|summarise|upload|all)\n";
257 print STDERR "where run_test can further take the following optional parameters:\n";
258 print STDERR "\t--modelOS (windows|darwin|linux)\n";
259 print STDERR "\t--justdiff|--nosvn\n";
260 print STDERR "\t<col1 col2 ...>\n";
261}
262
263#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
264
265sub setup_greenstone
266{
267 #clean up from previous tests
268 print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
269 for my $i ( 1..5 ) {
270 sleep 1; # 1 second
271 print STDERR ".";
272 }
273 print STDERR "\n";
274
275 # http://perldoc.perl.org/File/Path.html
276 print STDERR "cleaning up previous tests\n";
277 &File::Path::remove_tree($ENV{'DATA_DIR'});
278
279 print STDERR "creating the data dir\n";
280 &File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
281
282 chdir($ENV{'DATA_DIR'});
283
284 # use existing compiled-up greenstone installation, if a GSDLHOME set
285 if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
286 print STDERR "Found existing Greenstone home, will use that instead\n";
287 return;
288 }
289
290 # Else checkout a GS from svn into DATA_DIR
291
292 #svn checkout of main gsdl directory
293 print STDERR "checkout $gsdl:\n";
294 my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone2 $gsdl";
295 ##print STDERR "Checkout CMD: $cmd\n";
296
297 # # unlike backticks operator, system() will print the output of the command to the screen as it executes
298 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
299 my $status = system "$cmd"; #my $status = `$cmd`;
300 if($status != 0) {
301 print STDERR "@@@ SVN checkout of $gsdl failed\n";
302 exit -1;
303 }
304 print STDERR "done\n";
305
306 ##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
307
308 chdir("$ENV{'DATA_DIR'}$sep$gsdl");
309
310 ##print STDERR "@@@ OS: $^O.|".$Config{'archname64'}."|\n";
311
312 if($isWin) {
313 print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
314
315 # we're now in the GS2 folder, call makegs2 with silent param
316 $cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
317 $status = system $cmd;
318 if($status != 0) {
319 print STDERR "Greenstone compilation on Windows failed\n";
320 exit -1;
321 }
322
323 } else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
324
325 my $bit_arch=`uname -m`;
326
327 # imagmagick binary
328 print STDERR "Getting imagemagick binary\n";
329
330 my $os = $isMac ? "darwin" : "linux";
331 my $imagickzip = "imagemagick-$os";
332
333 if($isMac) {
334# $imagickzip .= "-10.5.tar.gz";
335 # at present, only the Imagemagick binaries created by Max for darwin work on the Macs
336 &File::Path::make_path($ENV{'DATA_DIR'}."$sep$gsdl$sep$os"); # need to ensure gsdl/bin/darwin exists
337 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
338 $status = system($cmd);
339 if($status != 0) {
340 print STDERR "@@@ Unable to get imagemagick for darwin\n";
341 }
342
343 # need ghostscript mac binary too for pdf to img conversions on mac
344 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
345 $status = system($cmd);
346 if($status != 0) {
347 print STDERR "@@@ Unable to get ghostscript for darwin\n";
348 }
349
350 # the imagemagick and ghostscript binaries have been set to executable on svn trac now
351# system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/imagemagick/bin/*");
352# system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/ghostscript/bin/*");
353 } else { # linux
354 my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
355 $imagickzip .= "$extension64.tar.gz";
356
357 # now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
358 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
359 $status = system ($cmd);
360 system("cd ext && tar -xvzf $imagickzip");
361 }
362
363 # gnomelib binary
364 print STDERR "setting up gnome-lib-minimal for compilation\n";
365
366 # To get gnome-lib, need to determine bit architecture of the linux/darwin
367 # http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
368 # $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
369 # well on ActivePerl.
370 # But since we know we're on a Linux/Darwin machine at this point, wecan just run `uname -m` and other linux cmds
371
372 my $gnome_lib_file = $isMac ? "darwin-intel" : "linux"; # assuming all darwin is intel, not ppc!!
373
374 $gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/);
375
376 #svn checkout gnome-lib for this linux/darwin
377 chdir("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
378
379 ##print STDERR "**** gnomelib: $gnome_lib_file\n";
380
381 # checkout and unpack gnome-lib-minimal
382
383 #svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
384 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
385 system $cmd;
386 system ("tar -xvzf gl.tar.gz");
387
388 chdir("gnome-lib-minimal");
389 ##print STDERR "*** ARCH: $bit_arch\n";
390
391 # need to run source devel.bash on gnome-lib followed by configure, make, make install
392 # in one go, in order to preserve the compile environment set up by sourcing devel.bash
393
394 # http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
395 # http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
396
397# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
398 $cmd = "bash -c \"";
399
400 $cmd .= "source ./devel.bash";
401 $cmd .= " && cd ../..";
402
403 #configure
404 # $cmd .= " && ./configure";
405 $cmd .= " && echo 'configure $gsdl: ' ";
406 $cmd .= " && echo '<configure>' >> $xmlout";
407 $cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
408 $cmd .= " && echo '</configure>' >> $xmlout";
409 $cmd .= " && echo 'done'";
410
411 #make
412 $cmd .= " && echo 'make $gsdl: '";
413 $cmd .= " && echo '<make>' >> $xmlout";
414 $cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
415 $cmd .= " && echo '</make>' >> $xmlout";
416 $cmd .= " && echo 'done'";
417
418 #make install
419 $cmd .= " && echo 'make install $gsdl: '";
420 $cmd .= " && echo '<make-install>' >> $xmlout";
421 $cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
422 $cmd .= " && echo '</make-install>' >> $xmlout";
423 $cmd .= " && echo 'done'";
424
425 $cmd .= "\""; # close off cmd to bash and run it
426 $status = system $cmd;
427
428 if(!$isMac) { # Linux
429 # Moving imagemagick after instead of before compilation, since bin/darwin gets overwritten during compilation
430 move("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick/$os", "$ENV{'DATA_DIR'}$sep$gsdl$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
431 unlink "$ENV{'DATA_DIR'}/$gsdl"."/ext/$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
432 &File::Path::remove_tree("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick"); # the untarred parent folder
433 }
434
435 }
436
437 if($status != 0) {
438 print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
439 exit -1;
440 }
441
442 # set the path to the greenstone_home variable
443 $greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
444
445}
446
447sub getPDFBox
448{
449 # current revision is 27763, but using "head" works
450 my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
451 #"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
452
453 # now get the PDFBox extension for PDFBox tutorial
454 print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
455
456 chdir($greenstone_home);
457 my $cmd = "";
458 if ($isWin) {
459 $cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
460
461 } elsif ($isMac) { # need to use curl not wget
462 $cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
463 }
464 else { # linux
465 $cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
466 }
467 my $status = system $cmd;
468 if($status != 0) {
469 print STDERR "@@@ Failed to set up PDFBox\n";
470 exit -1; # or proceed to testing other tutorials?
471 }
472}
473
474# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
475sub run_test
476{
477 my $num_cols = scalar(@collections); # remember the empty case
478
479 if($num_cols == 0) { # deal with all collections
480 push (@collections, "");
481 # putting the empty string in the array so that the "all colleections" case
482 # can be handled similarl to how the case of user-specified collections is handled
483
484 } else { # deal with user specified set of collections
485 # prefix the directory separator to each collection name
486 @collections = map { $sep.$_ } @collections;
487 }
488
489 my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
490 if(!-d $pdfbox) {
491 &getPDFBox();
492 }
493
494 open (my $xml_fh, '>'.$xmlout) || die "Could not open xml file $xmlout for appending: $!\n";
495
496 # perform the requested subcommands, outputting xml information
497 print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
498
499 my ($cmd, $status);
500 # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
501
502 if(!$use_local_rebuild) {
503 print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
504 for my $col (@collections) {
505 $cmd = "svn up $ENV{'TASK_HOME'}/model-collect$col"; #chdir("$ENV{'TASK_HOME'}/model-collect");
506 $status = system "$cmd";
507 }
508 }
509
510 # go to whichever greenstone_home we're using
511 chdir($greenstone_home);
512
513 # get svn info
514 print STDERR "getting svn info: $xmlout\n";
515 print $xml_fh "<svn-info>\n";
516 &run_and_print_cmd("svn info", $xml_fh);
517 print $xml_fh "</svn-info>\n";
518 print STDERR "done\n";
519
520 if(!$use_local_rebuild) {
521
522 #make two copies of the model-collect directory in gsdl
523 #one to be rebuilt and one as the basis for comparison
524 #strip both of all .svn directories
525
526 #copy the model collections to the collect folder to be rebuilt
527 print STDERR "installing test collections and model collections to new $gsdl installation... ";
528
529 #clean up
530 if(-d "collect") {
531 for my $col (@collections) {
532 if(-d "collect$col") {
533 &File::Path::remove_tree("collect$col") || die "Error could not delete collect: $!";
534 }
535 }
536 }
537
538 if($use_static_model) {
539 for my $col (@collections) {
540 #copy to collect and strip .svn subfolders
541 &File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
542 &copy_recursively("model-collect$col", "collect$col", ".svn");
543 }
544
545 } else { # the default situation: where we check out the model-collect from svn
546 for my $col (@collections) {
547 &File::Path::remove_tree("model-collect$col");
548
549 #copy to collect and strip .svn subfolders
550 &File::Path::make_path("collect$col"); # create the folder and copy contents across
551 &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect$col"), "collect$col", ".svn");
552
553 #make the model copy
554 &File::Path::make_path("model-collect$col");
555 &copy_recursively("collect$col", "model-collect$col"); # copy contents across
556 }
557 }
558
559 print STDERR "done\n";
560 }
561
562 #for each collection, import, build and diff with its model counterpart
563
564 # if working with all collections, read the list of collections from the folders in collect
565 if($num_cols == 0) {
566 @collections = (); # get rid of the empty string put in the array to represent "all collections"
567
568 opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
569
570 for my $collection (readdir $collect_handle) {
571 next if ($collection eq "." || $collection eq "..");
572 next if ($collection eq "modelcol");
573 push(@collections, $collection);
574 }
575 closedir $collect_handle; # close handle to collect dir
576 }
577
578 for my $collection (@collections) {
579
580 # next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
581 # next if ($collection !~ m/OAI|METS|DSpace|MGPP|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
582
583 #escape the filename (in case of space)
584 $collection =~ s@ @\\ @g;
585 #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
586
587 $collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
588
589 if (! -d "collect$sep$collection") {
590 print STDERR "Collection $collection does not exist\n";
591 next;
592 }
593
594 print STDERR "*** Found collection $collection\n";
595 print $xml_fh "<collection-test name=\"$collection\">\n";
596
597 if(!$use_local_rebuild) {
598 #import
599 # Ensure the OIDtype for importing is hash_on_full_filename
600 # "to make document identifiers more stable across upgrades of the software,
601 # although it means that duplicate documents contained in the collection are
602 # no longer detected automatically."
603 print STDERR "$collection - Importing:\n";
604 print $xml_fh "<import>\n";
605 &run_build_script("import.pl -removeold $collection"); #-OIDtype hash_on_full_filename
606 print $xml_fh "</import>\n";
607 print STDERR "done\n";
608
609 #build
610 print STDERR "$collection - Building:\n";
611 print $xml_fh "<build>\n";
612 &run_build_script("buildcol.pl -removeold $collection");
613 print $xml_fh "</build>\n";
614 print STDERR "done\n";
615
616 #rename the intermediate 'building' directory 'index'
617 print STDERR "$collection - Move \"building\" to \"index\"... ";
618 my $index = &filename_concat("collect", $collection, "index");
619 my $building = &filename_concat("collect", $collection, "building");
620 &File::Path::remove_tree($index);
621 # Renaming Directories, http://www.perlmonks.org/?node_id=177421
622 move($building, $index) or die "copy failed: $!"; # File::Copy::move
623 print STDERR "done\n";
624 }
625 #diffcol
626 print STDERR "$collection - Diffing:\n";
627 my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
628
629 # help diffcol to know on what os the model cols were generated
630 # and what os this test machine is (on which the test cols will be generated)
631 $cmd = "diffcol.pl -testos $test_os -modelos $model_os -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
632 &run_diff_script($cmd, $xml_fh, $diffcol_dir);
633
634 chdir($greenstone_home); # this is actually where we are
635 print STDERR "done\n";
636 print $xml_fh "</collection-test>\n";
637 }
638
639 print $xml_fh "</test>\n";
640 close($xml_fh);
641
642 print STDERR "done\n";
643}
644
645##***************************************************************
646# runs setup in greenstone_home before running the diff command
647sub run_diff_script {
648 my ($cmd, $fh, $diffcol_dir) = @_;
649
650 # we're in greenstone_home now
651 if(!$isWin) {
652 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && cd $diffcol_dir && ./$cmd\"";
653
654 } else { # Need to prefix cmd -c/-k as necessary
655 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
656## print STDERR "@@@@ Going to call command: $cmd\n";
657 }
658
659 return &run_and_print_cmd($cmd, $fh);
660}
661
662# runs setup in greenstone_home before running the given build command
663sub run_build_script {
664 my ($cmd, $fh) = @_;
665
666# chdir($greenstone_home);
667 # we are in $greenstone_home already, can directly run the build cmd on the collection
668 if(!$isWin) {
669 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && $cmd\"";
670
671 } else { # Need to prefix cmd -c/-k as necessary
672 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
673 }
674## print STDERR "@@@@ Going to call command: $cmd\n";
675
676 return system($cmd);
677 #return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
678}
679
680
681# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
682# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
683sub run_and_print_cmd {
684 my ($cmd, $fh) = @_;
685
686 open my $pin, "$cmd|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-|', 'powercfg -l') or die $!;
687
688 if(defined $fh) { # print cmd output both to the filehandle and to stdout
689 while (my $line = <$pin>) {
690 print $fh $line;
691# print STDOUT $line; # if also printing cmd output to STDOUT
692 }
693 }
694 else { # no filehandle, so just need to print to stdout
695
696 # unlike backticks operator, system() will print the output of the command to the screen as it executes
697 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
698
699 my $status = system $cmd;
700 if($status != 0) {
701 print STDERR "ERROR ($status) running $cmd: $!\n";
702 }
703 }
704 close($pin);
705}
706
707sub filename_concat {
708 my $first_file = shift(@_);
709 my (@filenames) = @_;
710
711 # If first_file is not null or empty, then add it back into the list
712 if (defined $first_file && $first_file =~ /\S/)
713 {
714 unshift(@filenames, $first_file);
715 }
716
717 my $filename = join($sep, @filenames);
718 $filename =~ s/[\\\/]$//; # remove trailing slashes if any
719 return $filename;
720}
721
722
723# The following code is from
724# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
725# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
726sub copy_recursively {
727 my ($from_dir, $to_dir, $regex) = @_;
728 opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
729
730# if(-d !$to_dir) {
731# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
732# }
733
734 for my $entry (readdir $dh) {
735 next if ($entry eq "." || $entry eq "..");
736 next if (defined $regex && $entry =~ /$regex/);
737 my $source = "$from_dir/$entry";
738 my $destination = "$to_dir/$entry";
739 if (-d $source) {
740 mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
741 copy_recursively($source, $destination, $regex);
742 } else {
743 copy($source, $destination) or die "copy failed: $!";
744 }
745 }
746 closedir $dh;
747 return;
748}
749
750sub summarise {
751
752 # make a summarised Xml report
753 print STDERR "Summarizing the xml report... ";
754 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.xml";
755 my $status = system($cmd);
756 print STDERR "done\n";
757
758 # make a summarised HTMl report
759 print STDERR "Creating an html summary report... ";
760 $cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.html";
761 $status = system($cmd);
762 print STDERR "done\n";
763
764 # Print whether the tests passed or failed
765 print STDERR "*******************************************\n";
766 print STDERR "Checking if successful... \n";
767 $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
768 $status = `$cmd`; #$status = system($cmd);
769 print STDERR "result: $status\n";
770 print STDERR "*******************************************\n";
771}
772
773sub upload {
774 # if the upload dir already existed, clear it of contents
775 if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
776 # don't want to keep previous days reports
777 # else we will have to manually clear them at some point
778 # just generate the set of reports for this run of task.pl upload
779 # and
780 &File::Path::remove_tree($ENV{'UPLOAD_DIR'});
781 }
782 # recreate the upload directory
783 &File::Path::make_path($ENV{'UPLOAD_DIR'});
784
785 # copy all *.xml and *.html files across to UPLOAD_DIR
786 opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
787 for my $entry (readdir $dh) {
788 next if ($entry !~ m/(\.xml|\.html?)$/);
789
790 # copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
791 # html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
792 my $os_entry = $entry;
793 $os_entry =~ s@\[email protected]@ if $isWin;
794 $os_entry = "diffcol-".$^O."-$os_entry";
795
796 # get the absolute path to the original files before copying them over
797 $entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
798
799 # copy them over with their new names
800## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
801 copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
802 }
803 closedir $dh;
804
805
806 # Upload the html file to puka
807 #default identity dir
808 if ( ! exists $ENV{'IDENTITY_DIR'} ) {
809 $ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
810 }
811 if (! exists $ENV{'SNAPSHOT_MODE'} ) {
812 $ENV{'SNAPSHOT_MODE'} = "caveat";
813 }
814
815 #use the correct key for uploading
816 $ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
817 if(-f $ENV{'IDENTITY_FILE'}) {
818 # if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
819
820 # the report we want to upload is actually just os-diffcol-report-$dateid.html
821 my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c *.htm* | "; #&& cat *.html | "; # && tar -c * |
822 $command .= ($^O eq "MSWin32" ? "plink" : "ssh");
823 $command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
824 #print "$command\n";
825 my $status = system("$command");
826 if($status != 0) {
827 print STDERR "*** Failed to upload test report to nzdl $status\n";
828 }
829 } else {
830 print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
831 }
832
833 print STDERR "Finished uploading\n";
834}
835
836# Sending emails with perl: http://learn.perl.org/examples/email.html
837# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
838# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
839sub mail_with_report_attached
840{
841 # email out with report attached, if the tests failed
842 print STDERR "Checking if successful... \n";
843 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
844 #my $result = system($cmd);
845 my $result = `$cmd`;
846
847 print STDERR "result: $result\n";
848
849 if($result ne "yes") {
850 my $msg = "$gsdl regression test for $dateid failed";
851 my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
852 my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$dateid.html");
853
854 if($isWin) {
855 if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
856 # http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
857 #blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
858
859 # need to install blat on windows
860 $cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
861 $result = system($cmd);
862 }
863 else {
864 $result = 1; # status from running mail command is 0 if success, 1 if fail
865 print STDERR "********************************************\n";
866 if ($use_blat) {
867 print STDERR "Need blat and SMTP set to send mail attachment\n" ;
868 } else {
869 print STDERR "Not set up to send mail on Windows\n";
870 }
871 print STDERR "Inspect report at: $attach_file\n";
872 print STDERR "********************************************\n";
873 }
874 } else { # linux
875 my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
876
877 if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
878 $cmd="echo '$gsdl regression test for $dateid failed.' | mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
879
880 print STDERR "********************************************\n";
881 print STDERR "No mutt installed, unable to mail attachment\n";
882 print STDERR "Inspect report at: $attach_file\n";
883 print STDERR "********************************************\n";
884 } else {
885 #$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
886 $cmd = "echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
887 }
888
889 # run the mail command
890 $result = system($cmd); #&run_and_print_cmd($cmd);
891 }
892
893
894 if($result != 0) {
895 print STDERR "*** Unable to send email: $?\n";
896 }
897 else {
898 print STDERR "Sent mail with report attached.\n";
899 }
900 } else {
901 print STDERR "********************************************\n";
902 print STDERR "Tests were successful. Not sending mail.\n";
903 print STDERR "********************************************\n";
904 }
905}
906
907# The old version of this program contained the following, consisting of 1 line of active code:
908
909 # Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
910 # Doing so will call this pl file and pass in "summarise" in ARGV
911 # This pl file will in turn call the task executable in this folder
912 # passing in "summarise" as a parameter.
913#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
914
915 ##system("/bin/bash -c \"./task @ARGV\"");
916 ##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
917
Note: See TracBrowser for help on using the repository browser.