source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 28114

Last change on this file since 28114 was 28114, checked in by ak19, 8 years ago

The run_test actions takes additional cmdline parameters. The first is nosvn, to reduce copying over time, especially useful on windows. The second is justdiff, which assumes the collections have already been built and you just want to diff them, which is handy when testing the diffing process. Further commandline parameters that work with run_test are any number of specific collections to run the test on, with or without one of nosvn and justdiff specified alongside. This then reduces the svn checkout and file copying and collection building to just the specified collections.

  • Property svn:executable set to *
File size: 32.8 KB
Line 
1#!/usr/bin/perl -w
2
3# This program is meant to run the nightly diffcol
4# It is meant to be an equivalent for the existing task bash script
5# But it is intended to be expanded to work for Windows and GS3
6# For windows, need to REMEMBER to set the correct shebangs at the top
7
8
9# TODO:
10# Have a caveat mode and a stable mode (as in snapshot/task.pl)
11#
12#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat|stable)" ) {
13# $major_version = 2;
14# $prefix="2t";
15# $rk="tk2"; # test kit
16#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat|stable)" ) {
17# $major_version = 3;
18# $prefix="3t";
19# $rk="tk3"; # test kit
20
21package diffcoltask;
22
23use Cwd;
24use Switch; # for switch(val) { case: ; ...}
25use File::Path; # for rmdir and mkdir type functions
26use File::Copy; # for recursive copying of folders but skipping .svn
27use File::Basename;
28
29use strict;
30no strict 'subs'; # allow barewords (eg STDERR) as function arguments
31
32
33my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
34my $isMac = ($^O =~ m/macos|darwin/i) ? 1 : 0;
35my $sep = $isWin ? "\\" : "/";
36my $pathsep = $isWin ? ";" : ":";
37#my $script_ext = $isWin ? ".bat" : ".bash";
38my $setup_script = "setup"; # needs to become gs3-setup for GS3
39my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
40
41my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
42my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
43
44# if use_local_rebuild is on, use_static_model should be on
45if ($use_local_rebuild && !$use_static_model) {
46 $use_static_model = 1;
47}
48
49
50# TASK_HOME should be the toplevel diffcol folder
51$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
52if($isWin) {
53 $ENV{'TASK_HOME'} =~ s@\/@\\@g;
54 # need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
55 # run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
56 # At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
57 # end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
58 require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
59 $ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
60}
61## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
62
63
64$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
65
66# we'll be using BLAT to send mail attachments on Windows
67my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
68if($isWin && $use_blat && ! -e $blat) {
69 print STDERR "\n***********************************\n";
70 print STDERR "No blat.exe found in $blat.\n";
71 print STDERR "Blat needed to send mail with attachments on Windows.\n";
72 print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
73 print STDERR "for your bit architecture and name the folder 'blat'\n";
74 print STDERR "***********************************\n\n";
75 $blat = 0;
76}
77
78
79$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
80$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
81$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
82$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
83##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
84
85# control if an existing compiled greenstone is used
86# or, if one should be checked out, which revision to checkout from svn
87$ENV{'SVN_OPT_REV'} = "-r head";
88#$ENV{'GSDLHOME'}=
89#$ENV{'GSDL3SRCHOME'}=
90
91
92# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
93my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
94
95#parse arguments
96my $action = "all";
97my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
98my @collections = ();
99
100if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
101 &printusage();
102 exit -1;
103}
104
105if(scalar(@ARGV) == 0) {
106 $action="all";
107}
108else {
109 switch ($ARGV[0]) {
110 case qr/^(-h|--?help|help)$/i { &printusage; exit 0; }
111 case qr/^(setup_greenstone|run_test|summarise|upload|all)$/ { $action=$ARGV[0]; }
112 else {
113 print STDERR "Bad subcommand.\n";
114 &printusage;
115 exit -1;
116 }
117 }
118
119 # run_test action can take a subaction: nosvn|justdiff.
120 # nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
121 # it out from the svn model-collect again.
122 # justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
123 if($action eq "run_test" && scalar(@ARGV) >= 2) {
124 push(@collections, @ARGV);
125 shift @collections; # remove action from array
126
127 if($ARGV[1] =~ m@^--@) {
128 shift @collections; # remove subaction from array
129
130 $subaction = $ARGV[1];
131 if($subaction eq "--justdiff") {
132 $use_local_rebuild = $use_static_model = 1;
133 } elsif ($subaction eq "--nosvn") {
134 $use_static_model = 1;
135 } else {
136 print STDERR "Bad subaction.\n";
137 &printusage;
138 exit -1;
139 }
140 }
141
142# foreach my $col (@collections) {
143# print STDERR "Collection: $col\n";
144# }
145 }
146}
147
148
149#check key environment vars are set
150if(!defined $ENV{'UPLOAD_DIR'}) {
151 print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
152 #return 1;
153}
154if(!defined $ENV{'DATA_DIR'}) {
155 print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
156 #return 1;
157}
158if(!defined $ENV{'MONITOR_EMAIL'}) {
159 print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
160 #return 1;
161}
162
163if($ENV{'DATA_DIR'} eq "/") {
164 print STDERR "DATA_DIR should not be the fs root\n";
165 #return 1;
166}
167
168print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
169print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
170
171#create an id for this test
172my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
173$year += 1900;
174$mon += 1;
175$mon = "0$mon" if ($mon < 10);
176$mday = "0$mday" if ($mday < 10);
177my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
178
179print STDERR "Starting test '$dateid'\n";
180
181
182# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
183$ENV{'CLASSPATH'} = "";
184my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
185my @files = <$jar_lib_path/*.jar>; # /full/path/to/diffcol/lib/*jar
186foreach my $file (@files) {
187 $file =~ s@\/@\\@g if $isWin;
188 $ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
189}
190##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
191
192
193#set the location of the full report
194my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$dateid.xml");
195##print STDERR "XML: $xmlout\n";
196
197# the toplevel folder of the greenstone installation being used
198my $greenstone_home="";
199# gsdl is the checkout folder and can be greenstone2 or greenstone3
200my $gsdl="greenstone2";
201
202
203# Check if using existing compiled-up greenstone installation
204# and set the greenstone_home location accordingly
205
206if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
207 print STDERR "Found existing Greenstone home, will use that instead\n";
208 $greenstone_home=$ENV{'GSDLHOME'};
209} else {
210 $greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
211}
212##print STDERR "GSHOME: $greenstone_home\n";
213
214#do the requested action
215if($action eq "setup_greenstone") {
216 &setup_greenstone;
217}
218elsif ($action eq "run_test") {
219 &run_test;
220}
221elsif ($action eq "summarise") {
222 &summarise;
223}
224elsif ($action eq "upload") {
225 &upload;
226 &mail_with_report_attached;
227}
228elsif ($action eq "all") {
229 &setup_greenstone;
230 &run_test;
231 &summarise;
232 &upload;
233 &mail_with_report_attached;
234}
235
236##********************************
237
238sub printusage
239{
240 print STDERR "Run as: $0 (help|setup_greenstone|run_test <--justdiff|--nosvn> <col1 col2 ...> |summarise|upload|all)\n";
241}
242
243#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
244
245sub setup_greenstone
246{
247 #clean up from previous tests
248 print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
249 for my $i ( 1..5 ) {
250 sleep 1; # 1 second
251 print STDERR ".";
252 }
253 print STDERR "\n";
254
255 # http://perldoc.perl.org/File/Path.html
256 print STDERR "cleaning up previous tests\n";
257 &File::Path::remove_tree($ENV{'DATA_DIR'});
258
259 print STDERR "creating the data dir\n";
260 &File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
261
262 chdir($ENV{'DATA_DIR'});
263
264 # use existing compiled-up greenstone installation, if a GSDLHOME set
265 if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
266 print STDERR "Found existing Greenstone home, will use that instead\n";
267 return;
268 }
269
270 # Else checkout a GS from svn into DATA_DIR
271
272 #svn checkout of main gsdl directory
273 print STDERR "checkout $gsdl:\n";
274 my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone2 $gsdl";
275 ##print STDERR "Checkout CMD: $cmd\n";
276
277 # # unlike backticks operator, system() will print the output of the command to the screen as it executes
278 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
279 my $status = system "$cmd"; #my $status = `$cmd`;
280 if($status != 0) {
281 print STDERR "@@@ SVN checkout of $gsdl failed\n";
282 exit -1;
283 }
284 print STDERR "done\n";
285
286 ##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
287
288 chdir("$ENV{'DATA_DIR'}$sep$gsdl");
289
290 ##print STDERR "@@@ OS: $^O.|".$Config{'archname64'}."|\n";
291
292 if($isWin) {
293 print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
294
295 # we're now in the GS2 folder, call makegs2 with silent param
296 $cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
297 $status = system $cmd;
298 if($status != 0) {
299 print STDERR "Greenstone compilation on Windows failed\n";
300 exit -1;
301 }
302
303 } else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
304
305 my $bit_arch=`uname -m`;
306
307 # imagmagick binary
308 print STDERR "Getting imagemagick binary\n";
309
310 my $os = $isMac ? "darwin" : "linux";
311 my $imagickzip = "imagemagick-$os";
312
313 if($isMac) {
314# $imagickzip .= "-10.5.tar.gz";
315 # at present, only the Imagemagick binaries created by Max for darwin work on the Macs
316 &File::Path::make_path($ENV{'DATA_DIR'}."$sep$gsdl$sep$os"); # need to ensure gsdl/bin/darwin exists
317 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
318 $status = system($cmd);
319 if($status != 0) {
320 print STDERR "@@@ Unable to get imagemagick for darwin\n";
321 }
322
323 # need ghostscript mac binary too for pdf to img conversions on mac
324 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
325 $status = system($cmd);
326 if($status != 0) {
327 print STDERR "@@@ Unable to get ghostscript for darwin\n";
328 }
329
330 # the imagemagick and ghostscript binaries have been set to executable on svn trac now
331# system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/imagemagick/bin/*");
332# system("chmod -R u+x $ENV{'DATA_DIR'}/$gsdl/bin/darwin/ghostscript/bin/*");
333 } else { # linux
334 my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
335 $imagickzip .= "$extension64.tar.gz";
336
337 # now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
338 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
339 $status = system ($cmd);
340 system("cd ext && tar -xvzf $imagickzip");
341 }
342
343 # gnomelib binary
344 print STDERR "setting up gnome-lib-minimal for compilation\n";
345
346 # To get gnome-lib, need to determine bit architecture of the linux/darwin
347 # http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
348 # $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
349 # well on ActivePerl.
350 # But since we know we're on a Linux/Darwin machine at this point, wecan just run `uname -m` and other linux cmds
351
352 my $gnome_lib_file = $isMac ? "darwin-intel" : "linux"; # assuming all darwin is intel, not ppc!!
353
354 $gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/);
355
356 #svn checkout gnome-lib for this linux/darwin
357 chdir("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
358
359 ##print STDERR "**** gnomelib: $gnome_lib_file\n";
360
361 # checkout and unpack gnome-lib-minimal
362
363 #svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
364 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
365 system $cmd;
366 system ("tar -xvzf gl.tar.gz");
367
368 chdir("gnome-lib-minimal");
369 ##print STDERR "*** ARCH: $bit_arch\n";
370
371 # need to run source devel.bash on gnome-lib followed by configure, make, make install
372 # in one go, in order to preserve the compile environment set up by sourcing devel.bash
373
374 # http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
375 # http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
376
377# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
378 $cmd = "bash -c \"";
379
380 $cmd .= "source ./devel.bash";
381 $cmd .= " && cd ../..";
382
383 #configure
384 # $cmd .= " && ./configure";
385 $cmd .= " && echo 'configure $gsdl: ' ";
386 $cmd .= " && echo '<configure>' >> $xmlout";
387 $cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
388 $cmd .= " && echo '</configure>' >> $xmlout";
389 $cmd .= " && echo 'done'";
390
391 #make
392 $cmd .= " && echo 'make $gsdl: '";
393 $cmd .= " && echo '<make>' >> $xmlout";
394 $cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
395 $cmd .= " && echo '</make>' >> $xmlout";
396 $cmd .= " && echo 'done'";
397
398 #make install
399 $cmd .= " && echo 'make install $gsdl: '";
400 $cmd .= " && echo '<make-install>' >> $xmlout";
401 $cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
402 $cmd .= " && echo '</make-install>' >> $xmlout";
403 $cmd .= " && echo 'done'";
404
405 $cmd .= "\""; # close off cmd to bash and run it
406 $status = system $cmd;
407
408 if(!$isMac) { # Linux
409 # Moving imagemagick after instead of before compilation, since bin/darwin gets overwritten during compilation
410 move("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick/$os", "$ENV{'DATA_DIR'}$sep$gsdl$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
411 unlink "$ENV{'DATA_DIR'}/$gsdl"."/ext/$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
412 &File::Path::remove_tree("$ENV{'DATA_DIR'}$sep$gsdl$sep"."ext/imagemagick"); # the untarred parent folder
413 }
414
415 }
416
417 if($status != 0) {
418 print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
419 exit -1;
420 }
421
422 # set the path to the greenstone_home variable
423 $greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
424
425}
426
427sub getPDFBox
428{
429 # current revision is 27763, but using "head" works
430 my $PDFBOX_TRAC_URL="http://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
431 #"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
432
433 # now get the PDFBox extension for PDFBox tutorial
434 print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
435
436 chdir($greenstone_home);
437 my $cmd = "";
438 if ($isWin) {
439 $cmd = "setup.bat && cd ext && wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
440
441 } elsif ($isMac) { # need to use curl not wget
442 $cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
443 }
444 else { # linux
445 $cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
446 }
447 my $status = system $cmd;
448 if($status != 0) {
449 print STDERR "@@@ Failed to set up PDFBox\n";
450 exit -1; # or proceed to testing other tutorials?
451 }
452}
453
454# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
455sub run_test
456{
457 my $num_cols = scalar(@collections); # remember the empty case
458
459 if($num_cols == 0) { # deal with all collections
460 push (@collections, "");
461 # putting the empty string in the array so that the "all colleections" case
462 # can be handled similarl to how the case of user-specified collections is handled
463
464 } else { # deal with user specified set of collections
465 # prefix the directory separator to each collection name
466 @collections = map { $sep.$_ } @collections;
467 }
468
469 my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
470 if(!-d $pdfbox) {
471 &getPDFBox();
472 }
473
474 open (my $xml_fh, '>'.$xmlout) || die "Could not open xml file $xmlout for appending: $!\n";
475
476 # perform the requested subcommands, outputting xml information
477 print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
478
479 my ($cmd, $status);
480 # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
481
482 if(!$use_local_rebuild) {
483 print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
484 for my $col (@collections) {
485 $cmd = "svn up $ENV{'TASK_HOME'}/model-collect$col"; #chdir("$ENV{'TASK_HOME'}/model-collect");
486 $status = system "$cmd";
487 }
488 }
489
490 # go to whichever greenstone_home we're using
491 chdir($greenstone_home);
492
493 # get svn info
494 print STDERR "getting svn info: $xmlout\n";
495 print $xml_fh "<svn-info>\n";
496 &run_and_print_cmd("svn info", $xml_fh);
497 print $xml_fh "</svn-info>\n";
498 print STDERR "done\n";
499
500 if(!$use_local_rebuild) {
501
502 #make two copies of the model-collect directory in gsdl
503 #one to be rebuilt and one as the basis for comparison
504 #strip both of all .svn directories
505
506 #copy the model collections to the collect folder to be rebuilt
507 print STDERR "installing test collections and model collections to new $gsdl installation... ";
508
509 #clean up
510 if(-d "collect") {
511 for my $col (@collections) {
512 &File::Path::remove_tree("collect$col") || die "Error could not delete collect: $!";
513 }
514 }
515
516 if($use_static_model) {
517 for my $col (@collections) {
518 #copy to collect and strip .svn subfolders
519 &File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
520 &copy_recursively("model-collect$col", "collect$col", ".svn");
521 }
522
523 } else { # the default situation: where we check out the model-collect from svn
524 for my $col (@collections) {
525 &File::Path::remove_tree("model-collect$col");
526
527 #copy to collect and strip .svn subfolders
528 &File::Path::make_path("collect$col"); # create the folder and copy contents across
529 &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect$col"), "collect$col", ".svn");
530
531 #make the model copy
532 &File::Path::make_path("model-collect$col");
533 &copy_recursively("collect$col", "model-collect$col"); # copy contents across
534 }
535 }
536
537 print STDERR "done\n";
538 }
539
540 #for each collection, import, build and diff with its model counterpart
541
542 # if working with all collections, read the list of collections from the folders in collect
543 if($num_cols == 0) {
544 @collections = (); # get rid of the empty string put in the array to represent "all collections"
545
546 opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
547
548 for my $collection (readdir $collect_handle) {
549 next if ($collection eq "." || $collection eq "..");
550 next if ($collection eq "modelcol");
551 push(@collections, $collection);
552 }
553 closedir $collect_handle; # close handle to collect dir
554 }
555
556 for my $collection (@collections) {
557
558 # next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
559 # next if ($collection !~ m/OAI|METS|DSpace|MGPP|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
560
561 #escape the filename (in case of space)
562 $collection =~ s@ @\\ @g;
563 #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
564
565 $collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
566
567 if (! -d "collect$sep$collection") {
568 print STDERR "Collection $collection does not exist\n";
569 next;
570 }
571
572 print STDERR "*** Found collection $collection\n";
573 print $xml_fh "<collection-test name=\"$collection\">\n";
574
575 if(!$use_local_rebuild) {
576 #import
577 # Ensure the OIDtype for importing is hash_on_full_filename
578 # "to make document identifiers more stable across upgrades of the software,
579 # although it means that duplicate documents contained in the collection are
580 # no longer detected automatically."
581 print STDERR "$collection - Importing:\n";
582 print $xml_fh "<import>\n";
583 &run_build_script("import.pl -removeold $collection"); #-OIDtype hash_on_full_filename
584 print $xml_fh "</import>\n";
585 print STDERR "done\n";
586
587 #build
588 print STDERR "$collection - Building:\n";
589 print $xml_fh "<build>\n";
590 &run_build_script("buildcol.pl -removeold $collection");
591 print $xml_fh "</build>\n";
592 print STDERR "done\n";
593
594 #rename the intermediate 'building' directory 'index'
595 print STDERR "$collection - Move \"building\" to \"index\"... ";
596 my $index = &filename_concat("collect", $collection, "index");
597 my $building = &filename_concat("collect", $collection, "building");
598 &File::Path::remove_tree($index);
599 # Renaming Directories, http://www.perlmonks.org/?node_id=177421
600 move($building, $index) or die "copy failed: $!"; # File::Copy::move
601 print STDERR "done\n";
602 }
603 #diffcol
604 print STDERR "$collection - Diffing:\n";
605 my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
606 $cmd = "diffcol.pl -output xml -verbosity 10 $collection"; # need to run with ./diffcol.pl if bash script
607 &run_diff_script($cmd, $xml_fh, $diffcol_dir);
608
609 chdir($greenstone_home); # this is actually where we are
610 print STDERR "done\n";
611 print $xml_fh "</collection-test>\n";
612 }
613
614 print $xml_fh "</test>\n";
615 close($xml_fh);
616
617 print STDERR "done\n";
618}
619
620##***************************************************************
621# runs setup in greenstone_home before running the diff command
622sub run_diff_script {
623 my ($cmd, $fh, $diffcol_dir) = @_;
624
625 # we're in greenstone_home now
626 if(!$isWin) {
627 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && cd $diffcol_dir && ./$cmd\"";
628
629 } else { # Need to prefix cmd -c/-k as necessary
630 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
631## print STDERR "@@@@ Going to call command: $cmd\n";
632 }
633
634 return &run_and_print_cmd($cmd, $fh);
635}
636
637# runs setup in greenstone_home before running the given build command
638sub run_build_script {
639 my ($cmd, $fh) = @_;
640
641# chdir($greenstone_home);
642 # we are in $greenstone_home already, can directly run the build cmd on the collection
643 if(!$isWin) {
644 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.bash && $cmd\"";
645
646 } else { # Need to prefix cmd -c/-k as necessary
647 $cmd = "cmd /c \"set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
648 }
649## print STDERR "@@@@ Going to call command: $cmd\n";
650
651 return system($cmd);
652 #return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
653}
654
655
656# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
657# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
658sub run_and_print_cmd {
659 my ($cmd, $fh) = @_;
660
661 open my $pin, "$cmd|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-|', 'powercfg -l') or die $!;
662
663 if(defined $fh) { # print cmd output both to the filehandle and to stdout
664 while (my $line = <$pin>) {
665 print $fh $line;
666# print STDOUT $line; # if also printing cmd output to STDOUT
667 }
668 }
669 else { # no filehandle, so just need to print to stdout
670
671 # unlike backticks operator, system() will print the output of the command to the screen as it executes
672 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
673
674 my $status = system $cmd;
675 if($status != 0) {
676 print STDERR "ERROR ($status) running $cmd: $!\n";
677 }
678 }
679 close($pin);
680}
681
682sub filename_concat {
683 my $first_file = shift(@_);
684 my (@filenames) = @_;
685
686 # If first_file is not null or empty, then add it back into the list
687 if (defined $first_file && $first_file =~ /\S/)
688 {
689 unshift(@filenames, $first_file);
690 }
691
692 my $filename = join($sep, @filenames);
693 $filename =~ s/[\\\/]$//; # remove trailing slashes if any
694 return $filename;
695}
696
697
698# The following code is from
699# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
700# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
701sub copy_recursively {
702 my ($from_dir, $to_dir, $regex) = @_;
703 opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
704
705# if(-d !$to_dir) {
706# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
707# }
708
709 for my $entry (readdir $dh) {
710 next if ($entry eq "." || $entry eq "..");
711 next if (defined $regex && $entry =~ /$regex/);
712 my $source = "$from_dir/$entry";
713 my $destination = "$to_dir/$entry";
714 if (-d $source) {
715 mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
716 copy_recursively($source, $destination, $regex);
717 } else {
718 copy($source, $destination) or die "copy failed: $!";
719 }
720 }
721 closedir $dh;
722 return;
723}
724
725sub summarise {
726
727 # make a summarised Xml report
728 print STDERR "Summarizing the xml report... ";
729 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.xml";
730 my $status = system($cmd);
731 print STDERR "done\n";
732
733 # make a summarised HTMl report
734 print STDERR "Creating an html summary report... ";
735 $cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$dateid.html";
736 $status = system($cmd);
737 print STDERR "done\n";
738
739 # Print whether the tests passed or failed
740 print STDERR "*******************************************\n";
741 print STDERR "Checking if successful... \n";
742 $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
743 $status = `$cmd`; #$status = system($cmd);
744 print STDERR "result: $status\n";
745 print STDERR "*******************************************\n";
746}
747
748sub upload {
749 # if the upload dir already existed, clear it of contents
750 if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
751 # don't want to keep previous days reports
752 # else we will have to manually clear them at some point
753 # just generate the set of reports for this run of task.pl upload
754 # and
755 &File::Path::remove_tree($ENV{'UPLOAD_DIR'});
756 }
757 # recreate the upload directory
758 &File::Path::make_path($ENV{'UPLOAD_DIR'});
759
760 # copy all *.xml and *.html files across to UPLOAD_DIR
761 opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
762 for my $entry (readdir $dh) {
763 next if ($entry !~ m/(\.xml|\.html?)$/);
764
765 # copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
766 # html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
767 my $os_entry = $entry;
768 $os_entry =~ s@\.html$@.htm@ if $isWin;
769 $os_entry = "diffcol-".$^O."-$os_entry";
770
771 # get the absolute path to the original files before copying them over
772 $entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
773
774 # copy them over with their new names
775## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
776 copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
777 }
778 closedir $dh;
779
780
781 # Upload the html file to puka
782 #default identity dir
783 if ( ! exists $ENV{'IDENTITY_DIR'} ) {
784 $ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
785 }
786 if (! exists $ENV{'SNAPSHOT_MODE'} ) {
787 $ENV{'SNAPSHOT_MODE'} = "caveat";
788 }
789
790 #use the correct key for uploading
791 $ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
792 if(-f $ENV{'IDENTITY_FILE'}) {
793 # if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
794
795 # the report we want to upload is actually just os-diffcol-report-$dateid.html
796 my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c *.htm* | "; #&& cat *.html | "; # && tar -c * |
797 $command .= ($^O eq "MSWin32" ? "plink" : "ssh");
798 $command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
799 #print "$command\n";
800 my $status = system("$command");
801 if($status != 0) {
802 print STDERR "*** Failed to upload test report to nzdl $status\n";
803 }
804 } else {
805 print STDERR "*** Cannot upload the test report to nzdl from this machine\n";
806 }
807
808 print STDERR "Finished uploading\n";
809}
810
811# Sending emails with perl: http://learn.perl.org/examples/email.html
812# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
813# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
814sub mail_with_report_attached
815{
816 # email out with report attached, if the tests failed
817 print STDERR "Checking if successful... \n";
818 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
819 #my $result = system($cmd);
820 my $result = `$cmd`;
821
822 print STDERR "result: $result\n";
823
824 if($result ne "yes") {
825 my $msg = "$gsdl regression test for $dateid failed";
826 my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
827 my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$dateid.html");
828
829 if($isWin) {
830 if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
831 # http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
832 #blat -to user@example.com -server smtp.example.com -f batch_script@example.com -subject "subject" -body "body"
833
834 # need to install blat on windows
835 $cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
836 $result = system($cmd);
837 }
838 else {
839 $result = 1; # status from running mail command is 0 if success, 1 if fail
840 print STDERR "********************************************\n";
841 if ($use_blat) {
842 print STDERR "Need blat and SMTP set to send mail attachment\n" ;
843 } else {
844 print STDERR "Not set up to send mail on Windows\n";
845 }
846 print STDERR "Inspect report at: $attach_file\n";
847 print STDERR "********************************************\n";
848 }
849 } else { # linux
850 my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
851
852 if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
853 $cmd="echo '$gsdl regression test for $dateid failed.' | mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
854
855 print STDERR "********************************************\n";
856 print STDERR "No mutt installed, unable to mail attachment\n";
857 print STDERR "Inspect report at: $attach_file\n";
858 print STDERR "********************************************\n";
859 } else {
860 #$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
861 $cmd = "echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
862 }
863
864 # run the mail command
865 $result = system($cmd); #&run_and_print_cmd($cmd);
866 }
867
868
869 if($result != 0) {
870 print STDERR "*** Unable to send email: $?\n";
871 }
872 else {
873 print STDERR "Sent mail with report attached.\n";
874 }
875 } else {
876 print STDERR "********************************************\n";
877 print STDERR "Tests were successful. Not sending mail.\n";
878 print STDERR "********************************************\n";
879 }
880}
881
882# The old version of this program contained the following, consisting of 1 line of active code:
883
884 # Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
885 # Doing so will call this pl file and pass in "summarise" in ARGV
886 # This pl file will in turn call the task executable in this folder
887 # passing in "summarise" as a parameter.
888#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
889
890 ##system("/bin/bash -c \"./task @ARGV\"");
891 ##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
892
Note: See TracBrowser for help on using the repository browser.