source: other-projects/nightly-tasks/diffcol/trunk/task.pl@ 36655

Last change on this file since 36655 was 36655, checked in by anupama, 20 months ago

First stage of getting GS3 diffcol to work. 1. For now skipping absence of build.cfg and collect.cfg to fix any remaining issues. 2. The gs3-model-collect regenerated on Linux a few days back didn't contain the COLLNAME-inf-tmp.(gdb|jdb|lg) files whereas the test colls on this Windows machine do. So adding it to list of files to ignore because they only get generated on some OS. 3. buildConfig.xml contains metadata elements with lowercase m for metadata, and for buildConfig.xml need to ignore 2 more time-sensitive fields when diffing: buildDate and earliestDatestamp, which are now added to list of fieldnames to ignore, just as buildConfig.xml is added to list of files to process in ways somewhat similar to doc.xml (i.e. comparing some metadata fields, and ignoring time-sensitive ones or other arbitrary ones). 4. A fresh SVN checkout of GS3 shows it doesn't come with openoffice and doesn't use it for building GS3 model collections. And there's no libreoffice on this Windows diffcol test machine (where the test-colls are built and diffed against model-colls prepared on linux), so the open-office extension can not be used on here anyway. 4. With these settings, but see the potentially big hack in step 1, all but Word-PDF-Enhanced collection pass diffcol testing. The Word-PDF-Enhanced collection is the windows_scripting collection and so the model-collection should have been regenerated on Windows, as it originally was for this collection (see commit r30029).
  • Property svn:executable set to *
File size: 44.8 KB
Line 
1#!/usr/bin/perl -w
2
3# This program is meant to run the nightly diffcol
4# It is meant to be an equivalent for the existing task bash script
5# But it is intended to be expanded to work for Windows and GS3
6# For windows, need to REMEMBER to set the correct shebangs at the top
7
8
9# TODO:
10# Have a caveat mode and a stable mode (as in snapshot/task.pl)
11#
12#} elsif ( $ENV{'TASK_NAME'} =~ "gs2-diffcol-(caveat|stable)" ) {
13# $major_version = 2;
14# $prefix="2t";
15# $rk="tk2"; # test kit
16#} elsif ( $ENV{'TASK_NAME'} =~ "gs3-diffcol-(caveat|stable)" ) {
17# $major_version = 3;
18# $prefix="3t";
19# $rk="tk3"; # test kit
20
21package diffcoltask;
22
23use Cwd;
24#use Switch; # for switch(val) { case: ; ...}
25use File::Path; # for rmdir and mkdir type functions
26use File::Copy; # for recursive copying of folders but skipping .svn
27use File::Basename;
28
29use strict;
30no strict 'subs'; # allow barewords (eg STDERR) as function arguments
31
32
33my $isWin = ($^O =~ m/mswin/i) ? 1 : 0;
34my $isMac = ($^O =~ m/macos|darwin/i) ? 1 : 0;
35
36my $osversion="";
37# Need to get the correct gnome-lib-minimal for the OS
38# darwin11* Lion, darwin12* Mountain Lion, darwin9* and darwin10* are Leopard and Snow Leopard
39if ($^O eq "darwin") {
40 $osversion=`uname -r`; # e.g. 12.x.x
41 #$osversion =~ s@\..*$@@; # e.g.12
42 $osversion = ($osversion =~ m@^1[1-9](\.)?@i) ? "Lion-" : "";
43}
44
45# we're using a GS compiled up wget with https support (at release-kits/bin, on PATH)
46# But we may need to use --no-check-certificate for some https urls.
47my $wget = "wget --no-check-certificate";
48
49my $sep = $isWin ? "\\" : "/";
50my $pathsep = $isWin ? ";" : ":";
51#my $script_ext = $isWin ? ".bat" : ".bash";
52my $setup_script = "setup"; # will become gs3-setup for GS3
53my $use_blat = 0; # if we ever get blat to send mail/attachments on Windows working, set this to 1
54
55my $install_type = "svn";
56my $install_version = "2";
57
58my $use_local_rebuild = 0; # set to 1 (true) if just diffing and so we needn't copy model-collection over to the test collection again nor rebuild it (This is useful when having built the collection locally once before)
59my $use_static_model = 0; # set to 1 (true) if working with a non-svn model-collection. Defaults to 1 if $use_local_rebuild is turned on
60
61# if use_local_rebuild is on, use_static_model should be on
62if ($use_local_rebuild && !$use_static_model) {
63 $use_static_model = 1;
64}
65
66my $test_os = $isWin ? "windows" : ($isMac ? "darwin" : "linux");
67my $model_os = "linux"; # default
68
69my $debugging = 0;
70
71# TASK_HOME should be the toplevel diffcol folder
72$ENV{'TASK_HOME'} = getcwd unless defined $ENV{'TASK_HOME'};
73if($isWin) {
74 $ENV{'TASK_HOME'} =~ s@\/@\\@g;
75 # need to convert TASK_HOME path name to resolve very subtle bug when running task.pl via
76 # run-gs2-diffcol.bat which uses environment.pl's TASK_HOME setting via envi
77 # At that point TASK_HOME is already defined but ends up lowercase, so that entries in archiveinf-doc
78 # end up sorted differently when db2txt -sort is applied compared to if TASK_HOME had kept its case.
79 require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
80 $ENV{'TASK_HOME'} = &Win32::GetLongPathName($ENV{'TASK_HOME'});
81}
82## print STDERR "@@@ TASK_HOME: ".$ENV{'TASK_HOME'}."\n";
83
84
85$ENV{'BIN_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "bin");
86
87# we'll be using BLAT to send mail attachments on Windows
88my $blat = $use_blat ? &filename_concat($ENV{'BIN_DIR'}, "blat", "full", "blat.exe") : 0;
89if($isWin && $use_blat && ! -e $blat) {
90 print STDERR "\n***********************************\n";
91 print STDERR "No blat.exe found in $blat.\n";
92 print STDERR "Blat needed to send mail with attachments on Windows.\n";
93 print STDERR "Extract the blat zip file found in $ENV{'BIN_DIR'}\n";
94 print STDERR "for your bit architecture and name the folder 'blat'\n";
95 print STDERR "***********************************\n\n";
96 $blat = 0;
97}
98
99
100$ENV{'DATA_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-data");
101$ENV{'UPLOAD_DIR'} = &filename_concat($ENV{'TASK_HOME'}, "diffcol-reports");
102$ENV{'MONITOR_EMAIL'} = "greenstone_team\@cs.waikato.ac.nz"; # need to escape @ sign
103$ENV{'GSDL_SMTP'} = ""; #"smtp.gmail.com";
104##print STDERR "@@@ email: ".$ENV{'MONITOR_EMAIL'}."\n";
105
106# When installing diffcol the first time, force the user to create the data-dir and upload-dir
107die "\n@@@@ data dir: ".$ENV{'DATA_DIR'}." does not exist\n" unless (-d $ENV{'DATA_DIR'});
108die "\n@@@@ reports/upload dir: ".$ENV{'UPLOAD_DIR'}." does not exist\n" unless (-d $ENV{'UPLOAD_DIR'});
109
110# control if an existing compiled greenstone is used
111# or, if one should be checked out, which revision to checkout from svn
112$ENV{'SVN_OPT_REV'} = "-r head";
113#$ENV{'GSDLHOME'}=
114#$ENV{'GSDL3SRCHOME'}=
115
116
117# if the first arg is a digit, it's the new envi verbosity param. Take it off the array
118my $envi_verbose = shift(@ARGV) if(exists $ARGV[0] && $ARGV[0] =~ m/^\d+$/);
119
120#parse arguments
121my $action = "all";
122my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
123my @collections = (); # list of collections that run_test should process
124
125if(scalar(@ARGV) == 0) {
126 $action="all";
127}
128
129# process any arguments that are --gs2|--gs3 and --bin|--svn, and delete them from the array
130# if none provided, it's gs2 and svn by default.
131for (my $i = $#ARGV; $i >= 0; --$i) {
132 if($ARGV[$i] =~ m/--(bin|svn)/) {
133 $install_type = $1;
134 splice @ARGV, $i, 1; # remove the element from the argument array
135 } elsif($ARGV[$i] =~ m/--gs(2|3)/) {
136 $install_version = $1;
137 $setup_script = $install_version eq "3" ? "gs3-setup" : "setup"; # needs to become gs3-setup for GS3
138 splice @ARGV, $i, 1; # remove the element from the argument array
139 }
140}
141
142$ENV{'GSVERSION'} = $install_version;
143
144# run_test can take any number of args
145if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
146 print STDERR "**** Wrong number of arguments\n";
147 &printusage();
148 exit -1;
149}
150
151if(scalar(@ARGV) > 0) {
152 #switch ($ARGV[0]) {
153 #case qr/^(-h|--?help|help)$/i { &printusage; exit 0; }
154 #case qr/^(setup_greenstone|run_test|summarise|upload|all)$/ { $action=$ARGV[0]; }
155 #else {
156 #print STDERR "**** Bad subcommand.\n";
157 #&printusage;
158 #exit -1;
159 #}
160 #}
161
162
163 my $switch = $ARGV[0];
164 if($switch =~ m/^(-h|--?help|help)$/i) {
165 &printusage;
166 exit 0;
167 } elsif ($switch =~ m/^(setup_greenstone|run_test|summarise|upload|all)$/) {
168 $action=$ARGV[0];
169 } else {
170 print STDERR "**** Bad subcommand.\n";
171 &printusage;
172 exit -1;
173 }
174
175 # run_test action can take a subaction: nosvn|justdiff. It can also take --modelOS (windows|linux|darwin)
176 # nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying
177 # it out from the svn model-collect again.
178 # justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
179
180 if($action eq "run_test" && scalar(@ARGV) >= 2) {
181 push(@collections, @ARGV);
182 shift @collections; # remove action from array
183
184 for (my $i=0; $i < scalar(@ARGV); $i++) {
185 if($ARGV[$i] =~ m@^--@) {
186 shift @collections; # remove subaction/flag from array
187
188 $subaction = $ARGV[$i];
189 if($subaction eq "--justdiff") {
190 $use_local_rebuild = $use_static_model = 1;
191 } elsif ($subaction eq "--nosvn") {
192 $use_static_model = 1;
193 #} elsif ($subaction =~ m/\-\-testOS/i && defined $ARGV[$i+1]) {
194 # $test_os = $ARGV[$i+1];
195 # $i++;
196 # shift @collections; # remove test_os value from array
197 } elsif ($subaction =~ m/\-\-modelOS/i && defined $ARGV[$i+1] && $ARGV[$i+1] =~ m/windows|linux|darwin/i) {
198 $model_os = $ARGV[$i+1];
199 $i++;
200 shift @collections; # remove model_os value from array
201 #print STDERR "Model_os specified: $model_os\n";
202 } elsif ($subaction =~ m/\-\-debug/i) {
203 $debugging = 1;
204 print STDERR "*** DEBUG MODE: debug output files will be in toplevel folder\n";
205 } else {
206 print STDERR "**** Bad subaction/value: ".$ARGV[$i]."\n";
207 &printusage;
208 exit -1;
209 }
210 }
211 }
212
213# foreach my $col (@collections) {
214# print STDERR "Collection: $col\n";
215# }
216 }
217}
218
219print STDERR "Install type $install_type\n";
220print STDERR "Install version $install_version\n";
221
222#check key environment vars are set
223if(!defined $ENV{'UPLOAD_DIR'}) {
224 print STDERR "Please set a UPLOAD_DIR for the test in an environment.sh file\n";
225 #return 1;
226}
227if(!defined $ENV{'DATA_DIR'}) {
228 print STDERR "Please set a DATA_DIR for the test in an environment.sh file\n";
229 #return 1;
230}
231if(!defined $ENV{'MONITOR_EMAIL'}) {
232 print STDERR "Please set a MONITOR_EMAIL for the test in an environment.sh file\n";
233 #return 1;
234}
235
236if($ENV{'DATA_DIR'} eq "/") {
237 print STDERR "DATA_DIR should not be the fs root\n";
238 #return 1;
239}
240
241print STDERR "DATA_DIR: ".$ENV{'DATA_DIR'}."\n";
242print STDERR "UPLOAD_DIR: ".$ENV{'UPLOAD_DIR'}."\n";
243
244#create an id for this test
245my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
246$year += 1900;
247$mon += 1;
248$mon = "0$mon" if ($mon < 10);
249$mday = "0$mday" if ($mday < 10);
250my $dateid="$year.$mon.$mday"; #my $dateid=($year+1900)."-".($mon+1)."-$mday";
251
252print STDERR "Starting test '$dateid'\n";
253
254
255# http://stackoverflow.com/questions/2149368/how-can-i-loop-through-files-in-a-directory-in-perl
256$ENV{'CLASSPATH'} = "";
257my $jar_lib_path = $ENV{'TASK_HOME'}.$sep."lib";
258my @files = <$jar_lib_path/*.jar>; # /full/path/to/diffcol/lib/*jar
259foreach my $file (@files) {
260 $file =~ s@\/@\\@g if $isWin;
261 $ENV{'CLASSPATH'}=$file.$pathsep.$ENV{'CLASSPATH'};
262}
263##print STDERR "**** classpath: ".$ENV{'CLASSPATH'}."\n";
264
265
266#set the location of the full report
267my $xmlout=filename_concat($ENV{'DATA_DIR'}, "full-report-$install_version-$dateid.xml");
268##print STDERR "XML: $xmlout\n";
269
270# the toplevel folder of the greenstone installation being used
271my $greenstone3_home="";
272my $greenstone_home=""; # gs2build for gs3, toplevel install folder for gs2
273# gsdl is the checkout folder and can be greenstone2 or greenstone3
274my $gsdl="greenstone2";
275
276# Check if using existing compiled-up greenstone installation
277# and set the greenstone_home location accordingly
278
279if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
280 print STDERR "Found existing Greenstone home, will use that instead\n";
281 $greenstone_home=$ENV{'GSDLHOME'} if defined $ENV{'GSDLHOME'};
282
283 if(defined $ENV{'GSDL3SRCHOME'}) {
284 print STDERR "*** GSDL3SRCHOME set, changing to using installed GS3\n";
285 $gsdl = "greenstone3";
286 $install_version = "3";
287 $greenstone3_home=$ENV{'GSDL3SRCHOME'};
288 $greenstone_home=filename_concat($greenstone3_home, "gs2build") unless defined $ENV{'GSDLHOME'};
289 }
290} else {
291 if($install_version eq "3") {
292 $gsdl = "greenstone3";
293 $greenstone3_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
294 $greenstone_home=filename_concat($greenstone3_home, "gs2build");
295 } else {
296 $greenstone_home=filename_concat($ENV{'DATA_DIR'}, $gsdl);
297 }
298}
299##print STDERR "GSHOME: $greenstone_home\n";
300
301#do the requested action
302if($action eq "setup_greenstone") {
303 &setup_greenstone;
304}
305elsif ($action eq "run_test") {
306 &run_test;
307}
308elsif ($action eq "summarise") {
309 &summarise;
310}
311elsif ($action eq "upload") {
312 &upload;
313 &mail_with_report_attached;
314}
315elsif ($action eq "all") {
316 &setup_greenstone;
317 &run_test;
318 &summarise;
319 &upload;
320 &mail_with_report_attached;
321}
322
323##********************************
324
325sub printusage
326{
327# print STDERR "Run as: $0 (help|setup_greenstone|run_test <--modelOS windows|darwin|linux> <--justdiff|--nosvn> <col1 col2 ...> |summarise|upload|all)\n";
328 print STDERR "Run as: $0 (help|setup_greenstone|run_test|summarise|upload|all)\n";
329 print STDERR "where run_test can further take the following optional parameters:\n";
330 print STDERR "\t--modelOS (windows|darwin|linux)\n";
331 print STDERR "\t--justdiff|--nosvn\n";
332 print STDERR "\t--debug\n";
333 print STDERR "\t<col1 col2 ...>\n";
334 print STDERR "where setup_greenstone can further take the following optional parameters:\n";
335 print STDERR "\t--gs2|--gs3\n";
336 print STDERR "\t--svn|--bin\n";
337}
338
339#http://stackoverflow.com/questions/7427262/read-a-file-and-save-it-in-variable-using-shell-script
340
341sub setup_greenstone
342{
343 if($install_version eq "3" && !defined $ENV{'JAVA_HOME'}) {
344 print "*** JAVA_HOME not set. Set it and add its bin folder to the PATH\n";
345 exit 0;
346 }
347
348 #clean up from previous tests
349 print STDERR "about to clean up any old tests (Ctrl-C to cancel)"; # no newline
350 for my $i ( 1..5 ) {
351 sleep 1; # 1 second
352 print STDERR ".";
353 }
354 print STDERR "\n";
355
356 # http://perldoc.perl.org/File/Path.html
357 print STDERR "cleaning up previous tests\n";
358 #&File::Path::remove_tree($ENV{'DATA_DIR'});
359 if(-d $greenstone_home && $install_version eq "2") {
360 &File::Path::remove_tree($greenstone_home);
361 }
362 if(-d $greenstone3_home && $install_version eq "3") {
363 &File::Path::remove_tree($greenstone3_home);
364 }
365 unlink glob "$ENV{'DATA_DIR'}$sep"."*report-$install_version*";
366 unlink "$ENV{'DATA_DIR'}$sep"."compilation-errors";
367 print STDERR "creating the data dir\n";
368 #&File::Path::make_path($ENV{'DATA_DIR'}); # works like mkdir -p
369
370 chdir($ENV{'DATA_DIR'});
371 #print STDERR "@@@@ current dir: ".`pwd`."\n";
372 #print STDERR "@@@@ data dir: ".$ENV{'DATA_DIR'}."\n";
373
374 # use existing compiled-up greenstone installation, if a GSDLHOME set
375 if(defined $ENV{'GSDL3SRCHOME'} || defined $ENV{'GSDLHOME'}) {
376 print STDERR "Found existing Greenstone home, will use that instead\n";
377 return;
378 }
379
380 # Else checkout a GS from svn into DATA_DIR
381 if($install_type eq "svn") {
382 &checkout_gs_from_svn();
383 ##print STDERR "$ENV{'DATA_DIR'}$sep$gsdl\n";
384
385 if($install_version eq "2") {
386 &compile_gs2_svn();
387 } else {
388 &compile_gs3_svn();
389 }
390 }
391
392}
393
394sub checkout_gs_from_svn {
395
396 #svn checkout of main gsdl directory
397 print STDERR "checkout $gsdl:\n";
398 my $cmd = "svn co ".$ENV{'SVN_OPT_REV'}." http://svn.greenstone.org/main/trunk/greenstone$install_version $gsdl";
399 ##print STDERR "Checkout CMD: $cmd\n";
400
401 # # unlike backticks operator, system() will print the output of the command to the screen as it executes
402 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
403 my $status = system "$cmd"; #my $status = `$cmd`;
404 if($status != 0) {
405 print STDERR "@@@ SVN checkout of $gsdl failed\n";
406 exit -1;
407 }
408 print STDERR "done\n";
409}
410
411sub getImageMagickBins {
412 my ($cmd, $status);
413 my $imagickzip = "";
414
415 chdir ("$greenstone_home");
416 if(!$isWin) { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
417
418 my $bit_arch=`uname -m`;
419
420 # imagemagick binary
421 print STDERR "Getting imagemagick binary\n";
422
423 my $os = $isMac ? "darwin" : "linux";
424 $imagickzip = "imagemagick-$os";
425
426 if($isMac) {
427# $imagickzip .= "-10.5.tar.gz";
428 # at present, only the Imagemagick binaries created by Max for darwin work on the Macs
429# &File::Path::make_path("$greenstone_home$sep"."bin"."$sep$os"); # need to ensure gsdl/bin/darwin exists
430 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/imagemagick bin/darwin/imagemagick";
431 $status = system($cmd);
432 if($status != 0) {
433 print STDERR "@@@ Unable to get imagemagick for darwin\n";
434 }
435
436 # need ghostscript mac binary too for pdf to img conversions on mac
437 $cmd = "svn export http://svn.greenstone.org/main/trunk/binaries/mac/intel/ghostscript bin/darwin/ghostscript";
438 $status = system($cmd);
439 if($status != 0) {
440 print STDERR "@@@ Unable to get ghostscript for darwin\n";
441 }
442
443 # the imagemagick and ghostscript binaries have been set to executable on svn trac now
444# system("chmod -R u+x $greenstone_home/bin/darwin/imagemagick/bin/*");
445# system("chmod -R u+x $greenstone_home/bin/darwin/ghostscript/bin/*");
446 } else { # linux
447 my $extension64 = ($bit_arch =~ m/64$/) ? "-x64" : "";
448 $imagickzip .= "$extension64.tar.gz";
449
450 # now these next imagemagick steps (and those near the end of this sub) are just for linux, no longer also for mac
451 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/imagemagick/trunk/$imagickzip ext/$imagickzip";
452 $status = system ($cmd);
453 system("cd ext && tar -xvzf $imagickzip");
454 }
455 }
456 return $imagickzip;
457}
458
459sub getGnomeLibExt
460{
461 if($isWin) {
462 return; # no gnome-lib for windows
463 }
464 my ($cmd, $status);
465 my $bit_arch=`uname -m`;
466 my $os = $isMac ? "darwin" : "linux";
467
468 # gnomelib binary
469 print STDERR "setting up gnome-lib-minimal for compilation\n";
470
471 # To get gnome-lib, need to determine bit architecture of the linux/darwin
472 # http://stackoverflow.com/questions/8963400/the-correct-way-to-read-a-data-file-into-an-array
473 # $Config{'archname64'} doesn't work on the Ubuntu and the Sys::Info package seems to not be supported
474 # well on ActivePerl.
475 # But since we know we're on a Linux/Darwin machine at this point, we can just run `uname -m` and other linux cmds
476
477 # osversion will be "Lion" or ""
478 # and assuming all darwin is intel, not ppc!!
479 my $gnome_lib_file = $isMac ? "darwin-".$osversion."intel" : "linux";
480
481 $gnome_lib_file .= "-x64" if($bit_arch =~ m/64$/ && !$isMac); # linux only case
482
483 #svn checkout gnome-lib for this linux/darwin
484 chdir("$greenstone_home$sep"."ext"); #cd $DATA_DIR/$gsdl/ext
485
486 ##print STDERR "**** gnomelib: $gnome_lib_file\n";
487
488 # checkout and unpack gnome-lib-minimal
489
490 #svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-linux-x64.tar.gz gl.tar.gz
491 $cmd = "svn export http://svn.greenstone.org/gs2-extensions/gnome-lib/trunk/gnome-lib-minimal-".$gnome_lib_file.".tar.gz gl.tar.gz";
492 system $cmd;
493 system ("tar -xvzf gl.tar.gz");
494
495 ##print STDERR "*** ARCH: $bit_arch\n";
496}
497
498
499
500sub compile_gs3_svn() {
501 my ($cmd, $status);
502 chdir ("$greenstone3_home");
503
504 $cmd = "ant"; # creates the build.properties file from the .in template
505 $status = system $cmd;
506 if($status != 0) {
507 print STDERR "Unable to run the ant command: $status\n";
508 exit -1;
509 }
510
511 $cmd = "ant -Dproperties.ok=y -Dcheckout.gnomelib.ext=true prepare"; # pass in confirmation to ant prepare step
512 $status = system $cmd;
513 if($status != 0) {
514 print STDERR "Failed to run $cmd command: $status\n";
515 exit -1;
516 }
517
518 my $imagickzip = &getImageMagickBins();
519 chdir ("$greenstone3_home");
520
521 $cmd = "ant -Dcheckout.gnomelib.ext=true install"; # Compile with gnome-lib.
522 $status = system $cmd;
523 if($status != 0) {
524 print STDERR "Failed to run $cmd command: $status\n";
525 exit -1;
526 }
527}
528
529sub compile_gs2_svn() {
530 my $imagickzip = &getImageMagickBins();
531 &getGnomeLibExt();
532
533 my ($cmd, $status);
534 my $os = $isMac ? "darwin" : "linux";
535
536 chdir("$greenstone_home"); #chdir("$ENV{'DATA_DIR'}$sep$gsdl"); # goes into toplevel gs2 or gs3 folder
537
538 ##print STDERR "@@@ OS: $^O.|".$Config{'archname64'}."|\n";
539
540 if($isWin) {
541 print STDERR "Compiling $gsdl using makegs2.bat running in auto (silent) mode\n";
542
543 # we're now in the GS2 folder, call makegs2 with silent param
544 $cmd = "makegs2.bat silent 2>> $ENV{'DATA_DIR'}/compilation-errors"; # STDERR is sent to compilation-errors file
545 $status = system $cmd;
546 if($status != 0) {
547 print STDERR "Greenstone compilation on Windows failed\n";
548 exit -1;
549 }
550
551 } else { # if we're on linux/darwin, need gnome-lib for the correct architecture. And need imagemagick to build imgs in collections
552
553 chdir("ext$sep"."gnome-lib-minimal");
554
555 # need to run source devel.bash on gnome-lib followed by configure, make, make install
556 # in one go, in order to preserve the compile environment set up by sourcing devel.bash
557
558 # http://stackoverflow.com/questions/7369145/activating-a-virtualenv-using-a-shell-script-doesnt-seem-to-work
559 # http://ubuntuforums.org/showthread.php?t=1932504 linking /bin/sh to bash instead of dash
560
561# $cmd = "bash -c \"source ./devel.bash && cd ../.. && ./configure --enable-apache-httpd && make && make install\"";
562 $cmd = "bash -c \"";
563
564 $cmd .= "source ./devel.bash";
565 $cmd .= " && cd ../..";
566
567 #configure
568 # $cmd .= " && ./configure";
569 $cmd .= " && echo 'configure $gsdl: ' ";
570 $cmd .= " && echo '<configure>' >> $xmlout";
571 $cmd .= " && ./configure 2>> $ENV{'DATA_DIR'}/compilation-errors"; # configure
572 $cmd .= " && echo '</configure>' >> $xmlout";
573 $cmd .= " && echo 'done'";
574
575 #make
576 $cmd .= " && echo 'make $gsdl: '";
577 $cmd .= " && echo '<make>' >> $xmlout";
578 $cmd .= " && make 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make
579 $cmd .= " && echo '</make>' >> $xmlout";
580 $cmd .= " && echo 'done'";
581
582 #make install
583 $cmd .= " && echo 'make install $gsdl: '";
584 $cmd .= " && echo '<make-install>' >> $xmlout";
585 $cmd .= " && make install 2>> $ENV{'DATA_DIR'}/compilation-errors"; # make install
586 $cmd .= " && echo '</make-install>' >> $xmlout";
587 $cmd .= " && echo 'done'";
588
589 $cmd .= "\""; # close off cmd to bash and run it
590 $status = system $cmd;
591
592 if(!$isMac) { # Linux
593 # Moving imagemagick after instead of before compilation, since bin/darwin and bin/linux gets overwritten during compilation
594 move("$greenstone_home$sep"."ext/imagemagick/$os", "$greenstone_home$sep"."bin/$os/imagemagick"); # http://www.perlmonks.org/?node_id=586537
595 unlink "$greenstone_home$sep"."ext$sep$imagickzip" or warn "Could not unlink ext/$imagickzip: $!";
596 &File::Path::remove_tree("$greenstone_home$sep"."ext$sep"."imagemagick"); # the untarred parent folder
597 }
598
599 }
600
601 if($status != 0) {
602 print STDERR "@@@ Compilation of Greenstone on Linux/Mac failed\n";
603 exit -1;
604 }
605
606 &getIsisGdl("$greenstone_home"); #&getIsisGdl("$ENV{'DATA_DIR'}/$gsdl");
607
608 # set the path to the greenstone_home variable
609 #$greenstone_home="$ENV{'DATA_DIR'}$sep$gsdl";
610}
611
612sub getPDFBox
613{
614 # current revision is 27763, but using "head" works
615 my $PDFBOX_TRAC_URL="https://trac.greenstone.org/export/head/gs2-extensions/pdf-box/trunk/pdf-box-java"; # both for .zip and .tar.gz extension
616 #"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/pdf-box/trunk/pdf-box-java";
617
618 # now get the PDFBox extension for PDFBox tutorial
619 print STDERR "Getting pdfbox from $PDFBOX_TRAC_URL:\n";
620
621 my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
622
623 chdir($greenstone_home);
624 my $cmd = "";
625 if ($isWin) {
626 #$cmd = "setup.bat && cd ext && $wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
627 $cmd = "set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && cd ext && $wget $PDFBOX_TRAC_URL.zip && unzip pdf-box-java.zip";
628 } elsif ($isMac) { # need to use curl not wget
629 $cmd = "cd ext && curl $PDFBOX_TRAC_URL.tar.gz > pdf-box-java.tar.gz && tar -xzf pdf-box-java.tar.gz";
630 }
631 else { # linux
632 $cmd = "bash -c \"export GSDLHOME=&& source $setup_script.$linux_ext && cd ext && $wget $PDFBOX_TRAC_URL.tar.gz && tar -xzf pdf-box-java.tar.gz\"";
633 }
634 my $status = system $cmd;
635 if($status != 0) {
636 print STDERR "@@@ Failed to set up PDFBox\n";
637 exit -1; # or proceed to testing other tutorials?
638 }
639}
640
641sub getOpenOfficeExt
642{
643 # current revision is 27763, but using "head" works
644 my $OOEXT_TRAC_URL="https://trac.greenstone.org/export/head/gs2-extensions/open-office/trunk/open-office-java"; # both for .zip and .tar.gz extension
645 #"http://trac.greenstone.org/export/".$ENV{'SVN_OPT_REV'}."/gs2-extensions/open-office/trunk/open-office-java";
646
647 # now get the OpenOffice extension for the AllDocTypes collection
648 print STDERR "Getting open office ext from $OOEXT_TRAC_URL:\n";
649
650 chdir($greenstone_home);
651 my $cmd = "";
652 if ($isWin) {
653 $cmd = "setup.bat && cd ext && $wget $OOEXT_TRAC_URL.zip && unzip open-office-java.zip";
654
655 } elsif ($isMac) { # need to use curl not wget
656 $cmd = "cd ext && curl $OOEXT_TRAC_URL.tar.gz > open-office-java.tar.gz && tar -xzf open-office-java.tar.gz";
657 }
658 else { # linux
659 $cmd = "bash -c \"export GSDLHOME=&& source setup.bash && cd ext && $wget $OOEXT_TRAC_URL.tar.gz && tar -xzf open-office-java.tar.gz\"";
660 }
661 my $status = system $cmd;
662 if($status != 0) {
663 print STDERR "@@@ Failed to set up the Open Office Extension\n";
664 exit -1; # or proceed to testing other tutorials?
665 }
666}
667
668sub getIsisGdl {
669 my $gsfolder = shift(@_);
670
671 if(!$isWin) {
672 chdir($greenstone_home);
673 my $bit_arch=`uname -m`;
674 if ($bit_arch =~ m/64$/) {
675 my $cmd = "";
676 if($isMac) {
677 $cmd = "cd $gsfolder/bin/darwin && curl https://www.greenstone.org/caveat-emptor/IsisGdl.macleopard > IsisGdl && chmod u+x IsisGdl";
678 } else { # linux
679 $cmd = "cd $gsfolder/bin/linux && $wget https://www.greenstone.org/caveat-emptor/IsisGdl.bin32 && mv IsisGdl.bin32 IsisGdl && chmod u+x IsisGdl";
680 }
681 my $isis_status = system $cmd;
682 if($isis_status != 0) {
683 print STDERR "Unable to get IsisGdl from caveat page\n";
684 }
685 }
686 }
687}
688
689# http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
690sub run_test
691{
692 my $collect_parent = $greenstone_home;
693 my $model_collect = "model-collect";
694 my $build_options = "";
695 if($install_version eq "3") {
696 $collect_parent = &filename_concat($greenstone3_home,"web","sites","localsite");
697 $model_collect = "gs3-model-collect";
698 $build_options = " -site localsite ";
699 chdir($greenstone3_home);
700 } else {
701 chdir($greenstone_home);
702 }
703
704 my $num_cols = scalar(@collections); # remember the empty case
705
706 if($num_cols == 0) { # deal with all collections
707 push (@collections, "");
708 # putting the empty string in the array so that the "all collections" case
709 # can be handled similar to how the case of user-specified collections is handled
710
711 } else { # deal with user specified set of collections
712 # prefix the directory separator to each collection name
713 @collections = map { $sep.$_ } @collections;
714 }
715
716 my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
717 if(!-d $pdfbox) { # && $install_version eq "2"
718 &getPDFBox();
719 }
720
721 my $openofficeext = &filename_concat($greenstone_home, "ext", "open-office");
722 if(!-d $openofficeext && $install_version eq "2") { # We don't checkout openoffice with GS3
723 # (and not all Win machines where me may want to run Win diffcol have openoffice installed)
724 &getOpenOfficeExt();
725 }
726
727 #&getIsisGdl("$greenstone_home");
728
729 open (my $xml_fh, '>'.$xmlout) || die "Could not open xml file $xmlout for appending: $!\n";
730
731 # perform the requested subcommands, outputting xml information
732 print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
733
734 my ($cmd, $status);
735 # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
736
737 if(!$use_local_rebuild) {
738 print $xml_fh "Updating $ENV{'TASK_HOME'}/$model_collect:\n";
739 for my $col (@collections) {
740 $cmd = "svn up $ENV{'TASK_HOME'}/$model_collect$col"; #chdir("$ENV{'TASK_HOME'}/$model_collect");
741 $status = system "$cmd";
742 }
743 }
744
745 # go to whichever collecthome parent we're using
746 chdir($collect_parent);
747
748 # get svn info
749 print STDERR "getting svn info: $xmlout\n";
750 print $xml_fh "<svn-info>\n";
751 &run_and_print_cmd("svn info", $xml_fh);
752 print $xml_fh "</svn-info>\n";
753 print STDERR "done\n";
754
755 if(!$use_local_rebuild) {
756
757 #make two copies of the model-collect directory in gsdl
758 #one to be rebuilt and one as the basis for comparison
759 #strip both of all .svn directories
760
761 #copy the model collections to the collect folder to be rebuilt
762 print STDERR "installing test collections and model collections to new $gsdl installation... ";
763 #clean up
764 if(-d "collect") {
765 for my $col (@collections) {
766 if(-d "collect$col") {
767 &File::Path::remove_tree("collect$col") || die "Error could not delete collect: $!";
768 }
769 }
770 }
771
772 if($use_static_model) {
773 for my $col (@collections) {
774 #copy to collect and strip .svn subfolders
775 &File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
776 &copy_recursively("model-collect$col", "collect$col", ".svn");
777 }
778
779 } else { # the default situation: where we check out the model-collect from svn
780 for my $col (@collections) {
781 &File::Path::remove_tree("model-collect$col");
782
783 #copy to collect and strip .svn subfolders
784 &File::Path::make_path("collect$col"); # create the folder and copy contents across
785 &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","$model_collect$col"), "collect$col", ".svn");
786
787 #make the model copy
788 &File::Path::make_path("model-collect$col");
789 &copy_recursively("collect$col", "model-collect$col"); # copy contents across
790 }
791 }
792
793 print STDERR "done\n";
794 }
795
796 #for each collection, import, build and diff with its model counterpart
797
798 # if working with all collections, read the list of collections from the folders in collect
799 if($num_cols == 0) {
800 @collections = (); # get rid of the empty string put in the array to represent "all collections"
801
802 opendir my($collect_handle), "collect" or die "Could not open dir $collect_parent/collect: $!";
803
804 for my $collection (readdir $collect_handle) {
805 next if ($collection eq "." || $collection eq "..");
806 next if ($collection eq "modelcol");
807 push(@collections, $collection);
808 }
809 closedir $collect_handle; # close handle to collect dir
810 }
811
812 for my $collection (@collections) {
813 chdir($collect_parent); # for GS3 too, ensure we're first in the collect folder for each collection
814
815 # next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
816 # next if ($collection !~ m/OAI|METS|DSpace|MGPP|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
817 # next if ($collection !~ m/PDFBox/); ## TEMPORARY, FOR TESTING THIS SCRIPT
818
819 #escape the filename (in case of space)
820 $collection =~ s@ @\\ @g;
821 #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
822
823 $collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
824
825 if (! -d "collect$sep$collection") {
826 print STDERR "**** Looked in current dir ".getcwd."\n";
827 print STDERR "Collection $collection does not exist\n";
828 next;
829 }
830
831 print STDERR "*** Found collection $collection\n";
832 print $xml_fh "<collection-test name=\"$collection\">\n";
833
834 # run the building scripts from the toplevel of the GS installation
835 if($install_version eq "3") {
836 chdir($greenstone3_home);
837 } else {
838 chdir($greenstone_home);
839 }
840
841 if(!$use_local_rebuild) {
842 #import
843 # Ensure the OIDtype for importing is hash_on_full_filename
844 # "to make document identifiers more stable across upgrades of the software,
845 # although it means that duplicate documents contained in the collection are
846 # no longer detected automatically."
847 print STDERR "$collection - Importing:\n";
848 print $xml_fh "<import>\n";
849 &run_build_script("import.pl $build_options -removeold $collection"); #-OIDtype hash_on_full_filename
850 print $xml_fh "</import>\n";
851 print STDERR "done\n";
852
853 #build
854 print STDERR "$collection - Building:\n";
855 print $xml_fh "<build>\n";
856 &run_build_script("buildcol.pl $build_options -removeold $collection");
857 print $xml_fh "</build>\n";
858 print STDERR "done\n";
859
860 #rename the intermediate 'building' directory 'index'
861 print STDERR "$collection - Move \"building\" to \"index\"... ";
862 my $index = &filename_concat($collect_parent, "collect", $collection, "index");
863 my $building = &filename_concat($collect_parent, "collect", $collection, "building");
864 &File::Path::remove_tree($index);
865 # Renaming Directories, http://www.perlmonks.org/?node_id=177421
866 move($building, $index) or die "copy failed: $!"; # File::Copy::move
867 print STDERR "done\n";
868 }
869 #diffcol
870 print STDERR "$collection - Diffing:\n";
871 my $diffcol_dir = &filename_concat($ENV{'TASK_HOME'},"diffcol");
872
873# chdir($collect_parent); # this is actually where we are
874 # help diffcol to know on what os the model cols were generated
875 # and what os this test machine is (on which the test cols will be generated)
876 my $debug_state = $debugging ? "-debug" : "";
877 $cmd = "diffcol.pl -testos $test_os -modelos $model_os -output xml -verbosity 10 $debug_state $collection"; # need to run with ./diffcol.pl if bash script
878
879 #print STDERR "Running diffcol cmd: $cmd\n";
880 &run_diff_script($cmd, $xml_fh, $diffcol_dir);
881
882 if($install_version eq "3") {
883 chdir($greenstone3_home); # this is actually where we are
884 } else {
885 chdir($greenstone_home); # this is actually where we are
886 }
887 print STDERR "done\n";
888 print $xml_fh "</collection-test>\n";
889 }
890
891 print $xml_fh "</test>\n";
892 close($xml_fh);
893
894 print STDERR "done\n";
895}
896
897##***************************************************************
898# runs setup in greenstone_home before running the diff command
899sub run_diff_script {
900 my ($cmd, $fh, $diffcol_dir) = @_;
901
902 my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
903
904 # we're in greenstone_home now
905 if(!$isWin) {
906 $cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && cd $diffcol_dir && ./$cmd\"";
907
908 } else { # Need to prefix cmd -c/-k as necessary
909 $cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && cd $diffcol_dir && perl -S $cmd\"";
910## print STDERR "@@@@ Going to call command: $cmd\n";
911 }
912
913 return &run_and_print_cmd($cmd, $fh);
914}
915
916# runs setup in greenstone_home before running the given build command
917sub run_build_script {
918 my ($cmd, $fh) = @_;
919 my $linux_ext = ($install_version eq "3") ? "sh" : "bash";
920
921# chdir($greenstone_home);
922 # we are in $greenstone_home already, can directly run the build cmd on the collection
923 if(!$isWin) {
924 $cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& source $setup_script.$linux_ext && $cmd\"";
925 #$cmd = "bash -c \"export GSDL3SRCHOME=&& export GSDLHOME=&& export PERL_HASH_SEED_DEBUG=1&& source $setup_script.$linux_ext && $cmd\"";
926
927 } else { # Need to prefix cmd -c/-k as necessary
928 $cmd = "cmd /c \"set GSDL3SRCHOME=&& set GSDLHOME=&& $setup_script.bat && perl -S $cmd\"";
929 }
930## print STDERR "@@@@ Going to call command: $cmd\n";
931
932 return system($cmd);
933 #return &run_and_print_cmd($cmd, $fh); # doesn't work on cmds chained with bash -c
934}
935
936
937# http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1activeperl%20sys::info
938# http://stackoverflow.com/questions/1477500/how-do-i-get-the-output-of-an-external-command-in-perl
939sub run_and_print_cmd {
940 my ($cmd, $fh) = @_;
941
942 open my $pin, "$cmd|" or die "unable to run cmd $cmd: $!"; # open(my $fh, '-|', 'powercfg -l') or die $!;
943
944 if(defined $fh) { # print cmd output both to the filehandle and to stdout
945 while (my $line = <$pin>) {
946 print $fh $line;
947# print STDOUT $line; # if also printing cmd output to STDOUT
948 }
949 }
950 else { # no filehandle, so just need to print to stdout
951
952 # unlike backticks operator, system() will print the output of the command to the screen as it executes
953 # http://stackoverflow.com/questions/758611/how-to-flush-output-in-backticks-in-perl?rq=1
954
955 my $status = system $cmd;
956 if($status != 0) {
957 print STDERR "ERROR ($status) running $cmd: $!\n";
958 }
959 }
960 close($pin);
961}
962
963sub filename_concat {
964 my $first_file = shift(@_);
965 my (@filenames) = @_;
966
967 # If first_file is not null or empty, then add it back into the list
968 if (defined $first_file && $first_file =~ /\S/)
969 {
970 unshift(@filenames, $first_file);
971 }
972
973 my $filename = join($sep, @filenames);
974 $filename =~ s/[\\\/]$//; # remove trailing slashes if any
975 return $filename;
976}
977
978
979# The following code is from
980# http://stackoverflow.com/questions/227613/how-can-i-copy-a-directory-recursively-and-filter-filenames-in-perl
981# It also states that "Perl's File::Copy is a bit broken (it doesn't copy permissions on Unix systems, for example)"
982sub copy_recursively {
983 my ($from_dir, $to_dir, $regex) = @_;
984 opendir my($dh), $from_dir or die "Could not open dir '$from_dir': $!";
985
986# if(-d !$to_dir) {
987# mkdir $to_dir or die "mkdir '$to_dir' failed: $!" if not -e $to_dir;
988# }
989
990 for my $entry (readdir $dh) {
991 next if ($entry eq "." || $entry eq "..");
992 next if (defined $regex && $entry =~ /$regex/);
993 my $source = "$from_dir/$entry";
994 my $destination = "$to_dir/$entry";
995 if (-d $source) {
996 mkdir $destination or die "mkdir '$destination' failed: $!" if not -e $destination;
997 copy_recursively($source, $destination, $regex);
998 } else {
999 copy($source, $destination) or die "copy failed: $!";
1000 }
1001 }
1002 closedir $dh;
1003 return;
1004}
1005
1006sub summarise {
1007
1008 # make a summarised Xml report
1009 print STDERR "Summarizing the xml report... ";
1010 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/xml-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml";
1011 my $status = system($cmd);
1012 print STDERR "done\n";
1013
1014 # make a summarised HTMl report
1015 print STDERR "Creating an html summary report... ";
1016 $cmd = "java org.apache.xalan.xslt.Process -IN $ENV{'DATA_DIR'}/report-$install_version-$dateid.xml -XSL $ENV{'TASK_HOME'}/xsl/html-report.xsl -OUT $ENV{'DATA_DIR'}/report-$install_version-$dateid.html";
1017
1018 print STDERR "@@@ Running summarise command: $cmd\n\n";
1019 $status = system($cmd);
1020
1021 print STDERR "done\n";
1022
1023 # Print whether the tests passed or failed
1024 print STDERR "*******************************************\n";
1025 print STDERR "Checking if successful... \n";
1026 $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
1027 $status = `$cmd`; #$status = system($cmd);
1028 print STDERR "result: $status\n";
1029 print STDERR "*******************************************\n";
1030}
1031
1032sub upload {
1033 # if the upload dir already existed, clear it of contents
1034 if (-d $ENV{'UPLOAD_DIR'}) { #else rm $UPLOAD_DIR/*
1035 # don't want to keep previous days reports
1036 # else we will have to manually clear them at some point
1037 # just generate the set of reports for this run of task.pl upload
1038 # and
1039 &File::Path::remove_tree($ENV{'UPLOAD_DIR'});
1040 }
1041 # recreate the upload directory
1042 &File::Path::make_path($ENV{'UPLOAD_DIR'});
1043
1044 # copy all *.xml and *.html files across to UPLOAD_DIR
1045 opendir my($dh), $ENV{'DATA_DIR'} or die "Could not open DATA_DIR: $!";
1046 for my $entry (readdir $dh) {
1047 next if ($entry !~ m/(\.xml|\.html?)$/);
1048 next if ($entry !~ m/(report-$install_version)/);
1049
1050 # copy the reports across with different names: with OS prefixed to them. And for the HTML file on Win, rename to HTM
1051 # html files uploaded from windows to nzdl are empty for no reason. Uploading as htm seems to work
1052 my $os_entry = $entry;
1053 $os_entry =~ s@\[email protected]@ if $isWin;
1054 if($isMac) {
1055 $osversion = "Leopard-" if ($osversion eq "");
1056 $os_entry = "diffcol-".$^O."-".$osversion."$os_entry"; # darwin-Lion for Lion/Mountain Lion
1057 } else {
1058 $os_entry = "diffcol-".$^O."-$os_entry";
1059 }
1060
1061 # if the test failed, prefix "failed" to the report so that it shows up with an error icon on the caveat page
1062 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
1063 my $result = `$cmd`;
1064 if($result ne "yes") {
1065 $os_entry =~ s/diffcol-/diffcol-FAIL-/;
1066 }
1067
1068 # get the absolute path to the original files before copying them over
1069 $entry = &filename_concat($ENV{'DATA_DIR'}, $entry);
1070
1071 # copy them over with their new names
1072## print STDERR "@@@@ copying across $entry to $ENV{'UPLOAD_DIR'} as $os_entry\n";
1073 copy($entry, "$ENV{'UPLOAD_DIR'}$sep$os_entry"); #copy($entry, "$ENV{'UPLOAD_DIR'}");
1074 }
1075 closedir $dh;
1076
1077
1078 # Upload the html file to puka
1079 #default identity dir
1080 if ( ! exists $ENV{'IDENTITY_DIR'} ) {
1081 $ENV{'IDENTITY_DIR'} = "$ENV{'HOME'}${sep}.ssh"; # "C:\\Research\\Nightly\\tools\\keys" on windows, see environment.pl
1082 }
1083 if (! exists $ENV{'SNAPSHOT_MODE'} ) {
1084 $ENV{'SNAPSHOT_MODE'} = "caveat";
1085 }
1086
1087 #use the correct key for uploading
1088 $ENV{'IDENTITY_FILE'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} . ($^O eq "MSWin32" ? ".ppk" : "");
1089
1090 if(-f $ENV{'IDENTITY_FILE'}) {
1091 # if you need to touch the file on windows: http://stackoverflow.com/questions/51435/windows-version-of-the-unix-touch-command
1092
1093 # the report we want to upload is actually just os-diffcol-report-$install_version-$dateid.html
1094 my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c *.htm* | "; #&& cat *.html | "; # && tar -c * |
1095 $command .= ($^O eq "MSWin32" ? "plink" : "ssh");
1096 $command .= " -T -i \"$ENV{'IDENTITY_FILE'}\" nzdl\@puka.cs.waikato.ac.nz";
1097 #print "$command\n";
1098 my $status = system("$command");
1099 if($status != 0) {
1100 print STDERR "*** Failed to upload test report to nzdl $status\n";
1101 }
1102
1103
1104 } else {
1105 print STDERR "*** Cannot upload the test report to nzdl from this machine - no identity file $ENV{'IDENTITY_FILE'}\n";
1106 }
1107
1108 #use the correct key for uploading - ed25519 for www-internal
1109 $ENV{'IDENTITY_FILE_ED25519'} = "$ENV{'IDENTITY_DIR'}${sep}upload-" . $ENV{'SNAPSHOT_MODE'} ."-ed25519". ($^O eq "MSWin32" ? ".ppk" : "");
1110
1111 if (-f $ENV{'IDENTITY_FILE_ED25519'}) {
1112 # for now, upload a copy to the new new machine, later to replace puka
1113 my $command = "cd \"$ENV{'UPLOAD_DIR'}\" && tar -c *.htm* | "; #&& cat *.html | "; # && tar -c * |
1114 $command .= ($^O eq "MSWin32" ? "plink" : "ssh");
1115 $command .= " -T -i \"$ENV{'IDENTITY_FILE_ED25519'}\" nzdl-gsorg\@www-internal.greenstone.org";
1116 #print "$command\n";
1117 my $status = system("$command");
1118 if($status != 0) {
1119 print STDERR "*** Failed to upload test report to www-internal $status\n";
1120 }
1121
1122 } else {
1123 print STDERR "**** Cannot upload the test report to www-internal from this machine - you need to generate the ed215519 identity file $ENV{'IDENTITY_FILE_ED25519'}\n";
1124 }
1125 print STDERR "Finished uploading\n";
1126}
1127
1128# Sending emails with perl: http://learn.perl.org/examples/email.html
1129# Sending email attachments with perl: http://www.perlmonks.org/?node_id=19430
1130# Sadly none of the packages are installed by default and use of MIME::Lite is discouraged
1131sub mail_with_report_attached
1132{
1133 # email out with report attached, if the tests failed wwwdev
1134 print STDERR "Checking if successful... \n";
1135 my $cmd = "java org.apache.xalan.xslt.Process -IN $xmlout -XSL $ENV{'TASK_HOME'}/xsl/passed-or-not.xsl";
1136 #my $result = system($cmd);
1137 my $result = `$cmd`;
1138
1139 print STDERR "result: $result\n";
1140
1141 if($result ne "yes") {
1142 my $msg = "$gsdl regression test for $dateid failed";
1143 my $subject = "Regression Test Failed"; #"$gsdl regression test for $dateid failed\n";
1144 my $attach_file = &filename_concat($ENV{'DATA_DIR'}, "report-$install_version-$dateid.html");
1145
1146 if($isWin) {
1147 if($use_blat && $blat && $ENV{'GSDL_SMTP'}) {
1148 # http://stackoverflow.com/questions/709635/sending-mail-from-batch-file
1149 #blat -to [email protected] -server smtp.example.com -f [email protected] -subject "subject" -body "body"
1150
1151 # need to install blat on windows
1152 $cmd = "$blat -to $ENV{'MONITOR_EMAIL'} -server $ENV{'GSDL_SMTP'} -f $ENV{'MONITOR_EMAIL'} -attach $attach_file -subject \"$subject\" -body \"$msg\"";
1153 $result = system($cmd);
1154 }
1155 else {
1156 $result = 1; # status from running mail command is 0 if success, 1 if fail
1157 print STDERR "********************************************\n";
1158 if ($use_blat) {
1159 print STDERR "Need blat and SMTP set to send mail attachment\n" ;
1160 } else {
1161 print STDERR "Not set up to send mail on Windows\n";
1162 }
1163 print STDERR "Inspect report at: $attach_file\n";
1164 print STDERR "********************************************\n";
1165 }
1166 } else { # linux
1167 my $status = system("command -v mutt > /dev/null 2>&1;"); #better way of doing "which mutt"
1168
1169 if($status != 0) { # mutt doesn't exist, can't send attachments, so send simple email
1170 $cmd="echo '$gsdl regression test for $dateid failed.' | mail -s 'Regression Test Failed' $ENV{'MONITOR_EMAIL'}";
1171
1172 print STDERR "********************************************\n";
1173 print STDERR "No mutt installed, unable to mail attachment\n";
1174 print STDERR "Inspect report at: $attach_file\n";
1175 print STDERR "********************************************\n";
1176 } else {
1177 #$cmd = "bash -c \"echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}\"";
1178 $cmd = "echo '$gsdl regression test for $dateid failed' | mutt -a $attach_file -s 'Regression Test Failed' -- $ENV{'MONITOR_EMAIL'}";
1179 }
1180
1181 # run the mail command
1182 $result = system($cmd); #&run_and_print_cmd($cmd);
1183 }
1184
1185
1186 if($result != 0) {
1187 print STDERR "*** Unable to send email: $?\n";
1188 }
1189 else {
1190 print STDERR "Sent mail with report attached.\n";
1191 }
1192 } else {
1193 print STDERR "********************************************\n";
1194 print STDERR "Tests were successful. Not sending mail.\n";
1195 print STDERR "********************************************\n";
1196 }
1197}
1198
1199# The old version of this program contained the following, consisting of 1 line of active code:
1200
1201 # Invoke as: sjmc@br:/research/sjm84/envi/bin$ ./envi diffcol summarise
1202 # Doing so will call this pl file and pass in "summarise" in ARGV
1203 # This pl file will in turn call the task executable in this folder
1204 # passing in "summarise" as a parameter.
1205#system("/bin/bash -c \"../etc/tasks/diffcol/task @ARGV\"");
1206
1207 ##system("/bin/bash -c \"./task @ARGV\"");
1208 ##print STDERR "/bin/bash -c ../etc/tasks/diffcol/task @ARGV"
1209
Note: See TracBrowser for help on using the repository browser.