Changeset 28114 for other-projects/nightly-tasks/diffcol/trunk

Timestamp:

2013-08-21T19:58:39+12:00 (11 years ago)

Author:

ak19

Message:

The run_test actions takes additional cmdline parameters. The first is nosvn, to reduce copying over time, especially useful on windows. The second is justdiff, which assumes the collections have already been built and you just want to diff them, which is handy when testing the diffing process. Further commandline parameters that work with run_test are any number of specific collections to run the test on, with or without one of nosvn and justdiff specified alongside. This then reduces the svn checkout and file copying and collection building to just the specified collections.

File:

• other-projects/nightly-tasks/diffcol/trunk/task.pl (modified) (9 diffs)

other-projects/nightly-tasks/diffcol/trunk/task.pl

@@ -95 +95 @@
 #parse arguments
 my $action = "all";
-if(scalar(@ARGV) > 1) {
-    &printusage();
-    exit 0;
+my $subaction = ""; # run_test can take subactions: --just_diff and --no_svn
+my @collections = ();
+
+if(scalar(@ARGV) > 1 && $ARGV[0] ne "run_test") {
+    &printusage();
+    exit -1;
 }
 
@@ -105 +108 @@
 else {    
     switch ($ARGV[0]) {
-    case qr/^(-h|-help|help)$/i { &printusage; exit 0; }
+    case qr/^(-h|--?help|help)$/i { &printusage; exit 0; }
     case qr/^(setup_greenstone|run_test|summarise|upload|all)$/ { $action=$ARGV[0]; }
     else { 
@@ -113 +116 @@
     }
     }
-}
+    
+    # run_test action can take a subaction: nosvn|justdiff. 
+    # nosvn: uses the model-collect as static and copies it over to collect, rebuilding what's currently in model-collect instead of copying 
+    # it out from the svn model-collect again.
+    # justdiff: same as nosvn, but doesn't copy over model-collection to collect, and doesn't rebuild either of them. Just does the diff part.
+    if($action eq "run_test" && scalar(@ARGV) >= 2) {
+        push(@collections, @ARGV);
+        shift @collections; # remove action from array
+        
+        if($ARGV[1] =~ m@^--@) {
+            shift @collections; # remove subaction from array
+            
+            $subaction = $ARGV[1];          
+            if($subaction eq "--justdiff") {
+                $use_local_rebuild = $use_static_model = 1;
+            } elsif ($subaction eq "--nosvn") {
+                $use_static_model = 1;
+            } else {
+                print STDERR "Bad subaction.\n"; 
+                &printusage; 
+                exit -1;
+            }           
+        }
+        
+#       foreach my $col (@collections) {
+#           print STDERR "Collection: $col\n";
+#       }
+    }
+}
+
 
 #check key environment vars are set
@@ -206 +238 @@
 sub printusage
 {
-    print STDERR "Run as: $0 (help|setup_greenstone|run_test|summarise|upload|all)\n";
+    print STDERR "Run as: $0 (help|setup_greenstone|run_test <--justdiff|--nosvn> <col1 col2 ...> |summarise|upload|all)\n";
 }
 
@@ -422 +454 @@
 # http://stackoverflow.com/questions/3377879/how-do-i-receive-command-output-immediately
 sub run_test
-{
+{   
+    my $num_cols = scalar(@collections); # remember the empty case
+    
+    if($num_cols == 0) { # deal with all collections
+        push (@collections, ""); 
+            # putting the empty string in the array so that the "all colleections" case 
+            # can be handled similarl to how the case of user-specified collections is handled
+        
+    } else { # deal with user specified set of collections
+        # prefix the directory separator to each collection name
+        @collections = map { $sep.$_ } @collections;
+    }
+
     my $pdfbox = &filename_concat($greenstone_home, "ext", "pdf-box");
     if(!-d $pdfbox) {
@@ -433 +477 @@
     print $xml_fh "<test time=\"$dateid\" id=\"$dateid\">\n";
     
-    my $cmd;
+    my ($cmd, $status);
     # make sure that diffcol/model-collect is up to date before copying it over to greenstone-home
     
     if(!$use_local_rebuild) {   
         print $xml_fh "Updating $ENV{'TASK_HOME'}/model-collect:\n";
-        $cmd = "svn up $ENV{'TASK_HOME'}/model-collect"; #chdir("$ENV{'TASK_HOME'}/model-collect");
-        my $status = system "$cmd";
+        for my $col (@collections) {
+            $cmd = "svn up $ENV{'TASK_HOME'}/model-collect$col"; #chdir("$ENV{'TASK_HOME'}/model-collect");
+            $status = system "$cmd";
+        }
     }
 
@@ -452 +498 @@
     print STDERR "done\n";
 
-    #make two copies of the model-collect directory in gsdl
-    #one to be rebuilt and one as the basis for comparison
-    #strip both of all .svn directories
-    
-    #copy the model collections to the collect folder to be rebuilt
-    print STDERR "installing test collections and model collections to new $gsdl installation... ";
-
-    if(!$use_local_rebuild) {   
+    if(!$use_local_rebuild) {
+        
+        #make two copies of the model-collect directory in gsdl
+        #one to be rebuilt and one as the basis for comparison
+        #strip both of all .svn directories
+    
+        #copy the model collections to the collect folder to be rebuilt
+        print STDERR "installing test collections and model collections to new $gsdl installation... ";
+    
         #clean up
         if(-d "collect") {
-        &File::Path::remove_tree("collect") || die "Error could not delete collect: $!";
+            for my $col (@collections) {
+                &File::Path::remove_tree("collect$col") || die "Error could not delete collect: $!";            
+            }   
         }
 
         if($use_static_model) {     
-        #copy to collect and strip .svn subfolders
-        &File::Path::make_path("collect"); # create the collect folder and copy contents from static model-collection across
-        &copy_recursively("model-collect", "collect", ".svn");
+            for my $col (@collections) {        
+                #copy to collect and strip .svn subfolders
+                &File::Path::make_path("collect$col"); # create the collect folder and copy contents from static model-collection across
+                &copy_recursively("model-collect$col", "collect$col", ".svn");
+            }   
 
         } else { # the default situation: where we check out the model-collect from svn
-        &File::Path::remove_tree("model-collect");
-        
-        #copy to collect and strip .svn subfolders
-        &File::Path::make_path("collect"); # create the folder and copy contents across
-        &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect"), "collect", ".svn");
-        
-        #make the model copy
-        &File::Path::make_path("model-collect"); 
-        &copy_recursively("collect", "model-collect"); # copy contents across
+            for my $col (@collections) {
+                &File::Path::remove_tree("model-collect$col");
+                
+                #copy to collect and strip .svn subfolders
+                &File::Path::make_path("collect$col"); # create the folder and copy contents across
+                &copy_recursively(&filename_concat("$ENV{'TASK_HOME'}","model-collect$col"), "collect$col", ".svn");
+                
+                #make the model copy
+                &File::Path::make_path("model-collect$col"); 
+                &copy_recursively("collect$col", "model-collect$col"); # copy contents across
+            }
         }
 
@@ -485 +538 @@
     }   
 
-    #for each collection, import, build and diff with its model counterpart
-    opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
-    for my $collection (readdir $collect_handle) {
-    next if ($collection eq "." || $collection eq "..");
-    next if ($collection eq "modelcol");
-#   next if ($collection ne "Section-Tagging"); ## TEMPORARY, FOR TESTING THIS SCRIPT
-#   next if ($collection !~ m/MARC/); ## TEMPORARY, FOR TESTING THIS SCRIPT
+    #for each collection, import, build and diff with its model counterpart    
+    
+    # if working with all collections, read the list of collections from the folders in collect
+    if($num_cols == 0) {
+        @collections = (); # get rid of the empty string put in the array to represent "all collections"
+        
+        opendir my($collect_handle), "collect" or die "Could not open dir $greenstone_home/collect: $!";
+    
+        for my $collection (readdir $collect_handle) {
+            next if ($collection eq "." || $collection eq "..");
+            next if ($collection eq "modelcol");
+            push(@collections, $collection);            
+        }
+        closedir $collect_handle; # close handle to collect dir
+    }
+    
+    for my $collection (@collections) {
+
+    #   next if ($collection ne "Demo-Lucene"); ## TEMPORARY, FOR TESTING THIS SCRIPT
+    #   next if ($collection !~ m/OAI|METS|DSpace|MGPP|Lucene/); ## TEMPORARY, FOR TESTING THIS SCRIPT
 
     #escape the filename (in case of space)
     $collection =~ s@ @\\ @g;
     #getting just the basename of the collection would have been necessary had we not cd-ed into $gsdl
-
+    
+    $collection =~ s@^[\\/]@@g; # take the dir-sep prefix away again for user-specified collection names
+    
+    if (! -d "collect$sep$collection") {
+        print STDERR "Collection $collection does not exist\n";
+        next;
+    }
+    
     print STDERR "*** Found collection $collection\n";
     print $xml_fh "<collection-test name=\"$collection\">\n";
@@ -538 +611 @@
     print $xml_fh "</collection-test>\n";
     }
-    closedir $collect_handle; # close handle to collect dir
 
     print $xml_fh "</test>\n";