Changeset 36807 for other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl

Timestamp:

2022-10-16T18:51:49+13:00 (20 months ago)

Author:

anupama

Message:

We used to run diffcol as a nightly task only for GS2. Commit 36655 was the first stage of getting diffcol to work for GS3, but skipped a lot of important code branches (like comparing the index\text\j/gdb files) in order to fix up the easier parts of the code. Now that I think the remainder of the diffcol scripts have been got to work with diffcol for GS3, where the index\text\flatdb files are compared and diffcol works for them, I can commit the important changes as well as commented out debugging statements made to the diffcol scripts that get the full diffcol code to work for GS3 diffcol. I will recommit again after removing the debugging statements. And I still need to do a full local diffcol run again, as well as testing if diffcol still works after locally undoing my sort field changes to some GS3 model cols (the recent commits to Tudor, Word-PDF, Images-GPS and Multimedia collections) to see if Dr Bainbridge's PERL_HASH_SEED env var addition fixes all of those collections diffcol failures, making the extra sorting redundant. In that case, I will recommit those model collections after updating their col configurations to not do the extra sorting.

File:

• other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl (modified) (7 diffs)

other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl

@@ -196 +196 @@
     ## where the model-col needs to be built & committed from a Win machine),
     ## need to comment out the following 3 newly added lines and fix any remaining issues
-    if($ENV{'GSVERSION'} eq "3") {
-        return @Errors;
-    }
-    
+#   if($ENV{'GSVERSION'} eq "3") {
+#       return @Errors;
+#   }
+  if($ENV{'GSVERSION'} eq "3") {
     # Testing the build.cfg
     my $strModelBcfg =  &FileUtils::filenameConcatenate($strModelCol,"index","build.cfg");
@@ -310 +310 @@
 
     VobPrint ("\n",$intLevel);
-    
+  } 
     # Testing databases
 
     # index
-    my $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"index","text","$strColName.gdb");
-    my $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"index","text","$strColName.gdb");    
+    my $flat_db_ext = ".gdb";
+    
+    my $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"index","text","$strColName$flat_db_ext");
+    if(!(-e $strModelGdb)){ # try jdb
+        $flat_db_ext = ".jdb";
+        $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"index","text","$strColName$flat_db_ext");
+    }
+    
+    #print STDERR "*********** USING DB EXTENSION: $flat_db_ext\n";
+    #print STDERR "@@@ PERL_PERTURB_KEYS: ".$ENV{'PERL_PERTURB_KEYS'}."\n";
+    
+    my $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"index","text","$strColName$flat_db_ext");    
     my $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol); # returns 0 if no error
     if($strGdbError) {
@@ -322 +332 @@
     
     # archives
-    $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-doc.gdb");
-    $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-doc.gdb");    
+    $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-doc$flat_db_ext");
+    $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-doc$flat_db_ext");    
     $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
     if($strGdbError) {
@@ -329 +339 @@
     }
     
-    $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-src.gdb");
-        $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-src.gdb");
+    $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-src$flat_db_ext");
+        $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-src$flat_db_ext");
         $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
         if($strGdbError) {
@@ -352 +362 @@
     {
     #my $strGdbError = gdbdiff::test_gdb($strModelGdb, $strTestGdb);
+    
+#   $gv_blnDebugging = 1;
+    
     $strGdbError = gdbdiff::test_gdb($strModelGdb, $strTestGdb, $strColName,$gv_test_os, $gv_model_os,$strTestCol,$strModelCol, $gv_blnDebugging);
+#   $gv_blnDebugging = 0;
+    
     if($strGdbError ne "")
     {
@@ -607 +622 @@
         # Files to be skipped because they get generated on one OS but not the other
         # On windows, files of the form col.invf.state.\d\d\d\d get generated (e.g. Small-HTML.invf.state.1228) that aren't there on linux
-        my $skipfiles_re = qr/(\.invf\.state\.\d+$)|~$|earliestDatestamp|fail\.log|-inf-tmp\.(gdb|jdb|lg)$/; # Create a regex of all files to be skipped, see http://perldoc.perl.org/perlop.html
+        my $skipfiles_re = qr/(\.invf\.state\.\d+$)|~$|earliestDatestamp|fail\.log|oai-inf-tmp\.(gdb|jdb|lg)$/; # Create a regex of all files to be skipped, see http://perldoc.perl.org/perlop.html
         @aryInModel = grep { $_ !~ m/$skipfiles_re/ } @aryInModel; # http://stackoverflow.com/questions/174292/what-is-the-best-way-to-delete-a-value-from-an-array-in-perl
         @aryInTest = grep { $_ !~ m/$skipfiles_re/ } @aryInTest;
@@ -1006 +1021 @@
     $gv_test_os = $test_os; # if not specified, defaults to "compute"
     $gv_model_os = $model_os; # tends to be linux
+    
+    #print STDERR "@@@@@ test_os: $test_os\n";
+    #print STDERR "@@@@@ model_os: $model_os\n";
 
     $gv_blnDebugging = $debugging;