Changeset 27730 for other-projects/nightly-tasks/diffcol/trunk/diffcol

Timestamp:

2013-06-28T20:21:41+12:00 (11 years ago)

Author:

ak19

Message:

More diffing issues detected when diffcol ran over the first Word and PDF tutorial. 1. Two ex.File.* fields differ, one to do with date/timestamp and another with permissions (the latter might be avoidable). These 2 ex.File.* metadata fields have been marked for ignoring. 2. The building generated tmp folders containing randomly named subfolders whose names consisted only of digits, possibly named using timestamps. These differ between builds and needed to be normalised also. One final issue remains, and that is that a ghostscript document, one on which the gs command failed, is converted to html with a different line-lengths or line-wraps on the CentOS in which the model col was generated and the Ubuntu on which the testcol was generated. This results in the entire content section of the doc.xml to be marked different between model and test.

Location:

other-projects/nightly-tasks/diffcol/trunk/diffcol

Files:

• diffcol.pl (modified) (10 diffs)
• gdbdiff.pm (modified) (1 diff)

other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl

@@ -115 +115 @@
 sub IndivTest
 {
-    my ($strModelCol,$strTestCol) = @_;
+    my ($strModelCol,$strTestCol,$strColName) = @_;
     my @Errors = ();
     my $intNumberOfErrors = 0;
@@ -127 +127 @@
             my $strTestFolder = &FileUtils::filenameConcatenate($strTestCol,$strEachFolder);
             $intNumberOfErrors = scalar(@Errors);
-            push(@Errors,TestEach($strModelFolder,$strTestFolder,0));
+            push(@Errors,TestEach($strModelFolder,$strTestFolder,0,$strColName));
             $intNumberOfErrors = scalar(@Errors) - $intNumberOfErrors;
             VobPrint("End Comparing \"$strEachFolder\"\n",0);
@@ -404 +404 @@
         VobPrint("Detail Testing Start\n",$intLevel);
     }
-    push(@Errors,TestEach($strModelCol,$strTestCol,$intLevel));
+    push(@Errors,TestEach($strModelCol,$strTestCol,$intLevel,$strColName));
     $intNumberDiffs = scalar(@Errors) - $intNumberDiffs;
 
@@ -564 +564 @@
 sub TestEach
 {
-    my ($strModel,$strTest,$intLevel) = @_;
+    my ($strModel,$strTest,$intLevel,$strColName) = @_;
     my @Errors = ();
 
@@ -593 +593 @@
                 if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png)$/g))
                 {
-                    push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel));
+                    push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel,$strColName));
                 }
                 else
@@ -613 +613 @@
                 if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png)$/g))
                 {
-                    push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel));
+                    push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel,$strColName));
                 }
             }
@@ -625 +625 @@
     else
     {
-        my $ignore_line_re = "<Metadata name=\"(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ImageSize)\">.*</Metadata>\\s*\\n*";
+        my $ignore_line_re = "<Metadata name=\"(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|ImageSize)\">.*</Metadata>\\s*\\n*";    
+        
         my $strResult;
 
@@ -643 +644 @@
         $model_contents =~ s/$ignore_line_re//g;
         $test_contents =~ s/$ignore_line_re//g;
+
+        # tmp dirs have subdirs with random numbers in name, remove randomly named subdir portion of path
+        $model_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g; 
+        $test_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
         
+        # remove all absolute paths upto collect folder from <Metadata /> elements
+        $model_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;
+        $test_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;
+
+
             # equalise/normalise the two doc.xml files for OS differences, if there are any
             my $testIsWin = &isDocXMLFileWindows($test_contents);
@@ -687 +697 @@
         else
         {
+#           print STDERR "**** Diff is: $strResult\n"; # print any differences to the screen
+
             my $strOutput = "Difference Report:\n$strResult\n";
             if ( $strOutputFormat eq "xml" ) {
@@ -873 +885 @@
             if ($gv_strMode eq "Individual")
             {
-                @ErrorsInEachCol = IndivTest($strModelCol,$strTestCol);
+                @ErrorsInEachCol = IndivTest($strModelCol,$strTestCol,$strColName);
             }
             #----##

other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm

@@ -66 +66 @@
     # The total_numbytes field can vary depending on how many backslashes exist in the urls in the main body text, as each
     # of these windows slashes get escaped with another backslash, and the resulting string is used as key into rel link db
-    my $ignore_line_re = "\n<(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|total_numbytes)>([^\n])*";
+    my $ignore_line_re = "\n<(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|total_numbytes)>([^\n])*";
     $model_text =~ s/$ignore_line_re//g;
     $test_text =~ s/$ignore_line_re//g;
 
+    # tmp dirs have subdirs with random numbers in name, remove subdir
+    $model_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
+    $test_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
 
     # if the OS doesn't match and one of them is windows, extra work needs to be done to bring the db files