Changeset 27743 for other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl

Timestamp:

2013-07-03T21:37:18+12:00 (11 years ago)

Author:

ak19

Message:

Basic Word-PDF collection now has the same number of diffing errors on Windows upon diffcol as on Linux and Mac. Needed to do a lot of special processing for windows: to remove carriage returns introduced into doc.xml when doing a multiread on the html version of a pdf doc after it has been converted to html. (And similarly, needed to get rid of windows carriage returns introduced into ex.Title meta for pdf01.pdf converted to HTML. This was handled in HTMLPlugin). Further special tags need either to be ignored, if they're time stamps, or specially handled if they're filepaths. Not sure if it's the encoding setting in multiread or maybe the locale that is introducing the carriage returns, but am dealing with this at the point of diffcol since it's not a 'problem' in Greenstone, just an inconsistency across OS-es. There's still one diffcol error remaining for this collection on all 3 OS: one word document has a different word wrap length on the machine where the model col was built compared to the wrap length on the other machines. This may be a setting to wvware or else libreoffice/staroffice, if these are used.

File:

• other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl (modified) (4 diffs)

other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl

@@ -591 +591 @@
                 my $strNewTest = &FileUtils::filenameConcatenate($strTest,$strEachFile);
                 # now additionally ignoring the earliestDatestamp file and the index/idx/*.idh binary file when diffing file 
-                if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png)$/g))
+                if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png|wmf)$/g)) # wmf = windows meta file
                 {
                     push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel,$strColName));
@@ -611 +611 @@
                 my $strNewModel = &FileUtils::filenameConcatenate($strModel,$strEachFile);
                 my $strNewTest = &FileUtils::filenameConcatenate($strTest,$strEachFile);
-                if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png)$/g))
+                if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png|wmf)$/g))
                 {
                     push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel,$strColName));
@@ -645 +645 @@
         $test_contents =~ s/$ignore_line_re//g;
 
+
+            # equalise/normalise the two doc.xml files for OS differences, if there are any
+            my $testIsWin = &isDocXMLFileWindows($test_contents);
+            my $modelIsWin = &isDocXMLFileWindows($model_contents);
+            
+            if($testIsWin != $modelIsWin) { # one of the 2 collections is built on windows, the other on linux, so need to make newlines constant
+            
+                my $win_contents = $testIsWin ? \$test_contents : \$model_contents;
+            
+                # remove all carriage returns \r - introduced into doc.xml by multiread after pdf converted to html
+                $$win_contents =~ s@[\r]@@g;
+            
+                # make all single windows slashes into single unix slashes
+                $$win_contents =~ s@([^\\])\\([^\\])@$1\/$2@g;
+                # make windows \r newlines into constant \n newlines. Already handled when \r got replaced
+                #$$win_contents =~ s@\r\n@\n@mg; # #http://stackoverflow.com/questions/650743/in-perl-how-to-do-you-remove-m-from-a-file
+                
+                #FOR MAC: old macs use CR carriage return (see http://www.perlmonks.org/?node_id=745018), so replace with \n?)
+                # $$win_contents =~ s@\r@\n@mg;
+            }
+            
+        
+
         # tmp dirs have subdirs with random numbers in name, remove randomly named subdir portion of path
         $model_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g; 
@@ -651 +674 @@
         # remove all absolute paths upto collect folder from <Metadata /> elements
         $model_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;
-        $test_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;
-
-
-            # equalise/normalise the two doc.xml files for OS differences, if there are any
-            my $testIsWin = &isDocXMLFileWindows($test_contents);
-            my $modelIsWin = &isDocXMLFileWindows($model_contents);
-            
-            if($testIsWin != $modelIsWin) { # one of the 2 collections is built on windows, the other on linux, so need to make newlines constant
-            
-                my $win_contents = $testIsWin ? \$test_contents : \$model_contents;
-            
-                # make all windows slashes into unix slashes
-                $$win_contents =~ s@\\@\/@g;
-                # make windows \r newlines into constant \n newlines        
-                $$win_contents =~ s@\r\n@\n@mg; # #http://stackoverflow.com/questions/650743/in-perl-how-to-do-you-remove-m-from-a-file
-                
-                #FOR MAC: old macs use CR carriage return (see http://www.perlmonks.org/?node_id=745018), so replace with \n?)
-                # $$win_contents =~ s@\r@\n@mg;
-            }
+        $test_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;    
+        
+#       my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory       
+#       &gdbdiff::print_string_to_file($model_contents, $savepath."model_doc.xml");
+#       &gdbdiff::print_string_to_file($test_contents, $savepath."test_doc.xml");
         
         $strResult = diff \$model_contents, \$test_contents, { STYLE => "OldStyle" };