Changeset 27743 for other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm

Timestamp:

2013-07-03T21:37:18+12:00 (11 years ago)

Author:

ak19

Message:

Basic Word-PDF collection now has the same number of diffing errors on Windows upon diffcol as on Linux and Mac. Needed to do a lot of special processing for windows: to remove carriage returns introduced into doc.xml when doing a multiread on the html version of a pdf doc after it has been converted to html. (And similarly, needed to get rid of windows carriage returns introduced into ex.Title meta for pdf01.pdf converted to HTML. This was handled in HTMLPlugin). Further special tags need either to be ignored, if they're time stamps, or specially handled if they're filepaths. Not sure if it's the encoding setting in multiread or maybe the locale that is introducing the carriage returns, but am dealing with this at the point of diffcol since it's not a 'problem' in Greenstone, just an inconsistency across OS-es. There's still one diffcol error remaining for this collection on all 3 OS: one word document has a different word wrap length on the machine where the model col was built compared to the wrap length on the other machines. This may be a setting to wvware or else libreoffice/staroffice, if these are used.

File:

• other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm (modified) (5 diffs)

other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm

@@ -59 +59 @@
     my $test_text = readin_gdb($test_cmd);
 
-#   my $savepath = &getcwd."/../"; # TASK_HOME env does not exist at this stage, but it's one level up from current directory
+#   my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory
 #   print_string_to_file($test_text, $savepath.$dbname."_test.out");    
 #   print_string_to_file($model_text, $savepath.$dbname."_model.out");
@@ -92 +92 @@
             # Better regex is of the form /BEGIN((?:(?!BEGIN).)*)END/, see http://docstore.mik.ua/orelly/perl/cookbook/ch06_16.htm
 
-            $model_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;          
-            $test_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;                       
+            $model_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+            $test_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+            #$model_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;         
+            #$test_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;                      
     }
     
@@ -116 +118 @@
             for my $line (split /^/, $$win_text) { # split the string into newlines
                 
-                if($line =~ m@^<assoc-file>(.*)(\s+)@s) {
-                    $line = $1; # may be a short file name
+                # assoc-file and meta-file contain filepaths, ensure these are long windows file paths now (will later convert to linux slashes)    
+                if($line =~ m@^<(assoc-file|meta-file)>(.*)(\s+)@s) { 
+                    $line = $2; # may be a short file name
                     # perhaps test here if it is a shortfilename? should match /CAPS....~number(.ext)/
                 
-                    $line = "<assoc-file>".&Win32::GetLongPathName($line)."$2"; # make it a long file name and prefix assoc-file to it again                    
+                    $line = "<$1>".&Win32::GetLongPathName($line)."$3"; # make it a long file name and prefix assoc-file/meta-file tagname to it again                  
                 }
                 $tmp .= $line;
@@ -127 +130 @@
         }
         
-        # slashes in windows text need to be turned into linux style slashes
-        $$win_text =~ s@\\@/@g;
+        
+        # index gdb file
+        if($dbname =~ m/$strColName/) {
+            my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
+            for my $line (split /^/, $$win_text) { # split the string into newlines
+                
+                if($line =~ m@^<ex.File.Directory>(.*)@s) { # word-pdf collection contains double windows backslashes
+                    (my $escaped_path = $1) =~ s@\\\\@\\@g;             
+                    $line = "<ex.File.Directory>$escaped_path";
+                } 
+                elsif($line =~ m@^<Title>(.*)@s) {
+#                   print STDERR "***** TITLE: |$1|\n";
+                
+                    # word-pdf collection: Title of ps files contain new lines at end when
+                    # GreenstoneXMLPlugin::xml_end_tag() writes the Title back out after utf8 decode
+                    # if($metadata_name eq "Title") { $metadata_value =~ s/[\n\r]*$//; }
+                
+                    (my $title = $1) =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
+                    $line = "<Title>$title\n"; # add single newline                 
+                }
+                $tmp .= $line;
+            }
+            $$win_text = $tmp;          
+        }
+        
+        
+        # slashes in windows metadata text need to be turned into linux style slashes
+        $$win_text =~ s@\\@/@g; #$$win_text =~ s@\\([^n|r|\|"])@/$1@g; # filepath something\rtf remains something\rtf
         
         # cut down absolute paths to files to just collect/colname/.../file, same as before
-        $$lin_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;
-        $$win_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;
+        $$lin_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg; # $$lin_text =~ s@^([^\\\/]*(//)?).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+        $$win_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;       
         
         # for the windows text, need to further get rid of the driveletter after [ or <meta>
@@ -166 +195 @@
     my ($dbtailname, $db_contents) = @_; # db filename without suffix
     
-    if($dbtailname !~ m/archiveinf/) { # only archiveinf-doc and archive-inf source need special Windows processing, not col.gdb
-        return 0;
-    }
+    #if($dbtailname !~ m/archiveinf/) { # only archiveinf-doc and archive-inf source need special Windows processing, not col.gdb
+    #   return 0;
+    #}
     return ($db_contents =~ m/\\/) ? 1 : 0; # windows slashes detected. Better test would be: [Something\something] OR <tag>something\something
     # for doc.xml: