Changeset 27766

Timestamp:

2013-07-05T22:20:39+12:00 (11 years ago)

Author:

ak19

Message:

Many additions to equalise the PDFBox collection metadata, since it generates an intermediate src file in the GSDLHOME/tmp directory whose path and random filename ends up in all the doc.xml and index/col.gdb files

Location:

other-projects/nightly-tasks/diffcol/trunk/diffcol

Files:

• diffcol.pl (modified) (3 diffs)
• gdbdiff.pm (modified) (2 diffs)

other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl

@@ -45 +45 @@
 use diffutil;
 use Text::Diff;
+use Cwd;
 
 #--Global Variables Declaration-----------
@@ -667 +668 @@
             
         
-
         # tmp dirs have subdirs with random numbers in name, remove randomly named subdir portion of path
+        # these tmpdirs are located inside the collection directory
         $model_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g; 
         $test_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
@@ -674 +675 @@
         # remove all absolute paths upto collect folder from <Metadata /> elements
         $model_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;
-        $test_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;    
+        $test_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;        
+        
+        # The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
+        # These tmpdirs are located inside the toplevel *greenstone* directory
+        (my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;      
+        my $tmpfile_regex = "<Metadata name=\"URL\">http://$gsdlhome_re/tmp/([^\.]*)(\..{3,4})</Metadata>"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long  
+        if($test_contents =~ m@$tmpfile_regex@) {           
+            # found a match, replace the tmp file name with "random", keeping the original file extension 
+            # in <Metadata name="OrigSource|URL|UTF8URL|gsdlconvertedfilename">
+        
+            my ($old_tmp_filename, $ext) = ($1, $2);            
+            my $new_tmp_filename = "random";            
+            
+            $tmpfile_regex = "(<Metadata name=\"(URL|UTF8URL|gsdlconvertedfilename|OrigSource)\">(http://)?)($gsdlhome_re)?(/tmp/)?$old_tmp_filename($ext</Metadata>)";
+            if($5) { 
+                $test_contents =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@g;
+            } else { # OrigSource contains only the filename
+                $test_contents =~ s@$tmpfile_regex@$1$new_tmp_filename$6@g;
+            }
+            
+            # modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename           
+            $tmpfile_regex = "(<Metadata name=\"(URL|UTF8URL|gsdlconvertedfilename|OrigSource)\">(http://)?)(.*)?(/tmp/)?.*($ext</Metadata>)";
+            if($5) { 
+                $model_contents =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@g;
+            } else { # OrigSource contains only the filename
+                $model_contents =~ s@$tmpfile_regex@$1$new_tmp_filename$6@g;
+            }
+        }
         
 #       my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory       

other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm

@@ -71 +71 @@
 
     # tmp dirs have subdirs with random numbers in name, remove subdir
+    # these tmpdirs are located inside the collection directory
     $model_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
     $test_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
@@ -167 +168 @@
         
     } # end of equalising differences between a windows collection's db file and linux coll's db file
+    
+    # The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
+    # These tmpdirs are located inside the toplevel *greenstone* directory
+    (my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;      
+    my $tmpfile_regex = "<URL>http://$gsdlhome_re/tmp/([^\.]*)(\..{3,4})"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long           
+    if($test_text =~ m@$tmpfile_regex@g) {          
+        # found a match, replace the tmp file name with "random", keeping the original file extension 
+        # in <OrigSource|URL|UTF8URL|gsdlconvertedfilename>
+    
+        # This code is slightly different from doc.xml because each document has its own doc.xml, so this needs to be done
+        # only once for doc.xml, but multiple times in index/col.gdb since it contains the random filenames of all docs in the col  
+        #my ($old_tmp_filename, $ext) = ($1, $2);
+        
+        my $new_tmp_filename = "random";            
+    
+        
+        $tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)($gsdlhome_re)?(/tmp/)?.*(\..{3,4})";
+        if($5) { 
+            $test_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+        } else { # OrigSource contains only the filename
+            $test_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+        }
+        
+        # modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename           
+        $tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)(.*)?(/tmp/)?.*(\..{3,4})";
+        if($5) { 
+            $model_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+        } else { # OrigSource contains only the filename
+            $model_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+        }
+        
+        # index/col.gdb also has entries for the random tmp file names in the form: [http://research/ak19/GS2bin_5July2013/tmp/F639.html]
+        # need to equalise these also
+        $test_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg;
+        $model_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg;
+    }   
 
     # now can go back to using $model_text and $test_text