Index: other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl
===================================================================
--- other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl	(revision 36868)
+++ other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl	(revision 36876)
@@ -42,5 +42,5 @@
 use logdiff;
 use cfgdiff;
-use gdbdiff;
+use flatdbdiff;
 use diffutil;
 use Text::Diff;
@@ -314,5 +314,5 @@
 	
 	my $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"index","text","$strColName$flat_db_ext");	
-	my $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol); # returns 0 if no error
+	my $strGdbError = &FlatdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol); # returns 0 if no error
 	if($strGdbError) {
 	    push(@Errors,$strGdbError);
@@ -322,5 +322,5 @@
 	$strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-doc$flat_db_ext");
 	$strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-doc$flat_db_ext");	
-	$strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
+	$strGdbError = &FlatdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
 	if($strGdbError) {
 	    push(@Errors,$strGdbError);
@@ -329,5 +329,5 @@
 	$strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-src$flat_db_ext");
         $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-src$flat_db_ext");
-        $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
+        $strGdbError = &FlatdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
         if($strGdbError) {
             push(@Errors,$strGdbError);
@@ -340,6 +340,6 @@
 
 
-# At present handles gdbm - need to expand to allow for jdbm and other db types
-sub GdbDiff
+# At present handles gdbm and jdbm - may need to expand to allow for other db types
+sub FlatdbDiff
 {
     my ($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol) = @_;
@@ -349,7 +349,7 @@
     if(-e $strModelGdb && -e $strTestGdb)
     {
-	#my $strGdbError = gdbdiff::test_gdb($strModelGdb, $strTestGdb);
-	
-	$strGdbError = gdbdiff::test_gdb($strModelGdb, $strTestGdb, $strColName,$gv_test_os, $gv_model_os,$strTestCol,$strModelCol, $gv_blnDebugging);
+	#my $strGdbError = flatdbdiff::test_flat_db($strModelGdb, $strTestGdb);
+	
+	$strGdbError = flatdbdiff::test_flat_db($strModelGdb, $strTestGdb, $strColName,$gv_test_os, $gv_model_os,$strTestCol,$strModelCol, $gv_blnDebugging);
 	
 	if($strGdbError ne "")
@@ -569,5 +569,5 @@
 
 # this function is only called on DocXMLFiles. 
-# so far, only doc.xml files need special Windows processing (db files' OS-sensitivity are handled in gdbdiff.pm)
+# so far, only doc.xml files need special Windows processing (db files' OS-sensitivity are handled in flatdbdiff.pm)
 # Returns true if the doc.xml contains windows style slashes in the gsdlsourcefilename meta field
 sub isDocOrMETSXMLFileWindows
@@ -780,9 +780,9 @@
 		if($gv_blnDebugging) {# && $gv_intVerbosity > 0) {
 		    my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory		
-		    &gdbdiff::print_string_to_file($model_contents, $savepath."model_docmets.xml");
-		    &gdbdiff::print_string_to_file($test_contents, $savepath."test_docmets.xml");
+		    &flatdbdiff::print_string_to_file($model_contents, $savepath."model_docmets.xml");
+		    &flatdbdiff::print_string_to_file($test_contents, $savepath."test_docmets.xml");
 #		    if($strModel =~ m/(HASH0164.dir)/) { # list the HASH dirs for which you want the doc.xml file generated, to inspect specific doc.xml files
-#			&gdbdiff::print_string_to_file($model_contents, $savepath."$1_model_doc.xml");
-#			&gdbdiff::print_string_to_file($test_contents, $savepath."$1_test_doc.xml");
+#			&flatdbdiff::print_string_to_file($model_contents, $savepath."$1_model_doc.xml");
+#			&flatdbdiff::print_string_to_file($test_contents, $savepath."$1_test_doc.xml");
 #		    }
 		}		
Index: other-projects/nightly-tasks/diffcol/trunk/diffcol/flatdbdiff.pm
===================================================================
--- other-projects/nightly-tasks/diffcol/trunk/diffcol/flatdbdiff.pm	(revision 36876)
+++ other-projects/nightly-tasks/diffcol/trunk/diffcol/flatdbdiff.pm	(revision 36876)
@@ -0,0 +1,369 @@
+package gdbdiff;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use util;
+use diffutil;
+use Text::Diff;
+use Cwd;
+
+if ($^O =~ m/mswin/i) {
+    require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
+}
+
+sub readin_gdb
+{
+    my ($cmd) = @_;
+
+    open(PIN,"$cmd|") 
+	|| die "Unable to open pipe to $cmd: $!\n";
+
+    my $text_content = "";
+
+    while (defined (my $line = <PIN>)) {
+	$text_content .= $line;
+    }
+
+    close(PIN);
+    return $text_content;
+}
+
+sub read_db 
+{
+    # need to sort text output of both test and model col database files, to normalise them for the comparison
+    # the -sort option to db2txt was added specifically to support diffcol
+    my($db_file) = @_;
+    my $db_cmd = "db2txt -sort $db_file 2>&1";
+    if($db_file =~ m/\.jdb$/) {
+	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
+	$db_cmd = "$runperl jdb2txt.pl -sort $db_file";
+    }
+
+    my $db_text = readin_gdb($db_cmd);
+    return $db_text;
+}
+
+sub text_to_db_to_text
+{
+    my($db_text, $db_filename) = @_;
+
+    # http://stackoverflow.com/questions/1909262/how-can-i-pipe-input-into-a-java-command-from-perl
+    
+    if($db_filename =~ m/\.jdb$/) {
+	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
+	open PIPE, "| $runperl txt2jdb.pl $db_filename";
+	print PIPE "$db_text";
+	close(PIPE);
+    } else {
+	open PIPE, "| txt2db $db_filename";
+	print PIPE "$db_text";
+	close(PIPE);
+    }
+    return &read_db("$db_filename");
+}
+
+# for debugging. Prints txt contents of db to file
+sub print_string_to_file 
+{
+    my ($text, $outfile) = @_;
+	
+    open(FOUT, ">$outfile") or die "ERROR failed to write to $outfile: $!\n";
+    print FOUT $text;
+    close(FOUT);
+}
+
+sub test_gdb
+{
+    my ($full_modeldb, $full_testdb, $strColName, $test_os, $model_os, $strTestCol, $strModelCol, $debugging) = @_;
+
+   # print "Now is testing database\n";
+	my ($dbname, $dirname, $suffix)= &File::Basename::fileparse($full_testdb, "\\.[^\\.]+\$");
+   
+    my $model_text = read_db($full_modeldb);
+    my $test_text = read_db($full_testdb);
+
+    my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory
+    if($debugging) {
+	print_string_to_file($test_text, $savepath.$dbname."_test.out1");	
+	print_string_to_file($model_text, $savepath.$dbname."_model.out1");
+    }
+
+    # filter out the fields that can be ignored in the two database files
+	# The total_numbytes field can vary depending on how many backslashes exist in the urls in the main body text, as each
+	# of these windows slashes get escaped with another backslash, and the resulting string is used as key into rel link db
+    my $ignore_line_re = "\n<(FileSize|lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|ex.File.FileAccessDate|ex.File.FileInodeChangeDate|ex.File.FileCreateDate|total_numbytes|ex.Composite.LightValue)>([^\n])*";
+    $model_text =~ s/$ignore_line_re//g;
+    $test_text =~ s/$ignore_line_re//g;
+
+    # tmp dirs have subdirs with random numbers in name, remove subdir
+	# these tmpdirs are located inside the collection directory
+    $model_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
+    $test_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
+
+
+#print STDERR "@@@@ DEBUGGING: $debugging\n";
+#print STDERR "******** full_modeldb: $full_modeldb\n$model_text\n\n";
+#print STDERR "******** full_testdb: $full_testdb\n$test_text\n\n";
+	
+	# if the OS doesn't match and one of them is windows, extra work needs to be done to bring the db files
+	# in test and model collection to an even base for comparison
+	
+	my $testIsWin = ($test_os ne "compute") ? ($test_os eq "windows") : &isDBWindowsSensitive($dbname, $test_text);
+	my $modelIsWin = ($model_os ne "compute") ? ($model_os eq "windows") : &isDBWindowsSensitive($dbname, $model_text);
+	
+	if($testIsWin == $modelIsWin) {		
+	# both linux or both windows, do the basic test we did on linux machines:
+		    # ignore absolute path prefixes in modelcol and testcol (necessary for archiveinf-doc and -src.gdb files)
+
+			# Remember the original model col on SVN could have been built anywhere, 
+			# and in the gdb files, absolute paths are stored to the collection location. 
+			# Crop these paths to the collect/<colname> point.
+			
+			# Entries are of the form [Entry] or <Entry>. In order to do a sensible diff,
+			# need to remove the prefix to the collect/colname folder in any (absolute) path that occurs in Entry
+			# E.g. [/full/path/collect/colname/import/file.ext] should become [collect/colname/import/file.ext]
+			# Better regex is of the form /BEGIN((?:(?!BEGIN).)*)END/, see http://docstore.mik.ua/orelly/perl/cookbook/ch06_16.htm
+
+			$model_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+			$test_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+			#$model_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;			
+			#$test_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;						
+	}
+	
+	else { # one of the collections was built on windows
+		# handling slashes and other differences between a model coll built on one OS (e.g. linux) 
+		# and a test collection built and diffed on another OS (windows)
+	
+		my ($win_text, $lin_text); # references
+		my $collection_path = $strTestCol; # full path to a windows collection
+		
+		if($testIsWin) {
+			$collection_path = $strTestCol; # test collection path is windows 
+			$win_text = \$test_text;
+			$lin_text = \$model_text;
+		} else {
+			$collection_path = $strModelCol; # model collection path is windows
+			$win_text = \$model_text;
+			$lin_text = \$test_text;
+		}
+		
+		if($dbname =~ m/archiveinf-doc/) {
+			
+			(my $collection_path_re = $collection_path) =~ s@\\@\\\\@g;
+			
+			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
+	
+			# convert short filenames to long perl: 
+			# http://www.mombu.com/programming/perl/t-convert-dos-83-filenames-to-win32-long-filenames-using-perl-525448.html
+			for my $line (split /^/, $$win_text) { # split the string into newlines
+				
+				# assoc-file and meta-file contain filepaths, ensure these are long windows file paths now (will later convert to linux slashes)	
+				if($line =~ m@^<(assoc-file|meta-file|src-file)>(.*)(\s+)@s) {
+					my ($field, $value, $suffix) = ($1, $2, $3);
+					$line = $value; # it may be a short file name	
+					
+					if($line !~ m/^\@/) { # if the path doesn't use a "relative" @GSPATH@ placeholder string, but is an absolute path instead						
+						# perhaps test here if it is a shortfilename? should match /CAPS....~number(.ext)/						
+						$line = "<$field>".&Win32::GetLongPathName($line)."$suffix"; # make it a long file name and prefix assoc-file/meta-file tagname to it again					
+					} 
+					else { # if $line contains @THISCOLLECTIONPATH@, still need to deal with DOS filenames suffixes: 
+						# replace placeholder with absolute path and expand to long filename, then insert placeholder in its original place again						
+						$line =~ s/\@THISCOLLECTPATH\@/$collection_path/;
+						$line = &Win32::GetLongPathName($line);
+						$line =~ s/^$collection_path_re/\@THISCOLLECTPATH\@/;
+						$line = "<$field>".$line."$suffix";
+					}
+				}
+				$tmp .= $line;
+			}
+			$$win_text = $tmp;			
+		}
+		
+		
+		# index gdb file
+		if($dbname =~ m/$strColName/) {
+			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
+			for my $line (split /^/, $$win_text) { # split the string into newlines			
+
+				# In the following regex, add any .gdb fieldnames that represent a path and so would contain double backslashes 
+				# on Windows (to escape the single backlash of win filepaths). They will be turned into single-backslashes here, 
+				# and converted into single forward slashes futher below when the txt version of the win gdb file is normalised 
+				# to compare it with the linux version. 
+				# E.g. On windows, the Word-PDF collection(s) contains double backslashes in the ex.File.Directory field
+				# the MARC-Exploded collection contains double backslashes in the null_file entry field of the .gdb file				
+				if($line =~ m@^<(ex.File.Directory|null_file)>(.*)@s) {
+					my ($fieldname, $escaped_path) = ($1, $2);
+					$escaped_path =~ s@\\\\@/@g; #(my $escaped_path = $2) =~ s@\\\\@\\@g;
+					$line = "<$fieldname>$escaped_path";
+				} 
+				elsif($line =~ m@^<Title>(.*)@s) {
+#					print STDERR "***** TITLE: |$1|\n";
+				
+					# word-pdf collection: Title of ps files contain new lines at end when
+					# GreenstoneXMLPlugin::xml_end_tag() writes the Title back out after utf8 decode
+					# if($metadata_name eq "Title") { $metadata_value =~ s/[\n\r]*$//; }
+				
+					(my $title = $1) =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
+					$line = "<Title>$title\n"; # add single newline					
+				}
+				$tmp .= $line;
+			}
+			$$win_text = $tmp;
+		
+			# slashes in windows metadata text need to be turned into linux style slashes. 
+			# index\col.gdb uses double backslashes, and single for \n,\t
+			#$$win_text =~ s@\\\\@/@g;
+		}
+		else { # archiveinf gdb file
+		
+			# slashes in windows metadata text need to be turned into linux style slashes. 
+			# In the two archivesinf gdb files, filepaths may use single backslashes
+			$$win_text =~ s@\\@/@g; #$$win_text =~ s@\\([^n|r|\|"])@/$1@g; # filepath something\rtf remains something\rtf
+		}
+		
+		# cut down absolute paths to files to just collect/colname/.../file, same as before
+		$$lin_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg; # $$lin_text =~ s@^([^\\\/]*(//)?).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+		$$win_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;		
+		
+		# for the windows text, need to further get rid of the driveletter after [ or <meta>
+		$$win_text =~ s@^(\[|<[^>]*>)[a-zA-Z]:collect@$1collect@mg;		
+		
+	} # end of equalising differences between a windows collection's db file and linux coll's db file
+	
+	
+	# Windows or linux: if index is a flat db file, then ensure the docIDs listed in <contains> field of
+	# both test and model flat db file are alphabetically sorted. So to the numbers in <mdoffset> field.
+	# Despite PERL_PERTURB_KEYS envvar being set to 0 on both machine when generating model collections
+	# and when test collections were generated on test machine, still collections like Images-GPS and some
+	# other colls list items in <contains> and <mdoffset> in different orders. So reordering alphabetically.	
+	#if($dbname =~ m/$strColName/) {	
+		# regex modifiers mge: multi-line, global (replace as many as match), e allows function call in substitution
+		##$model_text =~ s@^<contains>(.*)@sort_contains_field($1, "MODEL", $debugging)@mge;
+		##$test_text =~ s@^<contains>(.*)@sort_contains_field($1, "TEST", $debugging)@mge;
+	#	$model_text =~ s@^<(contains|mdoffset)>(.*)@sort_field_value($1, $2, "MODEL", $debugging)@mge;
+	#	$test_text =~ s@^<(contains|mdoffset)>(.*)@sort_field_value($1, $2, "TEST", $debugging)@mge;		
+	#}
+	
+	# The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
+	# These tmpdirs are located inside the toplevel *greenstone* directory
+	(my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;		
+	$gsdlhome_re = ".*" unless $$ENV{'GSDLHOME'};
+	my $tmpfile_regex = "<URL>http://$gsdlhome_re/tmp/([^\.]*?)(\..{3,4})"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long			
+	if($test_text =~ m@$tmpfile_regex@g) {			
+		# found a match, replace the tmp file name with "random", keeping the original file extension 
+		# in <OrigSource|URL|UTF8URL|gsdlconvertedfilename>
+	
+		# This code is slightly different from doc.xml because each document has its own doc.xml, so this needs to be done
+		# only once for doc.xml, but multiple times in index/col.gdb since it contains the random filenames of all docs in the col	
+		#my ($old_tmp_filename, $ext) = ($1, $2);
+		
+		my $new_tmp_filename = "random";			
+	
+		
+		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)($gsdlhome_re)?(/tmp/)?.*?(\..{3,4})";
+		if($5) { 
+			$test_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+		} else { # OrigSource contains only the filename
+			$test_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+		}
+		
+		# modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename			
+		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)(.*)?(/tmp/)?.*?(\..{3,4})";
+		if($5) { 
+			$model_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+		} else { # OrigSource contains only the filename
+			$model_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+		}
+		
+		# index/col.gdb also has entries for the random tmp file names in the form: [http://research/ak19/GS2bin_5July2013/tmp/F639.html]
+		# need to equalise these also. Sadly, when there are multiple intermediate files, their random tmp filenames are not 
+		# guaranteed to be generated in the same (alphabetical/numerical) order between model and test collection, so the 
+		# HASH OIDs, although all of them accounted for, appear in a different order. So we have to remove the Hash OIDs.
+		#$test_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg; # HASH OIDs can appear in different order
+		#$model_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg;
+		
+		$test_text  =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
+		$model_text =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
+		
+		# need to re- sort the keys, now that the absolute paths to tmp locations has been removed
+		# so that we get the tmp files in the same order in both model and test collections
+		
+		$model_text = text_to_db_to_text($model_text, "model.gdb");
+		$test_text = text_to_db_to_text($test_text, "test.gdb");
+	}	
+
+    # now can go back to using $model_text and $test_text
+
+    if($debugging) {
+	print_string_to_file($test_text, $savepath.$dbname."_test.out");	
+	print_string_to_file($model_text, $savepath.$dbname."_model.out");
+    }
+
+    my $report_type = "OldStyle"; # Can not change this type.
+    my $diff_gdb = diff \$model_text, \$test_text, { STYLE => $report_type };
+    
+    # leaving the ignore regex as it used to be in the following, in case it helps with single line comparisons
+    $diff_gdb = &diffutil::GenerateOutput($diff_gdb,"^<(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ImageSize)>.*");
+
+    if($diff_gdb eq "")
+    {
+	return "";
+    }
+    else
+    {
+	return "Difference Report: Differences found in the Database file: \n$diff_gdb";
+    }
+    # Call diff?
+}
+
+# Unused, but may come in handy when debugging again: regex substitution helper function
+sub sort_field_value {
+	my($fieldname, $fieldvalue, $displayStr, $debugging) = @_;
+	
+	print STDERR "\n$displayStr BEFORE sort: <$fieldname>$fieldvalue\n" if($debugging);
+	
+	$fieldvalue =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
+	my @values_list = split(';', $fieldvalue);
+	@values_list = sort @values_list;
+	$fieldvalue = "<$fieldname>".join(';', @values_list). "\n";
+	
+	print STDERR "$displayStr AFTER  sort: $fieldvalue\n" if($debugging);
+	
+	return $fieldvalue;
+}
+
+
+# returns true if the contents are windows AND it matters for the diffing on the db that it's windows
+# For col.gdb it does not seem to matter so far, if it is generated on a windows machine and to be compared to a linux-generated col.gdb
+sub isDBWindowsSensitive 
+{
+	my ($dbtailname, $db_contents) = @_; # db filename without suffix
+	
+#	return ($db_contents =~ m/\\/) ? 1 : 0; # windows slashes detected. Better test would be: [Something\something] OR <tag>something\something	
+	
+	if($dbtailname =~ m/^archiveinf-doc/) {
+		return ($db_contents =~ m@<src-file>[a-zA-Z]:\\@) ? 1 : 0; # <src-file>C:\path
+	} 
+	elsif($dbtailname =~ m/^archiveinf-src/) { # <src-file>C:\path
+		return ($db_contents =~ m@\[[a-zA-Z]:\\@) ? 1 : 0; # [C:\path]
+	} 
+	else { # index/col.gdb file
+		if ($db_contents =~ m@<URL>http://[a-zA-Z]:/@) { # <URL>http://C:/path
+			return 1;
+		}		
+		elsif ($db_contents =~ m@^(<URL>http://[a-zA-Z]:/)|(<null_file>[^\\]*\\)@m) { # <URL>http://C:/path OR <null_file>CMSwp-all.00000001\\00000035.nul			
+			return 1;
+		}
+		elsif ($db_contents =~ m@^(<ex.File.Directory>[a-zA-Z]:\\\\)@m) { # <ex.File.Directory>C:\\path\\path for OAI collection
+			return 1;
+		}
+		return 0;		
+	}	
+}
+
+1;
Index: other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm
===================================================================
--- other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm	(revision 36868)
+++ 	(revision )
@@ -1,369 +1,0 @@
-package gdbdiff;
-
-BEGIN {
-    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
-    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
-}
-
-use util;
-use diffutil;
-use Text::Diff;
-use Cwd;
-
-if ($^O =~ m/mswin/i) {
-    require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
-}
-
-sub readin_gdb
-{
-    my ($cmd) = @_;
-
-    open(PIN,"$cmd|") 
-	|| die "Unable to open pipe to $cmd: $!\n";
-
-    my $text_content = "";
-
-    while (defined (my $line = <PIN>)) {
-	$text_content .= $line;
-    }
-
-    close(PIN);
-    return $text_content;
-}
-
-sub read_db 
-{
-    # need to sort text output of both test and model col database files, to normalise them for the comparison
-    # the -sort option to db2txt was added specifically to support diffcol
-    my($db_file) = @_;
-    my $db_cmd = "db2txt -sort $db_file 2>&1";
-    if($db_file =~ m/\.jdb$/) {
-	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
-	$db_cmd = "$runperl jdb2txt.pl -sort $db_file";
-    }
-
-    my $db_text = readin_gdb($db_cmd);
-    return $db_text;
-}
-
-sub text_to_db_to_text
-{
-    my($db_text, $db_filename) = @_;
-
-    # http://stackoverflow.com/questions/1909262/how-can-i-pipe-input-into-a-java-command-from-perl
-    
-    if($db_filename =~ m/\.jdb$/) {
-	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
-	open PIPE, "| $runperl txt2jdb.pl $db_filename";
-	print PIPE "$db_text";
-	close(PIPE);
-    } else {
-	open PIPE, "| txt2db $db_filename";
-	print PIPE "$db_text";
-	close(PIPE);
-    }
-    return &read_db("$db_filename");
-}
-
-# for debugging. Prints txt contents of db to file
-sub print_string_to_file 
-{
-    my ($text, $outfile) = @_;
-	
-    open(FOUT, ">$outfile") or die "ERROR failed to write to $outfile: $!\n";
-    print FOUT $text;
-    close(FOUT);
-}
-
-sub test_gdb
-{
-    my ($full_modeldb, $full_testdb, $strColName, $test_os, $model_os, $strTestCol, $strModelCol, $debugging) = @_;
-
-   # print "Now is testing database\n";
-	my ($dbname, $dirname, $suffix)= &File::Basename::fileparse($full_testdb, "\\.[^\\.]+\$");
-   
-    my $model_text = read_db($full_modeldb);
-    my $test_text = read_db($full_testdb);
-
-    my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory
-    if($debugging) {
-	print_string_to_file($test_text, $savepath.$dbname."_test.out1");	
-	print_string_to_file($model_text, $savepath.$dbname."_model.out1");
-    }
-
-    # filter out the fields that can be ignored in the two database files
-	# The total_numbytes field can vary depending on how many backslashes exist in the urls in the main body text, as each
-	# of these windows slashes get escaped with another backslash, and the resulting string is used as key into rel link db
-    my $ignore_line_re = "\n<(FileSize|lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|ex.File.FileAccessDate|ex.File.FileInodeChangeDate|ex.File.FileCreateDate|total_numbytes|ex.Composite.LightValue)>([^\n])*";
-    $model_text =~ s/$ignore_line_re//g;
-    $test_text =~ s/$ignore_line_re//g;
-
-    # tmp dirs have subdirs with random numbers in name, remove subdir
-	# these tmpdirs are located inside the collection directory
-    $model_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
-    $test_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
-
-
-#print STDERR "@@@@ DEBUGGING: $debugging\n";
-#print STDERR "******** full_modeldb: $full_modeldb\n$model_text\n\n";
-#print STDERR "******** full_testdb: $full_testdb\n$test_text\n\n";
-	
-	# if the OS doesn't match and one of them is windows, extra work needs to be done to bring the db files
-	# in test and model collection to an even base for comparison
-	
-	my $testIsWin = ($test_os ne "compute") ? ($test_os eq "windows") : &isDBWindowsSensitive($dbname, $test_text);
-	my $modelIsWin = ($model_os ne "compute") ? ($model_os eq "windows") : &isDBWindowsSensitive($dbname, $model_text);
-	
-	if($testIsWin == $modelIsWin) {		
-	# both linux or both windows, do the basic test we did on linux machines:
-		    # ignore absolute path prefixes in modelcol and testcol (necessary for archiveinf-doc and -src.gdb files)
-
-			# Remember the original model col on SVN could have been built anywhere, 
-			# and in the gdb files, absolute paths are stored to the collection location. 
-			# Crop these paths to the collect/<colname> point.
-			
-			# Entries are of the form [Entry] or <Entry>. In order to do a sensible diff,
-			# need to remove the prefix to the collect/colname folder in any (absolute) path that occurs in Entry
-			# E.g. [/full/path/collect/colname/import/file.ext] should become [collect/colname/import/file.ext]
-			# Better regex is of the form /BEGIN((?:(?!BEGIN).)*)END/, see http://docstore.mik.ua/orelly/perl/cookbook/ch06_16.htm
-
-			$model_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
-			$test_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
-			#$model_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;			
-			#$test_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;						
-	}
-	
-	else { # one of the collections was built on windows
-		# handling slashes and other differences between a model coll built on one OS (e.g. linux) 
-		# and a test collection built and diffed on another OS (windows)
-	
-		my ($win_text, $lin_text); # references
-		my $collection_path = $strTestCol; # full path to a windows collection
-		
-		if($testIsWin) {
-			$collection_path = $strTestCol; # test collection path is windows 
-			$win_text = \$test_text;
-			$lin_text = \$model_text;
-		} else {
-			$collection_path = $strModelCol; # model collection path is windows
-			$win_text = \$model_text;
-			$lin_text = \$test_text;
-		}
-		
-		if($dbname =~ m/archiveinf-doc/) {
-			
-			(my $collection_path_re = $collection_path) =~ s@\\@\\\\@g;
-			
-			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
-	
-			# convert short filenames to long perl: 
-			# http://www.mombu.com/programming/perl/t-convert-dos-83-filenames-to-win32-long-filenames-using-perl-525448.html
-			for my $line (split /^/, $$win_text) { # split the string into newlines
-				
-				# assoc-file and meta-file contain filepaths, ensure these are long windows file paths now (will later convert to linux slashes)	
-				if($line =~ m@^<(assoc-file|meta-file|src-file)>(.*)(\s+)@s) {
-					my ($field, $value, $suffix) = ($1, $2, $3);
-					$line = $value; # it may be a short file name	
-					
-					if($line !~ m/^\@/) { # if the path doesn't use a "relative" @GSPATH@ placeholder string, but is an absolute path instead						
-						# perhaps test here if it is a shortfilename? should match /CAPS....~number(.ext)/						
-						$line = "<$field>".&Win32::GetLongPathName($line)."$suffix"; # make it a long file name and prefix assoc-file/meta-file tagname to it again					
-					} 
-					else { # if $line contains @THISCOLLECTIONPATH@, still need to deal with DOS filenames suffixes: 
-						# replace placeholder with absolute path and expand to long filename, then insert placeholder in its original place again						
-						$line =~ s/\@THISCOLLECTPATH\@/$collection_path/;
-						$line = &Win32::GetLongPathName($line);
-						$line =~ s/^$collection_path_re/\@THISCOLLECTPATH\@/;
-						$line = "<$field>".$line."$suffix";
-					}
-				}
-				$tmp .= $line;
-			}
-			$$win_text = $tmp;			
-		}
-		
-		
-		# index gdb file
-		if($dbname =~ m/$strColName/) {
-			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
-			for my $line (split /^/, $$win_text) { # split the string into newlines			
-
-				# In the following regex, add any .gdb fieldnames that represent a path and so would contain double backslashes 
-				# on Windows (to escape the single backlash of win filepaths). They will be turned into single-backslashes here, 
-				# and converted into single forward slashes futher below when the txt version of the win gdb file is normalised 
-				# to compare it with the linux version. 
-				# E.g. On windows, the Word-PDF collection(s) contains double backslashes in the ex.File.Directory field
-				# the MARC-Exploded collection contains double backslashes in the null_file entry field of the .gdb file				
-				if($line =~ m@^<(ex.File.Directory|null_file)>(.*)@s) {
-					my ($fieldname, $escaped_path) = ($1, $2);
-					$escaped_path =~ s@\\\\@/@g; #(my $escaped_path = $2) =~ s@\\\\@\\@g;
-					$line = "<$fieldname>$escaped_path";
-				} 
-				elsif($line =~ m@^<Title>(.*)@s) {
-#					print STDERR "***** TITLE: |$1|\n";
-				
-					# word-pdf collection: Title of ps files contain new lines at end when
-					# GreenstoneXMLPlugin::xml_end_tag() writes the Title back out after utf8 decode
-					# if($metadata_name eq "Title") { $metadata_value =~ s/[\n\r]*$//; }
-				
-					(my $title = $1) =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
-					$line = "<Title>$title\n"; # add single newline					
-				}
-				$tmp .= $line;
-			}
-			$$win_text = $tmp;
-		
-			# slashes in windows metadata text need to be turned into linux style slashes. 
-			# index\col.gdb uses double backslashes, and single for \n,\t
-			#$$win_text =~ s@\\\\@/@g;
-		}
-		else { # archiveinf gdb file
-		
-			# slashes in windows metadata text need to be turned into linux style slashes. 
-			# In the two archivesinf gdb files, filepaths may use single backslashes
-			$$win_text =~ s@\\@/@g; #$$win_text =~ s@\\([^n|r|\|"])@/$1@g; # filepath something\rtf remains something\rtf
-		}
-		
-		# cut down absolute paths to files to just collect/colname/.../file, same as before
-		$$lin_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg; # $$lin_text =~ s@^([^\\\/]*(//)?).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
-		$$win_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;		
-		
-		# for the windows text, need to further get rid of the driveletter after [ or <meta>
-		$$win_text =~ s@^(\[|<[^>]*>)[a-zA-Z]:collect@$1collect@mg;		
-		
-	} # end of equalising differences between a windows collection's db file and linux coll's db file
-	
-	
-	# Windows or linux: if index is a flat db file, then ensure the docIDs listed in <contains> field of
-	# both test and model flat db file are alphabetically sorted. So to the numbers in <mdoffset> field.
-	# Despite PERL_PERTURB_KEYS envvar being set to 0 on both machine when generating model collections
-	# and when test collections were generated on test machine, still collections like Images-GPS and some
-	# other colls list items in <contains> and <mdoffset> in different orders. So reordering alphabetically.	
-	#if($dbname =~ m/$strColName/) {	
-		# regex modifiers mge: multi-line, global (replace as many as match), e allows function call in substitution
-		##$model_text =~ s@^<contains>(.*)@sort_contains_field($1, "MODEL", $debugging)@mge;
-		##$test_text =~ s@^<contains>(.*)@sort_contains_field($1, "TEST", $debugging)@mge;
-	#	$model_text =~ s@^<(contains|mdoffset)>(.*)@sort_field_value($1, $2, "MODEL", $debugging)@mge;
-	#	$test_text =~ s@^<(contains|mdoffset)>(.*)@sort_field_value($1, $2, "TEST", $debugging)@mge;		
-	#}
-	
-	# The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
-	# These tmpdirs are located inside the toplevel *greenstone* directory
-	(my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;		
-	$gsdlhome_re = ".*" unless $$ENV{'GSDLHOME'};
-	my $tmpfile_regex = "<URL>http://$gsdlhome_re/tmp/([^\.]*?)(\..{3,4})"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long			
-	if($test_text =~ m@$tmpfile_regex@g) {			
-		# found a match, replace the tmp file name with "random", keeping the original file extension 
-		# in <OrigSource|URL|UTF8URL|gsdlconvertedfilename>
-	
-		# This code is slightly different from doc.xml because each document has its own doc.xml, so this needs to be done
-		# only once for doc.xml, but multiple times in index/col.gdb since it contains the random filenames of all docs in the col	
-		#my ($old_tmp_filename, $ext) = ($1, $2);
-		
-		my $new_tmp_filename = "random";			
-	
-		
-		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)($gsdlhome_re)?(/tmp/)?.*?(\..{3,4})";
-		if($5) { 
-			$test_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
-		} else { # OrigSource contains only the filename
-			$test_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
-		}
-		
-		# modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename			
-		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)(.*)?(/tmp/)?.*?(\..{3,4})";
-		if($5) { 
-			$model_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
-		} else { # OrigSource contains only the filename
-			$model_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
-		}
-		
-		# index/col.gdb also has entries for the random tmp file names in the form: [http://research/ak19/GS2bin_5July2013/tmp/F639.html]
-		# need to equalise these also. Sadly, when there are multiple intermediate files, their random tmp filenames are not 
-		# guaranteed to be generated in the same (alphabetical/numerical) order between model and test collection, so the 
-		# HASH OIDs, although all of them accounted for, appear in a different order. So we have to remove the Hash OIDs.
-		#$test_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg; # HASH OIDs can appear in different order
-		#$model_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg;
-		
-		$test_text  =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
-		$model_text =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
-		
-		# need to re- sort the keys, now that the absolute paths to tmp locations has been removed
-		# so that we get the tmp files in the same order in both model and test collections
-		
-		$model_text = text_to_db_to_text($model_text, "model.gdb");
-		$test_text = text_to_db_to_text($test_text, "test.gdb");
-	}	
-
-    # now can go back to using $model_text and $test_text
-
-    if($debugging) {
-	print_string_to_file($test_text, $savepath.$dbname."_test.out");	
-	print_string_to_file($model_text, $savepath.$dbname."_model.out");
-    }
-
-    my $report_type = "OldStyle"; # Can not change this type.
-    my $diff_gdb = diff \$model_text, \$test_text, { STYLE => $report_type };
-    
-    # leaving the ignore regex as it used to be in the following, in case it helps with single line comparisons
-    $diff_gdb = &diffutil::GenerateOutput($diff_gdb,"^<(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ImageSize)>.*");
-
-    if($diff_gdb eq "")
-    {
-	return "";
-    }
-    else
-    {
-	return "Difference Report: Differences found in the Database file: \n$diff_gdb";
-    }
-    # Call diff?
-}
-
-# Unused, but may come in handy when debugging again: regex substitution helper function
-sub sort_field_value {
-	my($fieldname, $fieldvalue, $displayStr, $debugging) = @_;
-	
-	print STDERR "\n$displayStr BEFORE sort: <$fieldname>$fieldvalue\n" if($debugging);
-	
-	$fieldvalue =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
-	my @values_list = split(';', $fieldvalue);
-	@values_list = sort @values_list;
-	$fieldvalue = "<$fieldname>".join(';', @values_list). "\n";
-	
-	print STDERR "$displayStr AFTER  sort: $fieldvalue\n" if($debugging);
-	
-	return $fieldvalue;
-}
-
-
-# returns true if the contents are windows AND it matters for the diffing on the db that it's windows
-# For col.gdb it does not seem to matter so far, if it is generated on a windows machine and to be compared to a linux-generated col.gdb
-sub isDBWindowsSensitive 
-{
-	my ($dbtailname, $db_contents) = @_; # db filename without suffix
-	
-#	return ($db_contents =~ m/\\/) ? 1 : 0; # windows slashes detected. Better test would be: [Something\something] OR <tag>something\something	
-	
-	if($dbtailname =~ m/^archiveinf-doc/) {
-		return ($db_contents =~ m@<src-file>[a-zA-Z]:\\@) ? 1 : 0; # <src-file>C:\path
-	} 
-	elsif($dbtailname =~ m/^archiveinf-src/) { # <src-file>C:\path
-		return ($db_contents =~ m@\[[a-zA-Z]:\\@) ? 1 : 0; # [C:\path]
-	} 
-	else { # index/col.gdb file
-		if ($db_contents =~ m@<URL>http://[a-zA-Z]:/@) { # <URL>http://C:/path
-			return 1;
-		}		
-		elsif ($db_contents =~ m@^(<URL>http://[a-zA-Z]:/)|(<null_file>[^\\]*\\)@m) { # <URL>http://C:/path OR <null_file>CMSwp-all.00000001\\00000035.nul			
-			return 1;
-		}
-		elsif ($db_contents =~ m@^(<ex.File.Directory>[a-zA-Z]:\\\\)@m) { # <ex.File.Directory>C:\\path\\path for OAI collection
-			return 1;
-		}
-		return 0;		
-	}	
-}
-
-1;
