Index: /other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl2
===================================================================
--- /other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl2	(revision 30652)
+++ /other-projects/nightly-tasks/diffcol/trunk/diffcol/diffcol.pl2	(revision 30652)
@@ -0,0 +1,1057 @@
+#!/usr/bin/perl -w
+
+#TODO: Individual Testing
+
+###########################################################################
+#
+# test.pl -- for testing is built collection is consistent with model collection
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package diffcol_mk2;
+
+BEGIN {
+	die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+	die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+	unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+	unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use parsargv;
+use util;
+use FileUtils;
+use logdiff;
+use cfgdiff;
+use gdbdiff;
+use diffutil;
+use Text::Diff;
+use Cwd;
+
+#--Global Variables Declaration-----------
+my ($gv_test_os, $gv_model_os); # still just file globals
+
+if($ENV{'GSDL3SRCHOME'}) {
+    $gv_strModelColRoot = &FileUtils::filenameConcatenate($ENV{'GSDL3SRCHOME'},"web","sites","localsite","/model-collect");
+    $gv_strTestColRoot = &FileUtils::filenameConcatenate($ENV{'GSDL3SRCHOME'},"web","sites","localsite","/collect");
+} else {
+    $gv_strModelColRoot = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"/model-collect");
+    $gv_strTestColRoot = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"/collect");
+}
+
+$gv_blnErrorStop = "false";
+$gv_blnErrorShow = "false";
+$gv_intVerbosity = 0;
+$gv_strMode = "Full";
+$strOutputFormat = "xml" unless defined $strOutputFormat; # global var with default
+
+%gv_IndivList = ("archives" => 0,
+		"etc" => 0,
+		"images" => 0,
+		"building" => 0,
+		"import" => 0,
+		"index" => 0,
+		"log" => 0,
+		"metadata" => 0,
+		"perllib" => 0,
+		"temp" => 0);
+#----##
+
+#--System Setup---------------------------
+sub SetMode
+{
+	my ($strModeList) = @_;
+	$strModeList =~ s/\|/ /g;
+	my @Modes = split(" ",$strModeList);
+
+	my $blnIndividual = "true";
+	my $blnInitial = "false";
+	my $blnFull = "false";
+
+
+	foreach $strEachMode (@Modes)
+	{
+		if($strEachMode eq "all")
+		{
+			$blnFull = "true";
+			$blnIndividual = "false";
+		}
+		elsif($strEachMode eq "init")
+		{
+			$blnInitial = "true";
+			$blnIndividual = "false";
+		}
+		else
+		{
+			if(defined $gv_IndivList{$strEachMode})
+			{
+				$gv_IndivList{$strEachMode} = 1;
+			}
+			else
+			{
+				die Help("Error: used undefined mode");
+			}
+		}
+	}
+
+	if($blnFull eq "true") {return "Full";}
+	elsif($blnInitial eq "true") {return "Initial";}
+	elsif($blnIndividual eq "true") {return "Individual";}
+	else {die "Error occured in function SetMode!!\n";}
+}
+#----##
+
+#--System Process-------------------------
+sub IndivTest
+{
+	my ($strModelCol,$strTestCol,$strColName) = @_;
+	my @Errors = ();
+	my $intNumberOfErrors = 0;
+	foreach $strEachFolder (keys %gv_IndivList)
+	{
+		if($gv_IndivList{$strEachFolder} == 1)
+		{
+			
+			VobPrint("Start Comparing \"$strEachFolder\"\n",0);
+			my $strModelFolder = &FileUtils::filenameConcatenate($strModelCol,$strEachFolder);
+			my $strTestFolder = &FileUtils::filenameConcatenate($strTestCol,$strEachFolder);
+			$intNumberOfErrors = scalar(@Errors);
+			push(@Errors,TestEach($strModelFolder,$strTestFolder,0,$strColName));
+			$intNumberOfErrors = scalar(@Errors) - $intNumberOfErrors;
+			VobPrint("End Comparing \"$strEachFolder\"\n",0);
+			VobPrint("Difference Found: $intNumberOfErrors\n",0);
+			VobPrint ("\n",0);
+		}
+	}
+	return @Errors;
+}
+
+sub InitTest
+{
+
+	my ($strModelCol,$strTestCol,$strColName) = @_;
+	my $intLevel = 1;
+	my @Errors;
+	
+	# Testing Log files
+#	my $strModelLog = &FileUtils::filenameConcatenate($strModelCol,"log");
+#	my $strTestLog = &FileUtils::filenameConcatenate($strTestCol,"log");
+#	
+#	if(-e $strModelLog && -e $strTestLog)
+#	{
+#	my $strLogError = logdiff::test_log($strModelLog,$strTestLog);
+#	if($strLogError ne "")
+#	{
+#		AlignPrint("Log Folder Comparison Result","Failed",$intLevel);
+#		VobPrint ("$strLogError\n",$intLevel);
+#
+#		$strLogError = "$strLogError"; 
+#		$strLogError = "Difference Found at Log Folder Testing\n".$strLogError."\n";
+#		push(@Errors,$strLogError);
+#	}
+#	else
+#	{
+#		AlignPrint("Log Folder Comparison Result","Succeed",$intLevel);
+#	}
+#	}
+#	else 
+#	{ 
+#	my $strErrorColName;
+#	my $strLogError;
+#	
+#	if(!(-e $strModelLog)){ $strErrorColName = $strErrorColName."(Model Collection)";}
+#	if(!(-e $strTestLog)){ $strErrorColName = $strErrorColName."(Test Collection)";}
+#
+#	AlignPrint("Log Folder Comparison Result","Failed",$intLevel);
+#	$strLogError = "Difference Report: No Log Folder found in $strErrorColName"; 
+#	VobPrint ("$strLogError\n",$intLevel);
+#	$strLogError = "Difference Found at Log Folder Testing (Log folders are only created using GLI)\n".$strLogError."\n";
+#
+#	push(@Errors,$strLogError);
+#	}
+#	VobPrint ("\n",$intLevel);
+	
+	# Testing the build.cfg
+	my $strModelBcfg =  &FileUtils::filenameConcatenate($strModelCol,"index","build.cfg");
+	my $strTestBcfg =  &FileUtils::filenameConcatenate($strTestCol,"index","build.cfg");
+
+	if(-e $strModelBcfg && -e $strTestBcfg)
+	{
+		my $strBcfgError = cfgdiff::test_cfg($strModelBcfg,$strTestBcfg,"build.cfg");
+		if($strBcfgError ne "")
+		{
+			if( $strOutputFormat eq "xml" ) {
+				print "<build-cfg succeeded=\"no\">\n<message>";
+			} else {
+				AlignPrint("Config File(build.cfg) Comparison Result","Failed",$intLevel);
+			}
+
+			VobPrint ("$strBcfgError",$intLevel);
+			if( $strOutputFormat eq "xml" ) {
+				print "</message></build-cfg>";
+			}
+
+			$strBcfgError = "$strBcfgError"; 
+			$strBcfgError = "Difference Found at Config File(build.cfg) Comparison\n".$strBcfgError."\n";
+			push(@Errors,$strBcfgError);
+		}
+		else
+		{
+			if( $strOutputFormat eq "xml" ) {
+				print "<build-cfg succeeded=\"yes\"/>";
+			} else {
+				AlignPrint("Config File(build.cfg) Comparison Result","Succeed",$intLevel);
+			}
+		}
+	}
+	else 
+	{
+		my $strErrorColName;
+		my $strBcfgError;
+		
+		if(!(-e $strModelBcfg)){ $strErrorColName = $strErrorColName."(Model Collection)";}
+		if(!(-e $strTestBcfg)){ $strErrorColName = $strErrorColName."(Test Collection)";}
+
+		AlignPrint("Config File(build.cfg) Comparison Result","Failed",$intLevel);
+		$strBcfgError = "Difference Report: No Config files found in $strErrorColName";
+		VobPrint ("$strBcfgError\n",$intLevel);
+		$strBcfgError = "Difference Found at Config File(build.cfg) Comparison\n".$strBcfgError."\n";
+
+		push(@Errors,$strBcfgError);
+	}
+	VobPrint ("\n",$intLevel);
+
+	# Testing the collect.cfg
+
+	my $strModelCcfg =  &FileUtils::filenameConcatenate($strModelCol,"etc","collect.cfg");
+	my $strTestCcfg =  &FileUtils::filenameConcatenate($strTestCol,"etc","collect.cfg");
+	
+	if(-e $strModelCcfg && -e $strTestCcfg)
+	{
+		my $strCcfgError = cfgdiff::test_cfg($strModelCcfg,$strTestCcfg,"collect.cfg");
+		if($strCcfgError ne "")
+		{
+			if( $strOutputFormat eq "xml" ) {
+				print "<collect-cfg succeeded=\"no\"><message>";
+			} else {
+				AlignPrint("Config File(collect.cfg) Comparison Result","Failed",$intLevel);
+			}
+
+			VobPrint ("$strCcfgError",$intLevel);
+
+			if( $strOutputFormat eq "xml" ) {
+				print "</message></collect-cfg>";
+			}
+
+			$strCcfgError = "$strCcfgError"; 
+			$strCcfgError = "Difference Found at Config File(collect.cfg) Comparison\n".$strCcfgError."\n";
+			push(@Errors,$strCcfgError);
+		}
+		else
+		{
+			if( $strOutputFormat eq "xml" ) {
+				print "<collect-cfg succeeded=\"yes\"/>";
+			} else {
+				AlignPrint("Config File(collect.cfg) Comparison Result","Succeed",$intLevel);
+			}
+		}
+	}
+	else 
+	{
+		my $strErrorColName;
+		my $strCcfgError;
+
+		if(!(-e $strModelCcfg)){ $strErrorColName = $strErrorColName."(Model Collection)";}
+		if(!(-e $strTestCcfg)){ $strErrorColName = $strErrorColName."(Test Collection)";}
+
+		AlignPrint("Config File(collect.cfg) Comparison Result","Failed",$intLevel);
+		$strCcfgError = "Difference Report: No Config files found in $strErrorColName";
+		VobPrint ("$strCcfgError\n",$intLevel);
+		$strCcfgError = "Difference Found at Config File(collect.cfg) Comparison\n".$strCcfgError."\n";
+	
+		push(@Errors,$strCcfgError);
+	}
+
+	VobPrint ("\n",$intLevel);
+	
+	# Testing databases
+
+	# index
+	my $dbext = ".gdb"; # assume we're working with gdbm
+	my $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"index","text","$strColName");
+	if(-f "$strModelGdb$dbext") {
+	    $strModelGdb .= $dbext;
+	} else { # gdbm file does not exist, try for jdbm file
+	    $gdbext = ".jdb";
+	    $strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"index","text","$strColName".$dbext);
+	}
+	my $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"index","text","$strColName".$dbext);	
+	my $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol); # returns 0 if no error
+	if($strGdbError) {
+	    push(@Errors,$strGdbError);
+	}
+	
+	# archives
+	$strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-doc".$dbext);
+	$strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-doc".$dbext);	
+	$strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
+	if($strGdbError) {
+	    push(@Errors,$strGdbError);
+	}
+	
+	$strModelGdb =  &FileUtils::filenameConcatenate($strModelCol,"archives","archiveinf-src".$dbext);
+        $strTestGdb =  &FileUtils::filenameConcatenate($strTestCol,"archives","archiveinf-src".$dbext);
+        $strGdbError = &GdbDiff($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol);
+        if($strGdbError) {
+            push(@Errors,$strGdbError);
+        }
+
+	VobPrint ("\n",$intLevel);
+
+	return @Errors;
+}
+
+
+# At present handles gdbm - need to expand to allow for jdbm and other db types
+sub GdbDiff
+{
+    my ($strModelGdb,$strTestGdb,$strOutputFormat,$intLevel,$strColName,$strTestCol,$strModelCol) = @_;
+
+    my $strGdbError = 0;
+
+    if(-e $strModelGdb && -e $strTestGdb)
+    {
+	#my $strGdbError = gdbdiff::test_gdb($strModelGdb, $strTestGdb);
+	$strGdbError = gdbdiff::test_gdb($strModelGdb, $strTestGdb, $strColName,$gv_test_os, $gv_model_os,$strTestCol,$strModelCol);
+	if($strGdbError ne "")
+	{
+	    if( $strOutputFormat eq "xml" ) {
+		print "<database succeeded=\"no\" location=\"$strModelGdb\"><message>";
+	    } else {
+		AlignPrint("Database Comparsion Result","Failed",$intLevel);
+	    }
+	    VobPrint ("$strGdbError\n",$intLevel);
+	    
+	    if( $strOutputFormat eq "xml" ) {
+		print "</message></database>";
+	    }
+	    
+	    $strGdbError = "$strGdbError";
+	    $strGdbError = "Difference Found at Database Comparsion\n".$strGdbError."\n";
+	    #push(@Errors,$strGdbError);
+	}
+	else
+	{
+	    if( $strOutputFormat eq "xml" ) {
+		print "<database succeeded=\"yes\" location=\"$strModelGdb\"/>";
+	    } else {
+		AlignPrint("Database Comparsion Result","Succeed",$intLevel);
+	    }
+	}
+    }
+    else
+    {
+	my $strErrorColName;
+	
+	if(!(-e $strModelGdb)){ $strErrorColName = $strErrorColName."(Model Collection)";}
+	if(!(-e $strTestGdb)){ $strErrorColName = $strErrorColName."(Test Collection)";}
+	
+	AlignPrint("Database Comparsion Result","Failed",$intLevel);
+	
+	$strGdbError = "Difference Report: No Database files found in $strErrorColName";
+	VobPrint ("$strGdbError\n",$intLevel);
+	
+	$strGdbError = "Difference Found at Database Comparison\n".$strGdbError."\n";
+
+    }
+    
+    return $strGdbError;
+}
+
+sub FullTest
+{
+	my ($strModelCol,$strTestCol,$strColName) = @_;
+	my @Errors = ();
+	my $intLevel = 0;
+	my $intNumberDiffs = 0;
+	
+	# <Initial Test>
+	if( $strOutputFormat eq "xml" ) {
+		#print "<initial-test>";
+	} else {
+		VobPrint("Initial Testing Start\n",$intLevel);
+	}
+
+	@Errors = InitTest($strModelCol,$strTestCol,$strColName);
+	$intNumberDiffs = scalar(@Errors);
+
+	if( $strOutputFormat eq "xml" ) {
+		#print "</initial-test>";
+	} else {
+		VobPrint("Initial Testing End\n",$intLevel);
+		VobPrint("Difference Found in Initial Testing: $intNumberDiffs\n",$intLevel);
+		VobPrint("\n",$intLevel);
+	}
+	# </Initial Test>
+
+	# <Detailed Test>
+	if( $strOutputFormat eq "xml" ) {
+		#print "<detailed-test>";
+	} else {
+		VobPrint("Detail Testing Start\n",$intLevel);
+	}
+	push(@Errors,TestEach($strModelCol,$strTestCol,$intLevel,$strColName));
+	$intNumberDiffs = scalar(@Errors) - $intNumberDiffs;
+
+	if( $strOutputFormat eq "xml" ) {
+		#print "</detailed-test>";
+	} else {
+		VobPrint("Detail Testing End\n",$intLevel);
+		VobPrint("Difference Found in Detail Testing: $intNumberDiffs\n",$intLevel);
+	}
+	# </Detailed Test>
+
+	return @Errors;
+}
+#----##
+
+
+#--Other System Utilities
+sub PrintUsage
+{
+	my ($strProgName) = @_;
+	if ( $strOutputFormat eq "xml" ) {
+		print "<error>usage incorrect</error>\n";
+	} else {
+		print STDERR "Usage: $strProgName test-col [more-col] [-verbosity d] [-mode modes] [-eshow] [-estop]\n";
+	}
+	Help("Error: used incorrect parameters");
+}
+
+sub Help
+{
+	my ($strError) = @_;
+	my $aryptHelps =
+	[ { 'name' => "verbosity",
+		'type' => "scale",
+		'argu' => "a integer" ,
+		'descrip' => "this parameter setup the verbosity of the testing result"},
+	  { 'name' => "mode",
+		'type' => "option",
+		'argu' => "mode type \"[all|init|archives|building|etc|images|import|index|perllib|tmp]\" default to \"all\"" ,
+		'descrip' => "setup testing mode: all-full testing, init-initial testing (include configuration file test,database testing and log testing), others-for individual folder testing"},
+	  { 'name' => "estop",
+		'type' => "flag",
+		'argu' => "NULL" ,
+		'descrip' => "Set then system will stop once it meet an error"},
+	  { 'name' => "eshow",
+		'type' => "flag",
+		'argu' => "NULL" ,
+		'descrip' => "Set then system will show the error summary"}
+	  ];
+	
+
+	if ( $strOutputFormat ne "xml" ) {
+		print "$strError\n";
+
+		foreach my $hashOneArg (@{$aryptHelps})
+		{
+			print "\n----------------------------\n";
+			print "Parameters: -".$hashOneArg->{"name"}."\n";
+			print "Type: ".$hashOneArg->{"type"}."\n";
+			print "Supply Argument: ".$hashOneArg->{"argu"}."\n";
+			print "Description: ".$hashOneArg->{"descrip"}."\n";
+			print "----------------------------\n";
+		}
+	}
+}
+
+sub OutputStart
+{
+	my ($strColName) = @_;
+	my $intPadding = 17 - length($strColName);
+
+	if ( $strOutputFormat eq "xml" ) {
+		print "<diffcol>\n";
+	} else {
+		print "+---------------------------------------------------------+\n";
+		print "|                                                         |\n";
+		print "|              Start Testing Collection: $strColName"," " x $intPadding,"|\n";
+		print "|                                                         |\n";
+		print "+---------------------------------------------------------+\n\n";
+	}
+}
+
+sub OutputEnd
+{
+	my ($strColName,$aryptErrors) = @_;
+	my $intPadding = 12 - length($strColName);
+	if ( $strOutputFormat eq "xml" ) {
+		print "</diffcol>\n";
+	} else {
+		print "\n";
+		print "+---------------------------------------------------------+\n";
+		print "|                                                         |\n";
+		print "|	        Result of Collection Testing: $strColName"," " x $intPadding,"|\n";
+		print "|                                                         |\n";
+		print "+---------------------------------------------------------+\n\n";
+	}
+
+	my $intTotalErrors = scalar(@{$aryptErrors});
+	if ( $strOutputFormat ne "xml" ) {
+		print "Checking completed, there is $intTotalErrors error(s) found.\n";
+	}
+
+	if($gv_blnErrorShow ne "off")
+	{
+		foreach my $strEachError (@{$aryptErrors})
+		{
+			if ( $strOutputFormat eq "xml" ) {
+				print "<error>";
+				print $strEachError;
+				print "</error>\n";
+			} else {
+				print "+---------------------------------------------------------+\n";
+				print "|                        Error                            |\n";
+				print "+---------------------------------------------------------+\n\n";
+				print "$strEachError\n\n";
+			}
+		}
+	}
+	else
+	{
+		if ( $strOutputFormat ne "xml" ) {
+			print "Use -eshow to show the error detail\n\n";
+		}
+	}
+}
+
+sub AlignPrint
+{
+	my ($strMainString,$strStatus,$intLevel) = @_;
+	my $intDot = 100 - length($strMainString) - length($strStatus);
+	VobPrint ($strMainString."."x$intDot."$strStatus\n",$intLevel);
+}
+
+
+# this function is only called on DocXMLFiles. 
+# so far, only doc.xml files need special Windows processing (db files' OS-sensitivity are handled in gdbdiff.pm)
+# Returns true if the doc.xml contains windows style slashes in the gsdlsourcefilename meta field
+sub isDocOrMETSXMLFileWindows
+{
+	my ($file_contents) = @_;
+	
+	#return ($file_contents =~ m/\\/) ? 1 : 0; # windows slashes detected. 
+	
+	# Is this a better test? look for gsdlsourcefilename, see if it contains windows slashes.
+	# what if $gsdlsourcefilename is not guaranteed to exist in all doc.xml files?
+	
+	# for doc.xml:
+	#     <Metadata name="gsdlsourcefilename">import/html_files/cleves.html</Metadata>
+	if($file_contents =~ m@<(.*?:)?Metadata name="gsdlsourcefilename">([^>]*)</(.*?:)?Metadata>@m) {
+		$gsdlsourcefilename = $2;
+		if($gsdlsourcefilename =~ m/\\/) { # windows slashes detected.
+			return 1;
+		}
+	} elsif($file_contents =~ m@<Doc (.*)? file="(.*)?\\doc.xml" ([^>]*)?>@) {	# windows slashes detected in doc.xml in index/text/HASHxxx.dir
+		return 1;
+	}
+	
+	return 0;	
+}
+
+sub TestEach
+{
+	my ($strModel,$strTest,$intLevel,$strColName) = @_;
+	my @Errors = ();
+
+	$intLevel++;
+	if (-d $strModel && -d $strTest)
+	{	
+		my @aryInModel = &diffutil::files_in_dir($strModel);
+		my @aryInTest = &diffutil::files_in_dir($strTest);
+		
+		# Files to be skipped because they get generated on one OS but not the other
+		# On windows, files of the form col.invf.state.\d\d\d\d get generated (e.g. Small-HTML.invf.state.1228) that aren't there on linux
+		my $skipfiles_re = qr/(\.invf\.state\.\d+$)|~$|earliestDatestamp|fail.log$/; # Create a regex of all files to be skipped, see http://perldoc.perl.org/perlop.html
+		@aryInModel = grep { $_ !~ m/$skipfiles_re/ } @aryInModel; # http://stackoverflow.com/questions/174292/what-is-the-best-way-to-delete-a-value-from-an-array-in-perl
+		@aryInTest = grep { $_ !~ m/$skipfiles_re/ } @aryInTest;
+		
+		# Now check all remaining files in the folder exist in both model and test collections
+		my @aryTwoPointers = FolderTesting(\@aryInModel,\@aryInTest,$strModel,$strTest,$intLevel);
+		my @aryCorrectFiles = @{$aryTwoPointers[1]};
+		@Errors = @{$aryTwoPointers[0]};
+		
+		if(scalar(@Errors) == 0)
+		{
+			foreach my $strEachFile (@aryInModel)
+			{
+				my $strNewModel = &FileUtils::filenameConcatenate($strModel,$strEachFile);
+				my $strNewTest = &FileUtils::filenameConcatenate($strTest,$strEachFile);
+				# now additionally ignoring the earliestDatestamp file and the index/idx/*.idh binary file when diffing file 
+				if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/collect\.bak$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png|wmf)$/g)) # wmf = windows meta file #  || $strEachFile =~ m/\~$/g to get rid of ~ files
+				{
+					push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel,$strColName));
+				}
+				else
+				{
+					if ( $strOutputFormat eq "xml" ) {
+						print "<file-comparision location=\"$strEachFile\" blocked=\"yes\" succeeded=\"yes\"/>";
+					} else {
+						VobPrint ("Blocked File Report: Test \"$strEachFile\" by using -mode \"init\"\n",$intLevel);
+					}
+				}
+			} 
+		}
+		else
+		{
+			foreach my $strEachFile (@aryCorrectFiles)
+			{
+				my $strNewModel = &FileUtils::filenameConcatenate($strModel,$strEachFile);
+				my $strNewTest = &FileUtils::filenameConcatenate($strTest,$strEachFile);
+				if(!($strEachFile eq "log" || $strEachFile eq "earliestDatestamp" || $strEachFile =~ m/\.cfg$/g || $strEachFile =~ m/collect\.bak$/g || $strEachFile =~ m/\.((g|j|l|b)db|idh|i.*|wa|td|tsd|ti|t|tl|w|jpe?g|gif|png|wmf)$/g)) #  || $strEachFile =~ m/\~$/g to get rid of ~ files
+				{
+					push(@Errors,TestEach($strNewModel,$strNewTest,$intLevel,$strColName));
+				}
+			}
+		}
+		if($intLevel == $gv_intVerbosity)
+		{
+			if(scalar(@Errors) == 0){ AlignPrint("Contents Comparsion","Succeed",$intLevel);}
+			else {  AlignPrint("Contents Comparsion","Failed",$intLevel);}
+		}
+	}
+	else
+	{
+	    # allow for a namespace prefix to <Metadata> as happens in GreenstoneMETS docmets.xml files, e.g. <gsdl3:Metadata></gsdl3:Metadata>
+	    my $ignore_line_re = "<(.*?:)?Metadata name=\"(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|ImageSize|FileSize|ex.Composite.LightValue)\">.*</(.*?:)?Metadata>\\s*\\n*";
+	    
+	    my $strResult;
+
+	    # for doc.xml and docmets.xml files, need to ignore many date fields. Filter these out before diffing,
+	    # in case these don't appear in the same order between collections, since
+	    # diffutil::GenerateOutput only handles the ignore_regex after a diff has been done
+	    # when they can show up as unnecessary diff 'errors'
+
+	    my ($model_contents, $test_contents);
+
+		# archives/doc.xml files, archives/docmets.xml files and index/text/doc.xml files
+		if($strModel =~ m/doc(mets)?\.xml$/ || ($strModel =~ m@index[\\/]text@ && $strModel =~ m/doc\.xml$/)) {
+
+		open(FIN,"<$strModel") or die "Unable to open $strModel...ERROR: $!\n";
+		sysread(FIN, $model_contents, -s FIN);
+		close(FIN);
+		open(FIN,"<$strTest") or die "Unable to open $strTest...ERROR: $!\n";
+		sysread(FIN, $test_contents, -s FIN);
+		close(FIN);
+
+		$model_contents =~ s/$ignore_line_re//g;
+		$test_contents =~ s/$ignore_line_re//g;
+
+
+		# equalise/normalise the two doc.xml/docmets.xml files for OS differences, if there are any
+		# before comparing a windows test with a linux model or vice-versa
+		my $testIsWin = ($gv_test_os ne "compute") ? ($gv_test_os eq "windows") : &isDocOrMETSXMLFileWindows($test_contents);
+		my $modelIsWin = ($gv_model_os ne "compute") ? ($gv_model_os eq "windows") : &isDocOrMETSXMLFileWindows($model_contents);
+		
+		if($testIsWin != $modelIsWin) { # one of the 2 collections is built on windows, the other on linux, so need to make newlines constant
+		
+			my $win_contents = $testIsWin ? \$test_contents : \$model_contents;
+			my $lin_contents = $testIsWin ? \$model_contents : \$test_contents;
+			
+			# remove all carriage returns \r - introduced into doc.xml by multiread after pdf converted to html
+			$$win_contents =~ s@[\r]@@g;			
+		
+			# make all single windows slashes into single unix slashes
+			# the 1 char look-ahead requires a double pass, otherwise import\3\3.pdf will get replaced with import/3\3.pdf
+			$$win_contents =~ s@([^\\])\\([^\\])@$1\/$2@g;
+			$$win_contents =~ s@([^\\])\\([^\\])@$1\/$2@g;				
+			
+			# make windows \r newlines into constant \n newlines. Already handled when \r got replaced
+			#$$win_contents =~ s@\r\n@\n@mg; # #http://stackoverflow.com/questions/650743/in-perl-how-to-do-you-remove-m-from-a-file
+			
+			#FOR MAC: old macs use CR carriage return (see http://www.perlmonks.org/?node_id=745018), so replace with \n?)
+			# $$win_contents =~ s@\r@\n@mg;
+			
+			if($strModel =~ m/doc\.xml$/) {	# processing particular to doc.xml
+				# remove solitary, stray carriage returns \r in the linux doc.xml, as occurs in the tudor collection owing to the source material
+				# containing solitary carriage returns instead of linefeed
+				$$lin_contents =~ s@[\r]@@g; #$$lin_contents =~ s@[\r][^\n]@@g;
+				
+				
+				# make all single back slash in the linux file into / slash, if when \ was used as a linux escape char in a path
+				# since we've converted *all* single backslashes in the windows doc.xml to / (whether it was meant as a windows path slash or not).
+				# Doing so is okay, since we're not modifying the doc.xml in the model or test collections, just normalising them in-memory for comparison
+				$$lin_contents =~ s@([^\\])\\([^\\])@$1\/$2@g;
+				$$lin_contents =~ s@([^\\])\\([^\\])@$1\/$2@g;
+				
+				# Advanced Beatles collection, 
+				# linux version contains: IMG SRC=_httpextlink_&amp;amp;rl=1&amp;amp;href=http:///\\&quot;http://www.boskowan.com/ (extra / slash)
+				# while windows contains: IMG SRC=_httpextlink_&amp;amp;rl=1&amp;amp;href=http://\\&quot;http://www.boskowan.com/
+				# Normalising to windows version for doing a diff
+				$$lin_contents =~ s@href=http:///@href=http://@g;				
+			}	
+		}
+		
+		# processing particular to doc.xml	
+		if($strModel =~ m/doc\.xml$/) {
+			# tmp dirs have subdirs with random numbers in name, remove randomly named subdir portion of path
+			# these tmpdirs are located inside the collection directory
+			$model_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g; 
+			$test_contents =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
+			
+			# remove all absolute paths upto collect folder from <Metadata /> elements
+			$model_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;
+			$test_contents =~ s@(<Metadata name=\"[^\"]*\">(http:\/\/)?).*(collect[\\\/]$strColName)@$1$3@g;		
+			
+			# The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
+			# These tmpdirs are located inside the toplevel *greenstone* directory
+			(my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;
+			$gsdlhome_re = ".*" unless $$ENV{'GSDLHOME'};
+			my $tmpfile_regex = "<Metadata name=\"URL\">http://$gsdlhome_re/tmp/([^\.]*)(\..{3,4})</Metadata>"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long	
+			
+			if($test_contents =~ m@$tmpfile_regex@) {			
+				# found a match, replace the tmp file name with "random", keeping the original file extension 
+				# in <Metadata name="OrigSource|URL|UTF8URL|gsdlconvertedfilename">
+			
+				my ($old_tmp_filename, $ext) = ($1, $2);			
+				my $new_tmp_filename = "random";			
+				
+				## The following does not work in the Multimedia collection, since there's a subfolder to tmp (the timestamp folder) which contains the output file.
+				#$tmpfile_regex = "(<Metadata name=\"(URL|UTF8URL|gsdlconvertedfilename|OrigSource)\">(http://)?)($gsdlhome_re)?(/tmp/)?$old_tmp_filename($ext</Metadata>)";
+				$tmpfile_regex = "(<Metadata name=\"(URL|UTF8URL|gsdlconvertedfilename|OrigSource)\">(http://)?)($gsdlhome_re)?(/tmp/)?.*?($ext</Metadata>)";
+				if($5) { 
+					$test_contents =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+				} else { # OrigSource contains only the filename
+					$test_contents =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+				}
+				
+				# modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename			
+				$tmpfile_regex = "(<Metadata name=\"(URL|UTF8URL|gsdlconvertedfilename|OrigSource)\">(http://)?)(.*)?(/tmp/)?.*?($ext</Metadata>)";
+				if($5) { 
+					$model_contents =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+				} else { # OrigSource contains only the filename
+					$model_contents =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+				}
+			}
+
+		} # finished special processing of doc.xml files
+		
+		my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory		
+#		&gdbdiff::print_string_to_file($model_contents, $savepath."model_docmets.xml");
+#		&gdbdiff::print_string_to_file($test_contents, $savepath."test_docmets.xml");
+#		if($strModel =~ m/(HASH0164.dir)/) { # list the HASH dirs for which you want the doc.xml file generated, to inspect specific doc.xml files
+#		&gdbdiff::print_string_to_file($model_contents, $savepath."$1_model_doc.xml");
+#		&gdbdiff::print_string_to_file($test_contents, $savepath."$1_test_doc.xml");
+#		}
+		
+
+		
+		# now can diff the normalised versions of the doc.xml/docmets.xml files:
+		$strResult = diff \$model_contents, \$test_contents, { STYLE => "OldStyle" };
+
+	    } else {
+		$strResult = diff $strModel, $strTest, { STYLE => "OldStyle" };
+	    }
+
+	    # The following tries to apply a regex to exclude fields after diffing. This is now no longer necessary since we filter
+	    # these fields out now before the diff, but leaving it in in case different regexes at this point helps with single line diffs
+	    $strResult = &diffutil::GenerateOutput($strResult,"^\\s*<Metadata name=\"(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate)\">.*</Metadata>\\s*\$");
+
+		#$strResult = GeneralOutput($strResult);
+		if ( $strOutputFormat eq "xml" ) {
+			#
+		} else {
+			VobPrint ("Comparing Files:\n\"$strModel\"\n\"$strTest\"\n",$intLevel);
+		}
+		if ($strResult eq "")
+		{
+			if ( $strOutputFormat eq "xml" ) {
+				print "<file-comparison location=\"$strModel\" succeeded=\"yes\"/>\n";
+			} else {
+				AlignPrint("Comparing File","Succeed",$intLevel);
+			}
+		}
+		else
+		{
+#		    print STDERR "**** Diff is: $strResult\n"; # print any differences to the screen
+
+			my $strOutput = "Difference Report:\n$strResult\n";
+			if ( $strOutputFormat eq "xml" ) {
+				print "<file-comparison location=\"$strModel\" succeeded=\"no\"><message>";
+			} else {
+				AlignPrint("Comparing File","Failed",$intLevel);
+			}
+
+			#$result=`file -b $strModel`; # linux specific test for binary file
+			$result = (-B $strModel) ? 1 : 0; # perl test for binary file, see http://perldoc.perl.org/functions/-X.html
+			if ( "$result" =~  "data" ) {
+			         VobPrint( "These binary files differ", $intLevel );
+			} else {
+			         VobPrint ( "$strOutput" , $intLevel);
+			}
+
+
+			if ( $strOutputFormat eq "xml" ) {
+				print "</message></file-comparison>";
+			}
+
+			if($gv_blnErrorStop ne "off") { exit; }	
+			push(@Errors,"File content comparison failed($strModel):\n$strOutput");
+		}
+	}
+
+	return @Errors;
+}
+
+
+sub FolderTesting
+{
+	my ($aryptModel,$aryptTest,$strModelFolder,$strTestFolder,$intLevel) = @_;
+	my %hashCount = ();
+	my @Errors = ();
+	my @CorrectFiles = ();
+	my @TwoPointers = (\@Errors,\@CorrectFiles);
+
+	if ( $strOutputFormat eq "xml" ) {
+		#print "<folder-comparison location=\"$strModelFolder\">\n";		
+	} else {
+		VobPrint ("Comparing Folder contents at \"$strModelFolder\"\n",$intLevel);
+	}
+
+	foreach my $strEachItem (@$aryptModel) {$hashCount{$strEachItem} = 'M'}
+	foreach my $strEachItem (@$aryptTest) 
+	{
+		if(defined $hashCount{$strEachItem} && $hashCount{$strEachItem} eq 'M') {$hashCount{$strEachItem} = 'B';}
+		else {$hashCount{$strEachItem} = 'T';}
+	}
+
+	if( scalar(@$aryptModel)==scalar(@$aryptTest) && scalar(@$aryptModel)==scalar(keys %hashCount) )
+	{
+		if ( $strOutputFormat eq "xml" ) {
+			print "<folder-comparison location=\"$strModelFolder\" succeeded=\"yes\"/>\n";
+		} else {
+			AlignPrint("Folder Comparsion","Succeed",$intLevel);
+		}
+		return @TwoPointers;
+	}
+	else
+	{
+		if ( $strOutputFormat eq "xml" ) {
+			print "<folder-comparison location=\"$strModelFolder\" succeeded=\"no\"><message>\n";
+		} else {
+			AlignPrint("Folder Comparsion","Failed",$intLevel);
+		}
+
+		foreach my $strEachItem (keys %hashCount)
+		{
+			if($hashCount{$strEachItem} ne 'B')
+			{ 
+				my $strOutput = "";
+				my $strReport = "";
+
+				if($hashCount{$strEachItem} eq 'M')
+				{
+					$strOutput = "Difference Found at FolderTesting: \"$strEachItem\" is not found in the Test Collection";
+					$strReport = "Difference Report: difference found at $strTestFolder";
+				}
+				elsif($hashCount{$strEachItem} eq 'T')
+				{
+					$strOutput = "Difference Found at FolderTesting: \"$strEachItem\" is not found in the Model Collection";
+					$strReport = "Difference Report: difference found at $strModelFolder";
+				}
+				else {die "Error occours in diffcol_mk2::TestingFolder\n"}
+
+				VobPrint ("$strOutput\n",$intLevel);
+				$strOutput = $strOutput."\n\t".$strReport."\n";
+				push(@Errors,$strOutput);
+			}
+			else {push(@CorrectFiles,$strEachItem);}
+		}
+		if( $strOutputFormat eq "xml" ) {
+			print "</message></folder-comparison>";
+		}
+
+		return @TwoPointers;
+	} 
+}
+
+sub VobPrint
+{
+	my ($strOutput, $intLevel) = @_;
+	my $strTab = "";
+	my $intTab = int($intLevel/2);
+	if($intLevel <= $gv_intVerbosity)
+	{
+		if($intLevel >= 1) 
+		{
+			$strTab = "\t"x($intTab+1);
+			$strOutput =~ s/\n$//;
+			$strOutput =~ s/\n/\n$strTab/g;
+			#$strTab =~ s/"\n"/"\n"."\t"x($intTab+1)/g;
+		}
+
+		if( $strOutputFormat eq "xml" ) {
+			$strOutput =~ s/&/&amp;/g;
+			$strOutput =~ s/</&lt;/g;
+			$strOutput =~ s/>/&gt;/g;
+		}
+
+      if ( length( $strOutput ) > 1000 ) {
+            $strOutput = substr( $strOutput, 0, 978);
+
+	    # make sure there are no stray ampersands/partial ampersands that need to be completed as &lt; or &gt; or &amp;
+	    if($strOutput =~ m/&(.{1,2})?$/ || $strOutput =~ m/&(am?p?)$/) { # &lt => &lt; or &g => &gt; or &a(m)=> &amp; or &amp => &amp;
+		if(defined $1 && $1) {
+		    my $rest = $1;
+			if($rest =~ m/^a/) {
+				$strOutput =~ s@am?p?$@amp;@;
+		    }
+		    elsif($rest eq "g" || $rest eq "l") {
+			$strOutput .= "t;"; # close the known tag
+		    }
+		    elsif($rest eq "gt" || $rest eq "lt") {
+			$strOutput .= ";";
+		    }			
+		} else { # & on its own
+		    #$strOutput = substr( $strOutput, 0, 977); # lop off the &
+		    $strOutput .= "gt;"; # 50% chance the closing tag is what was missing (else can make this &amp;)
+		    # but even so, when the xslt is applied to report it doesn't break as long as & is not left dangling
+		}
+	    }
+	    $strOutput .= "... (output truncated)";
+      }
+
+
+		print $strTab.$strOutput."\n";
+	}
+}
+#----##
+
+
+#--Main System----------------------------
+#-----------------------------------------
+# Name: main
+# Perameters: arguments from command line
+# Pre-condition: testing will start by calling this main function.
+# Post-condition: output the test results for one or more collections.
+#-----------------------------------------
+sub main
+{
+	my ($intVerbosity,$strErrorStop,$strErrorShow,$strMode,$test_os,$model_os);
+	my $strProgName = $0; 
+	my $intArgc = scalar(@ARGV);
+	
+	#--System Arguments Setup
+	if (!parsargv::parse(\@ARGV,
+		'estop//off', \$strErrorStop, 
+		'eshow//off', \$strErrorShow, 
+		'verbosity/\d+/1', \$intVerbosity, 
+		'mode/[\w\-]+/all', \$strMode,
+		'output/[\w\-]+/text', \$strOutputFormat,
+		'testos/(windows|linux|darwin|compute)/compute', \$test_os, # param-name,regex,default
+		'modelos/(windows|linux|darwin|compute)/compute', \$model_os # actually defaults to linux in task.pl
+		)) {
+		PrintUsage($strProgName);
+		die "\n";
+	}
+
+	if ($intArgc<1) {
+		PrintUsage($strProgName);
+		die "\n";
+	}
+
+	$gv_test_os = $test_os; # if not specified, defaults to "compute"
+	$gv_model_os = $model_os; # tends to be linux
+
+	$gv_blnErrorStop = $strErrorStop;
+	$gv_blnErrorShow = $strErrorShow;
+	$gv_intVerbosity = $intVerbosity;
+	$gv_strMode = SetMode($strMode);
+
+	#----##
+
+	#--Collection(s) Testing
+	foreach $strColName (@ARGV)
+	{
+		my @ErrorsInEachCol;
+		my $strModelCol = &FileUtils::filenameConcatenate($gv_strModelColRoot,$strColName);
+		my $strTestCol = &FileUtils::filenameConcatenate($gv_strTestColRoot,$strColName);
+
+		#--Output(Start)
+		OutputStart($strColName);
+		#----##
+		
+		if(-e $strModelCol && -e $strTestCol )
+		{
+			
+			#--Individual Testing
+			if ($gv_strMode eq "Individual")
+			{
+				@ErrorsInEachCol = IndivTest($strModelCol,$strTestCol,$strColName);
+			}
+			#----##
+			
+			#--Initial Testing
+			elsif	($gv_strMode eq "Initial")
+			{
+				@ErrorsInEachCol = InitTest($strModelCol,$strTestCol,$strColName);
+			}
+			#----##
+			
+			#--Full Testing
+			elsif	($gv_strMode eq "Full")
+			{
+				@ErrorsInEachCol = FullTest($strModelCol,$strTestCol,$strColName);
+			}
+			#----##
+			
+			#--Error Checking
+			else
+			{
+				if ( $strOutputFormat eq "xml" ) {
+					die "<error>Error occoured in main function</error>\n";
+				} else {
+					die "Error occoured in main function.\n";
+				}
+			}
+			#----##
+			
+		}
+		else
+		{
+			if( $strOutputFormat eq "xml" ) {
+				die "<error>Cannot find collection: $strColName</error>\n";
+			} else {
+				die "Error: cannot find collection: $strColName\n";
+			}
+		}
+		#----##
+
+		#--Output(Results and Errors)
+		OutputEnd($strColName,\@ErrorsInEachCol);
+		#----##
+
+	}
+}
+#----##
+
+&main();
Index: /other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm2
===================================================================
--- /other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm2	(revision 30652)
+++ /other-projects/nightly-tasks/diffcol/trunk/diffcol/gdbdiff.pm2	(revision 30652)
@@ -0,0 +1,333 @@
+package gdbdiff;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use util;
+use diffutil;
+use Text::Diff;
+use Cwd;
+
+if ($^O =~ m/mswin/i) {
+    require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
+}
+
+sub readin_gdb
+{
+    my ($cmd) = @_;
+
+    open(PIN,"$cmd|") 
+	|| die "Unable to open pipe to $cmd: $!\n";
+
+    my $text_content = "";
+
+    while (defined (my $line = <PIN>)) {
+	$text_content .= $line;
+    }
+
+    close(PIN);
+    return $text_content;
+}
+
+sub read_db 
+{
+    # need to sort text output of both test and model col database files, to normalise them for the comparison
+    # the -sort option to db2txt was added specifically to support diffcol
+    my($db_file) = @_;
+    my $db_cmd = "db2txt -sort $db_file 2>&1";
+    if($db_file =~ m/\.jdb$/) {
+	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
+	$db_cmd = "$runperl jdb2txt.pl -sort $db_file";
+    }
+
+    my $db_text = readin_gdb($db_cmd);
+    return $db_text;
+}
+
+sub text_to_db_to_text
+{
+    my($db_text, $db_filename) = @_;
+
+    # http://stackoverflow.com/questions/1909262/how-can-i-pipe-input-into-a-java-command-from-perl
+    
+    if($db_filename =~ m/\.jdb$/) {
+	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
+	open PIPE, "| $runperl txt2jdb.pl $db_filename";
+	print PIPE "$db_text";
+	close(PIPE);
+    } else {
+	open PIPE, "| txt2db $db_filename";
+	print PIPE "$db_text";
+	close(PIPE);
+    }
+    return &read_db("$db_filename");
+}
+
+# for debugging. Prints txt contents of db to file
+sub print_string_to_file 
+{
+    my ($text, $outfile) = @_;
+	
+    open(FOUT, ">$outfile") or die "ERROR failed to write to $outfile: $!\n";
+    print FOUT $text;
+    close(FOUT);
+}
+
+sub test_gdb
+{
+    my ($full_modeldb, $full_testdb, $strColName, $test_os, $model_os, $strTestCol, $strModelCol, $debugging) = @_;
+
+   # print "Now is testing database\n";
+	my ($dbname, $dirname, $suffix)= &File::Basename::fileparse($full_testdb, "\\.[^\\.]+\$");
+   
+    my $model_text = read_db($full_modeldb);
+    my $test_text = read_db($full_testdb);
+
+    my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory
+    if($debugging) {
+	print_string_to_file($test_text, $savepath.$dbname."_test.out1");	
+	print_string_to_file($model_text, $savepath.$dbname."_model.out1");
+    }
+
+    # filter out the fields that can be ignored in the two database files
+	# The total_numbytes field can vary depending on how many backslashes exist in the urls in the main body text, as each
+	# of these windows slashes get escaped with another backslash, and the resulting string is used as key into rel link db
+    my $ignore_line_re = "\n<(FileSize|lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|total_numbytes|ex.Composite.LightValue)>([^\n])*";
+    $model_text =~ s/$ignore_line_re//g;
+    $test_text =~ s/$ignore_line_re//g;
+
+    # tmp dirs have subdirs with random numbers in name, remove subdir
+	# these tmpdirs are located inside the collection directory
+    $model_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
+    $test_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
+
+	# if the OS doesn't match and one of them is windows, extra work needs to be done to bring the db files
+	# in test and model collection to an even base for comparison
+	
+	my $testIsWin = ($test_os ne "compute") ? ($test_os eq "windows") : &isDBWindowsSensitive($dbname, $test_text);
+	my $modelIsWin = ($model_os ne "compute") ? ($model_os eq "windows") : &isDBWindowsSensitive($dbname, $model_text);
+	
+	if($testIsWin == $modelIsWin) {		
+	# both linux or both windows, do the basic test we did on linux machines:
+		    # ignore absolute path prefixes in modelcol and testcol (necessary for archiveinf-doc and -src.gdb files)
+
+			# Remember the original model col on SVN could have been built anywhere, 
+			# and in the gdb files, absolute paths are stored to the collection location. 
+			# Crop these paths to the collect/<colname> point.
+			
+			# Entries are of the form [Entry] or <Entry>. In order to do a sensible diff,
+			# need to remove the prefix to the collect/colname folder in any (absolute) path that occurs in Entry
+			# E.g. [/full/path/collect/colname/import/file.ext] should become [collect/colname/import/file.ext]
+			# Better regex is of the form /BEGIN((?:(?!BEGIN).)*)END/, see http://docstore.mik.ua/orelly/perl/cookbook/ch06_16.htm
+
+			$model_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+			$test_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+			#$model_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;			
+			#$test_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;						
+	}
+	
+	else { # one of the collections was built on windows
+		# handling slashes and other differences between a model coll built on one OS (e.g. linux) 
+		# and a test collection built and diffed on another OS (windows)
+	
+		my ($win_text, $lin_text); # references
+		my $collection_path = $strTestCol; # full path to a windows collection
+		
+		if($testIsWin) {
+			$collection_path = $strTestCol; # test collection path is windows 
+			$win_text = \$test_text;
+			$lin_text = \$model_text;
+		} else {
+			$collection_path = $strModelCol; # model collection path is windows
+			$win_text = \$model_text;
+			$lin_text = \$test_text;
+		}
+		
+		if($dbname =~ m/archiveinf-doc/) {
+			
+			(my $collection_path_re = $collection_path) =~ s@\\@\\\\@g;
+			
+			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
+	
+			# convert short filenames to long perl: 
+			# http://www.mombu.com/programming/perl/t-convert-dos-83-filenames-to-win32-long-filenames-using-perl-525448.html
+			for my $line (split /^/, $$win_text) { # split the string into newlines
+				
+				# assoc-file and meta-file contain filepaths, ensure these are long windows file paths now (will later convert to linux slashes)	
+				if($line =~ m@^<(assoc-file|meta-file|src-file)>(.*)(\s+)@s) {
+					my ($field, $value, $suffix) = ($1, $2, $3);
+					$line = $value; # it may be a short file name	
+					
+					if($line !~ m/^\@/) { # if the path doesn't use a "relative" @GSPATH@ placeholder string, but is an absolute path instead						
+						# perhaps test here if it is a shortfilename? should match /CAPS....~number(.ext)/						
+						$line = "<$field>".&Win32::GetLongPathName($line)."$suffix"; # make it a long file name and prefix assoc-file/meta-file tagname to it again					
+					} 
+					else { # if $line contains @THISCOLLECTIONPATH@, still need to deal with DOS filenames suffixes: 
+						# replace placeholder with absolute path and expand to long filename, then insert placeholder in its original place again						
+						$line =~ s/\@THISCOLLECTPATH\@/$collection_path/;
+						$line = &Win32::GetLongPathName($line);
+						$line =~ s/^$collection_path_re/\@THISCOLLECTPATH\@/;
+						$line = "<$field>".$line."$suffix";
+					}
+				}
+				$tmp .= $line;
+			}
+			$$win_text = $tmp;			
+		}
+		
+		
+		# index gdb file
+		if($dbname =~ m/$strColName/) {
+			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
+			for my $line (split /^/, $$win_text) { # split the string into newlines			
+
+				# In the following regex, add any .gdb fieldnames that represent a path and so would contain double backslashes 
+				# on Windows (to escape the single backlash of win filepaths). They will be turned into single-backslashes here, 
+				# and converted into single forward slashes futher below when the txt version of the win gdb file is normalised 
+				# to compare it with the linux version. 
+				# E.g. On windows, the Word-PDF collection(s) contains double backslashes in the ex.File.Directory field
+				# the MARC-Exploded collection contains double backslashes in the null_file entry field of the .gdb file				
+				if($line =~ m@^<(ex.File.Directory|null_file)>(.*)@s) {
+					my ($fieldname, $escaped_path) = ($1, $2);
+					$escaped_path =~ s@\\\\@/@g; #(my $escaped_path = $2) =~ s@\\\\@\\@g;
+					$line = "<$fieldname>$escaped_path";
+				} 
+				elsif($line =~ m@^<Title>(.*)@s) {
+#					print STDERR "***** TITLE: |$1|\n";
+				
+					# word-pdf collection: Title of ps files contain new lines at end when
+					# GreenstoneXMLPlugin::xml_end_tag() writes the Title back out after utf8 decode
+					# if($metadata_name eq "Title") { $metadata_value =~ s/[\n\r]*$//; }
+				
+					(my $title = $1) =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
+					$line = "<Title>$title\n"; # add single newline					
+				}
+				$tmp .= $line;
+			}
+			$$win_text = $tmp;
+		
+			# slashes in windows metadata text need to be turned into linux style slashes. 
+			# index\col.gdb uses double backslashes, and single for \n,\t
+			#$$win_text =~ s@\\\\@/@g;
+		}
+		else { # archiveinf gdb file
+		
+			# slashes in windows metadata text need to be turned into linux style slashes. 
+			# In the two archivesinf gdb files, filepaths may use single backslashes
+			$$win_text =~ s@\\@/@g; #$$win_text =~ s@\\([^n|r|\|"])@/$1@g; # filepath something\rtf remains something\rtf
+		}
+		
+		# cut down absolute paths to files to just collect/colname/.../file, same as before
+		$$lin_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg; # $$lin_text =~ s@^([^\\\/]*(//)?).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
+		$$win_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;		
+		
+		# for the windows text, need to further get rid of the driveletter after [ or <meta>
+		$$win_text =~ s@^(\[|<[^>]*>)[a-zA-Z]:collect@$1collect@mg;		
+		
+	} # end of equalising differences between a windows collection's db file and linux coll's db file
+	
+	# The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
+	# These tmpdirs are located inside the toplevel *greenstone* directory
+	(my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;		
+	$gsdlhome_re = ".*" unless $$ENV{'GSDLHOME'};
+	my $tmpfile_regex = "<URL>http://$gsdlhome_re/tmp/([^\.]*?)(\..{3,4})"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long			
+	if($test_text =~ m@$tmpfile_regex@g) {			
+		# found a match, replace the tmp file name with "random", keeping the original file extension 
+		# in <OrigSource|URL|UTF8URL|gsdlconvertedfilename>
+	
+		# This code is slightly different from doc.xml because each document has its own doc.xml, so this needs to be done
+		# only once for doc.xml, but multiple times in index/col.gdb since it contains the random filenames of all docs in the col	
+		#my ($old_tmp_filename, $ext) = ($1, $2);
+		
+		my $new_tmp_filename = "random";			
+	
+		
+		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)($gsdlhome_re)?(/tmp/)?.*?(\..{3,4})";
+		if($5) { 
+			$test_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+		} else { # OrigSource contains only the filename
+			$test_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+		}
+		
+		# modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename			
+		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)(.*)?(/tmp/)?.*?(\..{3,4})";
+		if($5) { 
+			$model_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
+		} else { # OrigSource contains only the filename
+			$model_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
+		}
+		
+		# index/col.gdb also has entries for the random tmp file names in the form: [http://research/ak19/GS2bin_5July2013/tmp/F639.html]
+		# need to equalise these also. Sadly, when there are multiple intermediate files, their random tmp filenames are not 
+		# guaranteed to be generated in the same (alphabetical/numerical) order between model and test collection, so the 
+		# HASH OIDs, although all of them accounted for, appear in a different order. So we have to remove the Hash OIDs.
+		#$test_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg; # HASH OIDs can appear in different order
+		#$model_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg;
+		
+		$test_text  =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
+		$model_text =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
+		
+		# need to re- sort the keys, now that the absolute paths to tmp locations has been removed
+		# so that we get the tmp files in the same order in both model and test collections
+		
+		$model_text = text_to_db_to_text($model_text, "model.gdb");
+		$test_text = text_to_db_to_text($test_text, "test.gdb");
+	}	
+
+    # now can go back to using $model_text and $test_text
+
+    if($debugging) {
+	print_string_to_file($test_text, $savepath.$dbname."_test.out");	
+	print_string_to_file($model_text, $savepath.$dbname."_model.out");
+    }
+
+    my $report_type = "OldStyle"; # Can not change this type.
+    my $diff_gdb = diff \$model_text, \$test_text, { STYLE => $report_type };
+    
+    # leaving the ignore regex as it used to be in the following, in case it helps with single line comparisons
+    $diff_gdb = &diffutil::GenerateOutput($diff_gdb,"^<(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ImageSize)>.*");
+
+    if($diff_gdb eq "")
+    {
+	return "";
+    }
+    else
+    {
+	return "Difference Report: Differences found in the Database file: \n$diff_gdb";
+    }
+    # Call diff?
+}
+
+# returns true if the contents are windows AND it matters for the diffing on the db that it's windows
+# For col.gdb it does not seem to matter so far, if it is generated on a windows machine and to be compared to a linux-generated col.gdb
+sub isDBWindowsSensitive 
+{
+	my ($dbtailname, $db_contents) = @_; # db filename without suffix
+	
+#	return ($db_contents =~ m/\\/) ? 1 : 0; # windows slashes detected. Better test would be: [Something\something] OR <tag>something\something	
+	
+	if($dbtailname =~ m/^archiveinf-doc/) {
+		return ($db_contents =~ m@<src-file>[a-zA-Z]:\\@) ? 1 : 0; # <src-file>C:\path
+	} 
+	elsif($dbtailname =~ m/^archiveinf-src/) { # <src-file>C:\path
+		return ($db_contents =~ m@\[[a-zA-Z]:\\@) ? 1 : 0; # [C:\path]
+	} 
+	else { # index/col.gdb file
+		if ($db_contents =~ m@<URL>http://[a-zA-Z]:/@) { # <URL>http://C:/path
+			return 1;
+		}		
+		elsif ($db_contents =~ m@^(<URL>http://[a-zA-Z]:/)|(<null_file>[^\\]*\\)@m) { # <URL>http://C:/path OR <null_file>CMSwp-all.00000001\\00000035.nul			
+			return 1;
+		}
+		elsif ($db_contents =~ m@^(<ex.File.Directory>[a-zA-Z]:\\\\)@m) { # <ex.File.Directory>C:\\path\\path for OAI collection
+			return 1;
+		}
+		return 0;		
+	}	
+}
+
+1;
