Index: gs2-extensions/parallel-building/trunk/src/perllib/util.pm
===================================================================
--- gs2-extensions/parallel-building/trunk/src/perllib/util.pm	(revision 24626)
+++ gs2-extensions/parallel-building/trunk/src/perllib/util.pm	(revision 26961)
@@ -46,4 +46,11 @@
     # and are regular files
     foreach my $file (@files) {
+
+      if (&util::isHDFS($file))
+      {
+        &util::executeHDFSCommand('rm', $file);
+      }
+      else
+      {
 	if (!-e $file) {
 	    print STDERR "util::rm $file does not exist\n";
@@ -53,6 +60,7 @@
 	    push (@filefiles, $file);
 	}
-    }
-    
+      }
+    }
+
     # remove the files
     my $numremoved = unlink @filefiles;
@@ -96,4 +104,15 @@
     # recursively remove the files
     foreach my $file (@files_array) {
+
+      # HDFS support
+      if (&util::isHDFS($file))
+      {
+        # HDFS doesn't really lend itself to a choosy delete, unless you want
+        # it to be really, really, unbearably slow.
+        &util::executeHDFSCommand('rmr', $file);
+        next;
+      }
+
+
 	$file =~ s/[\/\\]+$//; # remove trailing slashes
 	
@@ -133,11 +152,10 @@
 
 # recursive removal
-sub rm_r {
-    my (@files) = @_;
-    
-    # use the more general (but reterospectively written function
-    # filtered_rm_r function()
-
-    filtered_rm_r(\@files,undef,undef); # no accept or reject expressions
+sub rm_r
+{
+  my (@files) = @_;
+  # use the more general (but reterospectively written function
+  # filtered_rm_r function()
+  filtered_rm_r(\@files,undef,undef); # no accept or reject expressions
 }
 
@@ -149,4 +167,22 @@
     my $dest = pop (@_);
     my (@srcfiles) = @_;
+
+    # moving a file within or into HDFS
+    if (&util::isHDFS($dest))
+    {
+      foreach my $src (@srcfiles)
+      {
+        if (&util::isHDFS($src))
+        {
+          &util::executeHDFSCommand('mv', $src, $dest);
+        }
+        else
+        {
+          &util::executeHDFSCommand('put', $src, $dest);
+          &util::rm_r($src);
+        }
+      }
+      return;
+    }
 
     # remove trailing slashes from source and destination files
@@ -166,4 +202,13 @@
     # move the files
     foreach my $file (@srcfiles) {
+
+      # moving a file out of HDFS
+      if (&util::isHDFS($file))
+      {
+        &util::executeHDFSCommand('get', $file, $dest);
+        &util::rm_r($file);
+        next;
+      }
+
 	my $tempdest = $dest;
 	if (-d $tempdest) {
@@ -174,44 +219,144 @@
 	    print STDERR "util::mv $file does not exist\n";
 	} else {
-	    rename ($file, $tempdest);
-	}
-    }
+	    if(!rename ($file, $tempdest)) {
+		print STDERR "**** Failed to rename $file to $tempdest\n";
+		&File::Copy::copy($file, $tempdest);
+		&rm($file);
+	    } 
+	    elsif(-e $file) { # rename (partially) succeeded) but srcfile still exists after rename
+		#print STDERR "*** srcfile $file still exists after rename to $tempdest\n";
+		if(!-e $tempdest) {
+		    print STDERR "@@@@ ERROR: $tempdest does not exist\n";
+		}
+		# Sometimes the rename operation fails (as does File::Copy::move).
+		# This turns out to be because the files are hardlinked. 
+		# Need to do a copy-delete in this case, however, the copy step is not necessary: 
+		# the srcfile got renamed into tempdest, but srcfile itself still exists, delete it.
+		#&File::Copy::copy($file, $tempdest);
+
+		&rm($file);	    
+	    }
+	}
+    }
+}
+
+# Move the contents of source directory into target directory
+# (as opposed to merely replacing target dir with the src dir)
+# This can overwrite any files with duplicate names in the target
+# but other files and folders in the target will continue to exist
+sub mv_dir_contents {
+	my ($src_dir, $dest_dir) = @_;
+	
+	# Obtain listing of all files within src_dir
+	# Note that readdir lists relative paths, as well as . and ..
+	opendir(DIR, "$src_dir");
+	my @files= readdir(DIR); 
+	close(DIR);
+	
+	my @full_path_files = ();
+	foreach my $file (@files) {	
+		# process all except . and ..
+		unless($file eq "." || $file eq "..") {
+			
+			my $dest_subdir = &filename_cat($dest_dir, $file); # $file is still a relative path
+		
+			# construct absolute paths
+			$file = &filename_cat($src_dir, $file); # $file is now an absolute path
+			
+			# Recurse on directories which have an equivalent in target dest_dir
+			# If $file is a directory that already exists in target $dest_dir, 
+			# then a simple move operation will fail (definitely on Windows).
+			if(-d $file && -d $dest_subdir) { 
+				#print STDERR "**** $file is a directory also existing in target, its contents to be copied to $dest_subdir\n";
+				&mv_dir_contents($file, $dest_subdir);
+				
+				# now all content is moved across, delete empty dir in source folder
+				if(&is_dir_empty($file)) {
+					if (!rmdir $file) {
+						print STDERR "ERROR. util::mv_dir_contents couldn't remove directory $file\n";
+					} 
+				} else { # error
+					print STDERR "ERROR. util::mv_dir_contents: subfolder $file still non-empty after moving contents to $dest_subdir\n";
+				}
+			} else { # process files and any directories that don't already exist with a simple move
+				push(@full_path_files, $file);
+			}			
+		}
+	}
+	
+	if(!&dir_exists($dest_dir)) { # create target toplevel folder or subfolders if they don't exist
+		&mk_dir($dest_dir);
+	}
+
+	#print STDERR "@@@@@ Copying files |".join(",", @full_path_files)."| to: $dest_dir\n";
+
+	if(@full_path_files) { # if non-empty, there's something to copy across
+		&mv(@full_path_files, $dest_dir);
+	}
 }
 
 
 # copies a file or a group of files
-sub cp {
-    my $dest = pop (@_);
-    my (@srcfiles) = @_;
-
-    # remove trailing slashes from source and destination files
-    $dest =~ s/[\\\/]+$//;
-    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
-
-    # a few sanity checks
-    if (scalar (@srcfiles) == 0) {
-	print STDERR "util::cp no destination directory given\n";
-	return;
-    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
-	print STDERR "util::cp if multiple source files are given the ".
-	    "destination must be a directory\n";
-	return;
-    }
-
-    # copy the files
-    foreach my $file (@srcfiles) {
-	my $tempdest = $dest;
-	if (-d $tempdest) {
-	    my ($filename) = $file =~ /([^\\\/]+)$/;
-	    $tempdest .= "/$filename";
-	}
-	if (!-e $file) {
-	    print STDERR "util::cp $file does not exist\n";
-	} elsif (!-f $file) {
-	    print STDERR "util::cp $file is not a plain file\n";
-	} else {
-	    &File::Copy::copy ($file, $tempdest);
-	}
-    }
+sub cp
+{
+  my $dest = pop (@_);
+  my (@srcfiles) = @_;
+
+  # remove trailing slashes from source and destination files
+  $dest =~ s/[\\\/]+$//;
+  map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+  # a few sanity checks
+  if (scalar (@srcfiles) == 0)
+  {
+    print STDERR "util::cp no destination directory given\n";
+    return 0;
+  }
+  elsif ((scalar (@srcfiles) > 1) && (!&util::dir_exists($dest)))
+  {
+    print STDERR "util::cp if multiple source files are given the destination must be a directory\n";
+    return 0;
+  }
+
+  # copying a file into or within HDFS
+  if (&util::isHDFS($dest))
+  {
+    foreach my $src (@srcfiles)
+    {
+      &util::executeHDFSCommand('put', $src, $dest);
+      &util::rm_r($src);
+    }
+    return;
+  }
+
+  # copy the files
+  foreach my $file (@srcfiles)
+  {
+    my $tempdest = $dest;
+    if (&util::dir_exists($tempdest))
+    {
+      my ($filename) = $file =~ /([^\\\/]+)$/;
+      $tempdest .= "/$filename";
+    }
+    if (!&util::file_exists($file))
+    {
+      if (&util::dir_exists($file))
+      {
+        print STDERR "util::cp $file is not a plain file\n";
+      }
+      else
+      {
+        print STDERR "util::cp $file does not exist\n";
+      }
+    }
+    elsif (&util::isHDFS($file))
+    {
+      &util::executeHDFSCommand('get', $file, $dest);
+    }
+    else
+    {
+      &File::Copy::copy ($file, $tempdest);
+    }
+  }
 }
 
@@ -393,16 +538,36 @@
 }
 
-sub mk_dir {
-    my ($dir) = @_;
-
+# /** @function mk_dir()
+#  *  Extend mkdir to allow it to silently fail in the case where code is
+#  *  'competing' to create a directory first.
+#  *  @param   $dir  the full path of the directory to create
+#  *  @param   $can_fail  1 if the mkdir can fail silently (optional)
+#  *  @return  1 on success, 0 on failure
+#  */
+sub mk_dir
+{
+  my ($dir, $can_fail) = @_;
+  my $mkdir_ok = 0;
+  if (&util::isHDFS($dir))
+  {
+    # unhelpfully HDFS mkdir returns 0 on success, -1 on failure
+    my $result = &util::executeHDFSCommand('mkdir', $dir);
+    if ($result == 0)
+    {
+      $mkdir_ok = 1;
+    }
+  }
+  else
+  {
     my $store_umask = umask(0002);
     my $mkdir_ok = mkdir ($dir, 0777);
     umask($store_umask);
-    
-    if (!$mkdir_ok) 
-    {
-	print STDERR "util::mk_dir could not create directory $dir\n";
-	return;
-    }
+  }
+  # only output an error if this call wasn't marked as can_fail
+  if (!$mkdir_ok && (!defined $can_fail || !$can_fail))
+  {
+    print STDERR "util::mk_dir could not create directory: $dir\n error: $!\n";
+  }
+  return $mkdir_ok;
 }
 
@@ -410,34 +575,53 @@
 # on this subroutine against File::Path::mkpath (). mk_all_dir() is apparently
 # slightly faster (surprisingly) - Stefan.
-sub mk_all_dir {
-    my ($dir) = @_;
-
-    # use / for the directory separator, remove duplicate and
-    # trailing slashes
-    $dir=~s/[\\\/]+/\//g; 
-    $dir=~s/[\\\/]+$//;
-
-    # make sure the cache directory exists
-    my $dirsofar = "";
-    my $first = 1;
-    foreach my $dirname (split ("/", $dir)) {
-	$dirsofar .= "/" unless $first;
-	$first = 0;
-
-	$dirsofar .= $dirname;
-
-	next if $dirname =~ /^(|[a-z]:)$/i;
-	if (!-e $dirsofar)
-	    {
-		my $store_umask = umask(0002);
-		my $mkdir_ok = mkdir ($dirsofar, 0777);
-		umask($store_umask);
-		if (!$mkdir_ok)
-		{
-		    print STDERR "util::mk_all_dir could not create directory $dirsofar\n";
-		    return;
-		}
-	    }
-    }
+sub mk_all_dir
+{
+  my ($dir) = @_;
+
+  ###rint "-> util::mk_all_dir($dir)\n";
+
+  # support for HDFS
+  if (&util::isHDFS($dir))
+  {
+    # HDFS's version of mkdir does it recursively anyway
+    my $result = &util::executeHDFSCommand('mkdir', $dir);
+    return ($result == 0);
+  }
+
+  # use / for the directory separator, remove duplicate and
+  # trailing slashes
+  $dir=~s/[\\\/]+/\//g;
+  $dir=~s/[\\\/]+$//;
+
+  # ensure the directory doesn't already exist
+  if (-e $dir)
+  {
+    return 0;
+  }
+
+  # make sure the cache directory exists
+  my $dirsofar = "";
+  my $first = 1;
+  foreach my $dirname (split ("/", $dir))
+  {
+    $dirsofar .= "/" unless $first;
+    $first = 0;
+
+    $dirsofar .= $dirname;
+
+    next if $dirname =~ /^(|[a-z]:)$/i;
+    if (!-e $dirsofar)
+    {
+      my $store_umask = umask(0002);
+      my $mkdir_ok = mkdir ($dirsofar, 0777);
+      umask($store_umask);
+      if (!$mkdir_ok)
+      {
+        print STDERR "util::mk_all_dir could not create directory $dirsofar\n";
+        return 0;
+      }
+    }
+  }
+  return (-e $dir);
 }
 
@@ -446,4 +630,6 @@
     my ($src, $dest, $verbosity) = @_;
 
+    print "&util::hard_link( $src, $dest, $verbosity)\n";
+
     # remove trailing slashes from source and destination files
     $src =~ s/[\\\/]+$//;
@@ -452,13 +638,13 @@
 ##    print STDERR "**** src = ", unicode::debug_unicode_string($src),"\n";
     # a few sanity checks
-    if (-e $dest) {
+    if (&util::file_exists($dest)) {
 	# destination file already exists
 	return;
     }
-    elsif (!-e $src) {
+    elsif (!&util::file_exists($src)) {
 	print STDERR "util::hard_link source file \"$src\" does not exist\n";
 	return 1;
     }
-    elsif (-d $src) {
+    elsif (!&util::file_exists($src) && &util::dir_exists($src)) {
 	print STDERR "util::hard_link source \"$src\" is a directory\n";
 	return 1;
@@ -466,6 +652,28 @@
 
     my $dest_dir = &File::Basename::dirname($dest);
-    mk_all_dir($dest_dir) if (!-e $dest_dir);
-
+    if (!&util::dir_exists($dest_dir))
+    {
+      mk_all_dir($dest_dir);
+    }
+
+    # HDFS Support - we can't ever link, copy instead
+    if (&util::isHDFS($src))
+    {
+      if (&util::isHDFS($dest))
+      {
+        &util::executeHDFSCommand('put', $src, $dest);
+        return 0;
+      }
+      else
+      {
+        &util::executeHDFSCommand('get', $src, $dest);
+        return 0;
+      }
+    }
+    elsif (&util::isHDFS($dest))
+    {
+      &util::executeHDFSCommand('put', $src, $dest);
+      return 0;
+    }
 
 	if (!link($src, $dest)) {
@@ -553,5 +761,4 @@
 }
 
-
 sub fd_exists
 {
@@ -563,4 +770,16 @@
 
     my $exists = 0;
+
+    # Support for HDFS [jmt12]
+    if (&util::isHDFS($filename_full_path))
+    {
+      # very limited test op support for HDFS
+      if ($test_op ne '-d' && $test_op ne '-e' && $test_op ne '-z')
+      {
+        $test_op = '-e';
+      }
+      my $result = &util::executeHDFSCommand('test ' . $test_op, $filename_full_path);
+      return ($result == 0);
+    }
 
     if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
@@ -587,5 +806,8 @@
 {
     my ($filename_full_path) = @_;
-
+#    if ($filename_full_path =~ /^hdfs:/)
+#    {
+#      print "-> util::file_exists(" . $filename_full_path . ")\n";
+#    }
     return fd_exists($filename_full_path,"-f");
 }
@@ -594,5 +816,8 @@
 {
     my ($filename_full_path) = @_;
-
+#    if ($filename_full_path =~ /^hdfs:/)
+#    {
+#      print "-> util::dir_exists(" . $filename_full_path . ")\n";
+#    }
     return fd_exists($filename_full_path,"-d");
 }
@@ -837,8 +1062,31 @@
     my $filename = shift (@_);
 
-    # need to put single backslash back to double so that regex works
-    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-	$filename =~ s/\\/\\\\/g;
-    }
+    # need to make single backslashes double so that regex works
+    $filename =~ s/\\/\\\\/g; # if ($ENV{'GSDLOS'} =~ /^windows$/i);    
+	
+    # note that the first part of a substitution is a regex, so RE chars need to be escaped,
+    # the second part of a substitution is not a regex, so for e.g. full-stop can be specified literally
+	$filename =~ s/\./\\./g; # in case there are extensions/other full stops, escape them
+	$filename =~ s@\(@\\(@g; # escape brackets
+	$filename =~ s@\)@\\)@g; # escape brackets
+	$filename =~ s@\[@\\[@g; # escape brackets
+	$filename =~ s@\]@\\]@g; # escape brackets
+	
+    return $filename;
+}
+
+sub unregex_filename {
+    my $filename = shift (@_);
+
+    # need to put doubled backslashes for regex back to single
+	$filename =~ s/\\\./\./g; # remove RE syntax for .
+	$filename =~ s@\\\(@(@g; # remove RE syntax for ( => "\(" turns into "("
+	$filename =~ s@\\\)@)@g; # remove RE syntax for ) => "\)" turns into ")"
+	$filename =~ s@\\\[@[@g; # remove RE syntax for [ => "\[" turns into "["
+	$filename =~ s@\\\]@]@g; # remove RE syntax for ] => "\]" turns into "]"
+	
+	# \\ goes to \
+	# This is the last step in reverse mirroring the order of steps in filename_to_regex()
+	$filename =~ s/\\\\/\\/g; # remove RE syntax for \    
     return $filename;
 }
@@ -869,4 +1117,7 @@
     $filename =~ s/[\\\/]$//;
 
+    # restore protocols if present [jmt12]
+    $filename =~ s/(file|hdfs|https?):\/([^\/])/$1:\/\/$2/g;
+
     return $filename;
 }
@@ -902,18 +1153,9 @@
 }
 
-my $oid_warned_about_periods = 0;
 
 sub tidy_up_oid {
-    my ($OID, $verbosity) = @_;
-    if (!defined $verbosity)
-    {
-      $verbosity = 2;
-    }
+    my ($OID) = @_;
     if ($OID =~ /\./) {
-      if (!$oid_warned_about_periods)
-      {
-        print STDERR "Warning, identifier $OID contains periods (.), removing them for this and future documents\n";
-        $oid_warned_about_periods = 1;
-      }
+	print STDERR "Warning, identifier $OID contains periods (.), removing them\n";
 	$OID =~ s/\.//g; #remove any periods
     }
@@ -935,22 +1177,11 @@
 
     # do not prepend any value/path that's already in the environment variable
-    if ($ENV{'GSDLOS'} =~ /^windows$/i) 
-    {
-	my $escaped_val = $val;
-	$escaped_val =~ s/\\/\\\\/g; # escape any Windows backslashes for upcoming regex
-	if (!defined($ENV{$var})) {
-	    $ENV{$var} = "$val";
-	}
-	elsif($ENV{$var} !~ m/$escaped_val/) { 
-	    $ENV{$var} = "$val;".$ENV{$var};
-	}
-    }
-    else {
-	if (!defined($ENV{$var})) {
-	    $ENV{$var} = "$val";
-	}
-	elsif($ENV{$var} !~ m/$val/) { 
-	    $ENV{$var} = "$val:".$ENV{$var};
-	}
+    
+    my $escaped_val = &filename_to_regex($val); # escape any backslashes and brackets for upcoming regex
+    if (!defined($ENV{$var})) {
+	$ENV{$var} = "$val";
+    }
+    elsif($ENV{$var} !~ m/$escaped_val/) { 
+	$ENV{$var} = "$val;".$ENV{$var};
     }
 }
@@ -958,25 +1189,14 @@
 sub envvar_append {
     my ($var,$val) = @_;
-
+    
     # do not append any value/path that's already in the environment variable
-    if ($ENV{'GSDLOS'} =~ /^windows$/i) 
-    {
-	my $escaped_val = $val;
-	$escaped_val =~ s/\\/\\\\/g; # escape any Windows backslashes for upcoming regex
-	if (!defined($ENV{$var})) {
-	    $ENV{$var} = "$val";
-	}
-	elsif($ENV{$var} !~ m/$escaped_val/) { 
-	    $ENV{$var} .= ";$val";
-	}
-    }
-    else {
-	if (!defined($ENV{$var})) {
-	    $ENV{$var} = "$val";
-	}
-	elsif($ENV{$var} !~ m/$val/) { 
-	    $ENV{$var} .= ":$val";
-	}
-    }    
+    
+    my $escaped_val = &filename_to_regex($val); # escape any backslashes and brackets for upcoming regex
+    if (!defined($ENV{$var})) {
+	$ENV{$var} = "$val";
+    }
+    elsif($ENV{$var} !~ m/$escaped_val/) { 
+	$ENV{$var} .= ";$val";
+    }
 }
 
@@ -1047,5 +1267,6 @@
 }
 
-
+# If filename is relative to within_dir, returns the relative path of filename to that directory
+# with slashes in the filename returned as they were in the original (absolute) filename.
 sub filename_within_directory
 {
@@ -1057,6 +1278,5 @@
     }
 	
-    $within_dir =~ s/\\/\\\\/g; # escape DOS style file separator
-
+	$within_dir = &filename_to_regex($within_dir); # escape DOS style file separator and brackets	
     if ($filename =~ m/^$within_dir(.*)$/) {
 	$filename = $1;
@@ -1065,4 +1285,76 @@
     return $filename;
 }
+
+# If filename is relative to within_dir, returns the relative path of filename to that directory in URL format.
+# Filename and within_dir can be any type of slashes, but will be compared as URLs (i.e. unix-style slashes).
+# The subpath returned will also be a URL type filename.
+sub filename_within_directory_url_format
+{
+    my ($filename,$within_dir) = @_;
+	
+	# convert parameters only to / slashes if Windows
+    
+	my $filename_urlformat = &filepath_to_url_format($filename);
+	my $within_dir_urlformat = &filepath_to_url_format($within_dir);
+
+	#if ($within_dir_urlformat !~ m/\/$/) {
+		# make sure directory ends with a slash
+		#$within_dir_urlformat .= "/";
+    #}
+	
+	my $within_dir_urlformat_re = &filename_to_regex($within_dir_urlformat); # escape any special RE characters, such as brackets
+	
+	#print STDERR "@@@@@ $filename_urlformat =~ $within_dir_urlformat_re\n";
+	
+	# dir prefix may or may not end with a slash (this is discarded when extracting the sub-filepath)
+    if ($filename_urlformat =~ m/^$within_dir_urlformat_re(?:\/)*(.*)$/) {
+		$filename_urlformat = $1;
+    }
+    
+    return $filename_urlformat;
+}
+
+# Convert parameter to use / slashes if Windows (if on Linux leave any \ as is,
+# since on Linux it doesn't represent a file separator but an escape char).
+sub filepath_to_url_format
+{
+	my ($filepath) = @_;
+	if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+		# Only need to worry about Windows, as Unix style directories already in url-format
+		# Convert Windows style \ => /
+		$filepath =~ s@\\@/@g;		
+	}
+	return $filepath;
+}
+
+# regex filepaths on windows may include \\ as path separator. Convert \\ to /
+sub filepath_regex_to_url_format
+{
+    my ($filepath) = @_;
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	# Only need to worry about Windows, as Unix style directories already in url-format
+	# Convert Windows style \\ => /
+	$filepath =~ s@\\\\@/@g;		
+    }
+    return $filepath;
+    
+}
+
+# Like File::Basename::fileparse, but expects filepath in url format (ie only / slash for dirsep)
+# and ignores trailing /
+# returns (file, dirs) dirs will be empty if no subdirs
+sub url_fileparse
+{
+    my ($filepath) = @_;
+    # remove trailing /
+    $filepath =~ s@/$@@;
+    if ($filepath !~ m@/@) {
+	return ($filepath, "");
+    } 
+    my ($dirs, $file) = $filepath =~ m@(.+/)([^/]+)@;
+    return ($file, $dirs);
+	
+}
+
 
 sub filename_within_collection
@@ -1185,4 +1477,11 @@
 {
     my ($filename) = @_;
+
+    # support for explicit protocol prefixes, for example file: or hdfs:,
+    # which must be absolute [jmt12]
+    if ($filename =~ /^\w+:\/\//)
+    {
+      return 1;
+    }
 
     if ($ENV{'GSDLOS'} =~ /^windows$/i) {
@@ -1502,4 +1801,12 @@
 }
 
+sub os_dir() {
+    
+    my $gsdlarch = "";
+    if(defined $ENV{'GSDLARCH'}) {
+	$gsdlarch = $ENV{'GSDLARCH'};
+    }
+    return $ENV{'GSDLOS'}.$gsdlarch;
+}
 
 # Returns the greenstone URL prefix extracted from the appropriate GS2/GS3 config file. 
@@ -1519,5 +1826,5 @@
     } else {
 	$defaultUrlprefix = "/greenstone";
-	$configfile = &util::filename_cat($ENV{'GSDLHOME'}, "cgi-bin", "gsdlsite.cfg");
+	$configfile = &util::filename_cat($ENV{'GSDLHOME'}, "cgi-bin", &os_dir(), "gsdlsite.cfg");
 	push(@propertynames, (qw/\nhttpprefix/, qw/\ngwcgi/)); # inspect one property then the other 
     }
@@ -1643,4 +1950,258 @@
 }
 
+# returns the path to the java command in the JRE included with GS (if any),
+# quoted to safeguard any spaces in this path, otherwise a simple java
+# command is returned which assumes and will try for a system java.
+sub get_java_command {
+    my $java = "java";
+    if(defined $ENV{'GSDLHOME'}) { # should be, as this script would be launched from the cmd line 
+	                           # after running setup.bat or from GLI which also runs setup.bat
+	my $java_bin = &util::filename_cat($ENV{'GSDLHOME'},"packages","jre","bin");
+	if(-d $java_bin) {
+	    $java = &util::filename_cat($java_bin,"java");
+	    $java = "\"".$java."\""; # quoted to preserve spaces in path
+	}
+    }
+    return $java;
+}
+
+
+# Given the qualified collection name (colgroup/collection), 
+# returns the collection and colgroup parts
+sub get_collection_parts {
+	# http://perldoc.perl.org/File/Basename.html
+	# my($filename, $directories, $suffix) = fileparse($path);
+	# "$directories contains everything up to and including the last directory separator in the $path 
+	# including the volume (if applicable). The remainder of the $path is the $filename."
+	#my ($collection, $colgroup) = &File::Basename::fileparse($qualified_collection);	
+
+	my $qualified_collection = shift(@_); 
+
+	# Since activate.pl can be launched from the command-line, including by a user, 
+	# best not to assume colgroup uses URL-style slashes as would be the case with GLI
+	# Also allow for the accidental inclusion of multiple slashes
+	my ($colgroup, $collection) = split(/[\/\\]+/, $qualified_collection); #split('/', $qualified_collection);
+	
+	if(!defined $collection) {
+		$collection = $colgroup;
+		$colgroup = "";
+	}
+	return ($collection, $colgroup);
+}
+
+# work out the "collectdir/collection" location
+sub resolve_collection_dir {
+	my ($collect_dir, $qualified_collection, $site) = @_; #, $gs_mode
+	
+	my ($colgroup, $collection) = &util::get_collection_parts($qualified_collection);	
+	
+	if (defined $collect_dir) {
+	    return &util::filename_cat($collect_dir,$colgroup, $collection);
+	}
+	else {
+	    if (defined $site) {
+			return &util::filename_cat($ENV{'GSDL3HOME'},"sites",$site,"collect",$colgroup, $collection);
+	    }
+	    else {
+			return &util::filename_cat($ENV{'GSDLHOME'},"collect",$colgroup, $collection);
+	    }
+	}
+}
+
+# ====== John's new fuctions to support HDFS ======
+
+# /**
+#  *  Executes a HDFS command without caring about the resulting output while
+#  *  still reacting appropriately to failed executions.
+#  */
+sub executeHDFSCommand
+{
+  my $action = shift(@_);
+  my $command = 'hadoop fs -' . $action . ' "' . join('" "', @_) . '"';
+  my $result = `$command 2>&1`;
+  my $return_value = $?;
+  ###rint STDERR "-> util::executeHDFSCommand('" . $command . "') => " . $return_value . "\n";
+  return $return_value;
+}
+
+# /** @function file_canread()
+#  */
+sub file_canread
+{
+  my ($filename_full_path) = @_;
+  # the HDFS support doesn't have '-r' so it will revert to '-e'
+  return fd_exists($filename_full_path,"-r");
+}
+# /** file_canread() **/
+
+sub file_openfdcommand
+{
+  my ($filename_full_path, $mode) = @_;
+  ##rint STDERR "-> util::file_openfdcommand('" . $filename_full_path . "', '" . $mode . "')\n";
+  # I'll set to read by default, as that is less destructive to precious files
+  # on your system...
+  if (!defined $mode)
+  {
+    $mode = '<';
+  }
+  my $open_fd_command = $mode . $filename_full_path;
+  if (&util::isHDFS($filename_full_path))
+  {
+    # currently don't really support append, but might be able to do something
+    # like:
+    # hadoop fs -cat /user/username/folder/csv1.csv \
+    #   /user/username/folder/csv2.csv | hadoop fs -put - \
+    #   /user/username/folder/output.csv
+    if ($mode eq '>>' || $mode eq '>')
+    {
+      # if the file already exists, put won't clobber it like a proper write
+      # would, so try to delete it (can fail)
+      if (&util::file_exists($filename_full_path))
+      {
+        &util::rm($filename_full_path);
+      }
+      # then create the command
+      $open_fd_command = '| ' . &util::generateHDFSCommand('put', '-', $filename_full_path);
+    }
+    else
+    {
+      $open_fd_command = &util::generateHDFSCommand('cat', $filename_full_path) . ' |';
+    }
+  }
+  return $open_fd_command;
+}
+
+# /** @function file_readdir()
+#  *  Provide a function to return the files within a directory that is aware
+#  *  of protocols other than file://
+#  *  @param $dirname  the full path to the directory
+#  *  @param $dir_ref  a reference to an array to populate with files
+#  */
+sub file_readdir
+{
+  my ($dirname, $dir_ref) = @_;
+  my $dir_read = 0;
+  if (&util::isHDFS($dirname))
+  {
+    my $hdfs_command = &util::generateHDFSCommand('ls', $dirname);
+    my $result = `$hdfs_command 2>&1`;
+    my @lines = split(/\r?\n/, $result);
+    foreach my $line (@lines)
+    {
+      if ($line =~ /\/([^\/]+)$/)
+      {
+        my $file = $1;
+        push(@{$dir_ref}, $file);
+      }
+    }
+    $dir_read = 1;
+  }
+  elsif (opendir(DIR, $dirname))
+  {
+    my @dirs = readdir(DIR);
+    push(@{$dir_ref}, @dirs);
+    closedir(DIR);
+    $dir_read = 1;
+  }
+  return $dir_read;
+}
+# /** file_readdir() **/
+
+# /**
+#  */
+sub file_lastmodified
+{
+  my ($filename_full_path) = @_;
+  my $last_modified = 0;
+  if (&util::isHDFS($filename_full_path))
+  {
+    my $file_stats = file_stats($filename_full_path);
+    my $mod_date = $file_stats->{'modification_date'};
+    my $mod_time = $file_stats->{'modification_time'};
+    # Last modified should be in number of days (as a float) since last
+    # modified - but I'll just return 0 for now
+    $last_modified = 0.00;
+  }
+  else
+  {
+    $last_modified = -M $filename_full_path;
+  }
+  return $last_modified;
+}
+# /** file_lastmodified() **/
+
+# /** @function file_size
+#  *  Replacement for "-s" in Greenstone buildtime, as we need a version that
+#  *  HDFS aware
+#  */
+sub file_size
+{
+  my ($filename_full_path) = @_;
+  my $size = 0;
+  if (&util::isHDFS($filename_full_path))
+  {
+    my $file_stats = file_stats($filename_full_path);
+    $size = $file_stats->{'filesize'};
+  }
+  else
+  {
+    $size = -s $filename_full_path;
+  }
+  return $size;
+}
+# /** file_size() **/
+
+sub file_stats
+{
+  my ($filename_full_path) = @_;
+  my $stats = {};
+  if (&util::isHDFS($filename_full_path))
+  {
+    # - LS is the only way to get these details from HDFS (-stat doesn't
+    #   provide enough information)
+    my $hdfs_command = &util::generateHDFSCommand('ls', $filename_full_path);
+    my $result = `$hdfs_command 2>&1`;
+    # - parse the results
+    if ($result =~ /([ds\-][rwx\-]+)\s+(\d+)\s+([^\s]+)\s+([^\s]+)\s+(\d+)\s+(\d\d\d\d-\d\d-\d\d)\s+(\d\d:\d\d)\s+([^\s]+)$/)
+    {
+      $stats->{'filename'} = $8;
+      $stats->{'replicas'} = $2;
+      $stats->{'filesize'} = $5;
+      $stats->{'modification_date'} = $6;
+      $stats->{'modification_time'} = $7;
+      $stats->{'permissions'} = $1;
+      $stats->{'userid'} = $3;
+      $stats->{'groupid'} = $4;
+    }
+    else
+    {
+      die("Error! Failed to parse HDFS ls result: " . $result . "\n");
+    }
+  }
+  return $stats;
+}
+# /** file_stats() **/
+
+# /**
+#  */
+sub generateHDFSCommand
+{
+  my $flags = shift(@_);
+  return 'hadoop fs -' . $flags . ' "' . join('" "', @_) . '"';
+}
+# /** generateHDFSCommand() **/
+
+# /** @function isHDFS()
+#  *  Determine if the given path exists within a HDFS system by checking for
+#  *  the expected protocol prefix.
+#  *  @param $full_path  the path to check
+#  *  @return 1 if within HDFS, 0 otherwise
+#  */
+sub isHDFS
+{
+  my ($full_path) = @_;
+  return (lc(substr($full_path, 0, 7)) eq 'hdfs://');
+}
+# /** isHDFS() **/
 
 1;
