Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils.pm	(revision 27384)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils.pm	(revision 27384)
@@ -0,0 +1,806 @@
+###########################################################################
+#
+# FileUtils.pm -- functions for dealing with files. Will delegate to the
+#                 appropriate filesystem driver based upon any file
+#                 protocol specified and dependent on configuration as
+#                 defined by the collection admin
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2013 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package FileUtils;
+
+# Perl Modules
+use strict;
+use Symbol qw<qualify>;
+
+# Greenstone Modules
+use util;
+
+# Configuration
+my $debug = 0;
+
+# /** @function _callFunction($driver_name, $function_name, ...)
+#  *  Make a function call to a dynamically loaded database driver.
+#  *  @param $driver_name - The name of the file protocol driver to load
+#  *  @param $function_name - The function within the driver to call
+#  *  @param <rest> - The parameters to be passed to the function called
+#  */
+sub _callFunction
+{
+  my $driver_name = shift(@_);
+  my $function_name = shift(@_);
+  &_prettyPrint(0, $driver_name, $function_name, @_) unless (!$debug);
+  # Need to look within fileutils directory
+  my $package_name = 'FileUtils::' . $driver_name;
+  # Try to load the requested infodb type
+  if (!&_loadDriver($package_name))
+  {
+    &printError('Failed to load requested file protocol driver: ' . $package_name, 1);
+  }
+  # Then make call to the newly created package
+  no strict;
+  # Better check that the function we are about to call exists
+  my $symbol = qualify($function_name, $package_name);
+  unless ( defined &{$symbol} )
+  {
+    &printError('Function not found: ' . $package_name . '::' . $function_name, 1);
+  }
+  # Call the function and get result if applicable
+  my $result = &{$symbol}(@_);
+  &_prettyPrint(1, $result) unless (!$debug);
+  return $result;
+}
+# /** callFunction() **/
+
+# /** @function _prettyPrint($type, ...)
+#  *  Print a debugging message to STDERR constructed from the <rest> based
+#  *  upon the type.
+#  *  @param type - If 0, output the start of a function with a listing of its
+#  *                parameters. If 1, output the result of a function. 2 is
+#  *                used for function errors. Default to simply printing what-
+#  *                ever else is in <rest>
+#  */
+sub _prettyPrint
+{
+  my $type = shift(@_);
+  if (!defined $type || $type > 2)
+  {
+    $type = -1;
+  }
+  my ($package, $filename, $line, $function) = caller(1);
+  my $message;
+  # Start of a function
+  if (0 == $type)
+  {
+    $message = $package . '::' . $function . '(';
+    my $argument = shift(@_);
+    my $first = 1;
+    while (defined $argument)
+    {
+      if (!$first)
+      {
+        $message .= ', ';
+      }
+      else
+      {
+        $first = 0;
+      }
+      if ($argument =~ /\D/)
+      {
+        $message .= '"' . $argument . '"';
+      }
+      else
+      {
+        $message .= $argument;
+      }
+      $argument = shift(@_);
+    }
+    $message .= ')';
+  }
+  # Result of a function
+  elsif (1 == $type)
+  {
+    $message = $package . '::' . $function . '() => ';
+    my $result = shift(@_);
+    if ($result =~ /\D/)
+    {
+      $message .= '"' . $result . '"';
+    }
+    else
+    {
+      $message .= $result;
+    }
+  }
+  elsif (2 == $type)
+  {
+    my $error = shift(@_);
+    $message = 'Error in ' . $package . '::' . $function . '()! ' . $error;
+  }
+  # Else we leave the message as it is
+  else
+  {
+    $message = join("\n", @_);
+  }
+  print STDERR "[" . time() . "] " . $message . "\n";
+}
+# /** _prettyPrint($type, $function, $message) **/
+
+# /** @function _determineDriver()
+#  *  Given a file path determine the appropriate protocol. For now anything
+#  *  other than a full path beginning with an explicit protocol will default
+#  *  to using 'local' file functions.
+#  *  @return 'local'
+#  */
+sub _determineDriver
+{
+  my $path = shift(@_);
+  &_prettyPrint(0, $path) unless (!$debug);
+  # Determine the appropriate driver from the protocol
+  my $driver = 'LocalFS';
+  # - this is were I'll eventually have the ability to configure
+  #   what driver handles what protocol, hopefully from the collect.cfg
+  my $colon_index = index($path, ':');
+  if ($colon_index > -1)
+  {
+    my $protocol = substr($path, 0, $colon_index);
+    # check the perl module exists
+    eval
+    {
+      require 'FileUtils/' . $protocol . '.pm';
+    };
+    if ($@)
+    {
+      die($@);
+      print STDERR 'Warning! FileUtils::_determineDriver() driver not found (defaulting to local filesystem):' . $protocol . "\n" . $@ . "\n";
+    }
+    else
+    {
+      $driver = $protocol;
+    }
+  }
+  &_prettyPrint(1, $driver) unless (!$debug);
+  return $driver;
+}
+# /** _determineDriver()
+
+# /** @function _loadDriver($class, ...)
+#  *  Runtime class loading for use in FileUtils to load various protocol
+#  *  drivers, possibly configured in the collect.cfg, at runtime.
+#  *  @param $class - The class name (including any path) to load
+#  *  @param <rest> - any function aliases you want exported
+#  */
+sub _loadDriver
+{
+  my $class = shift(@_);
+  &_prettyPrint(0, $class) unless (!$debug);
+  # Convert the Perl Module-like name into a file path
+  (my $file = "$class.pm") =~ s|::|/|g;
+  # - ensure we haven't already loaded this class
+  unless( $INC{$file} )
+  {
+    require $file;
+  }
+  # - this is the magic that actually instantiates the class (rubberstamp?)
+  # - we pass @_ to action any function aliases exports requested
+  eval
+  {
+    $class->import(@_);
+  };
+  # - by now the driver file should have been loaded
+  my $result = defined $INC{$file};
+  &_prettyPrint(1, $result) unless (!$debug);
+  return $result;
+}
+# /** _loadDriver($class, ...) **/
+
+################################################################################
+
+
+## @function printError()
+#
+sub printError
+{
+  my ($message, $fatal) = @_;
+  my ($package, $filename, $line, $function) = caller(1);
+  if (defined $!)
+  {
+    $message .= ' (' . $! . ')';
+  }
+  if (defined $fatal && $fatal)
+  {
+    die('Fatal Error! ' . $package . '::' . $function . '() - ' . $message ."\n");
+  }
+  else
+  {
+    print STDERR 'Error! ' . $package . '::' . $function . '() - ' . $message ."\n";
+  }
+}
+## printError()
+
+
+## @function printWarning
+#
+sub printWarning
+{
+  my ($message) = @_;
+  my ($package, $filename, $line, $function) = caller(1);
+  print STDERR 'Warning! ' . $package . '::' . $function . '() - ' . $message . "\n";
+}
+## printWarning()
+
+################################################################################
+######################## Legacy function name mappings  ########################
+################################################################################
+# Note: there are lots of functions involving files/directories/paths etc found
+# in utils.pm that are not represented here. My intention was to just have those
+# functions that need to be dynamic based on filesystem, or need some rejigging
+# to be filesystem aware. This is an argument, I guess, for moving some of the
+# other functions here so that they are nicely encapsulated - but the question
+# is what to do with functions like filename_within_directory_url_format() which
+# is more URL based than file based... dunno.
+################################################################################
+
+sub cachedir {return synchronizeDirectory(@_);}
+sub cp {return copyFiles(@_);}
+sub cp_r {print "implement cp_r()";}
+sub cp_r_nosvn {print "implement cp_r_nosvn()";}
+sub cp_r_toplevel {print "implement cp_r_toplevel()";}
+sub differentfiles {return &differentFiles(@_);}
+sub dir_exists {return &directoryExists(@_);}
+sub file_exists {return &fileExists(@_);}
+sub file_lastmodified {return &modificationTime(@_);}
+sub file_readdir {return readDirectory(@_);}
+sub file_size {return &fileSize(@_);}
+sub filename_cat {return filenameConcatenate(@_);}
+sub filename_is_absolute {return &isFilenameAbsolute();};
+sub filtered_rm_r {print "implement filtered_rm_r()";}
+sub hard_link {print "implement hard_link()";}
+sub is_dir_empty {return &isDirectoryEmpty();}
+sub mk_all_dir {return &makeAllDirectories(@_);}
+sub mk_dir {return &makeDirectory(@_);}
+sub mv {return &moveFiles(@_);}
+sub mv_dir_contents {print "implement mv_dir_contents()";}
+sub rm {print "implement rm()";}
+sub rm_debug {print "implement rm_debug()";}
+sub rm_r {print "implement rm_r()";}
+sub soft_link {print "implement soft_link()";}
+
+################################################################################
+########## Common functions  ##########
+################################################################################
+# Note: these are the file-based functions that are not dynamic in themselves,
+# but that need significant changes to support multiple possible filesystems.
+################################################################################
+
+
+## @function differentFiles
+# (previous util.pm version used -e, -d, and 'stat', none of which support
+#  *  filesystems such as hadoop)
+#  */
+sub differentFiles
+{
+  my ($file1, $file2, $verbosity) = @_;
+  if (!defined $verbosity)
+  {
+    $verbosity = 1
+  }
+
+  # remove trailing slashes
+  $file1 =~ s/\/+$//;
+  $file2 =~ s/\/+$//;
+
+  # chop off the last part of the path as the file/dir name
+  my ($file1name) = $file1 =~ /\/([^\/]*)$/;
+  my ($file2name) = $file2 =~ /\/([^\/]*)$/;
+
+  # - cheapest first; test the two filename strings are the same
+  if ($file1name ne $file2name)
+  {
+    print STDERR "filenames are not the same\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  if (!&pathExists($file1) || !&pathExists($file2))
+  {
+    print STDERR "one or other file doesn't exist\n" if ($verbosity >= 2);
+    return -1;
+  }
+
+  if (&directoryExists($file1))
+  {
+    if (!&directoryExists($file2))
+    {
+      print STDERR "one file is a directory\n" if ($verbosity >= 2);
+      return 1;
+    }
+    return 0;
+  }
+
+  # both must be regular files
+  unless (&fileExists($file1) && &fileExists($file2))
+  {
+    print STDERR "one file is not a regular file\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  # the size of the files must be the same
+  if (&fileSize($file1) != &fileSize($file2))
+  {
+    print STDERR "different sized files\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  # the second file cannot be older than the first
+  if (&modificationTime($file1) > &modificationTime($file2))
+  {
+    print STDERR "file is older\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  return 0;
+}
+# /** differentFiles() **/
+
+
+## @function fileGetContents()
+#
+sub fileGetContents
+{
+  my ($path) = @_;
+  my $content;
+  my $driver = &FileUtils::_determineDriver($path);
+  my $filesize = &FileUtils::_callFunction($driver, 'fileSize', $path);
+  my $fh;
+  &FileUtils::_callFunction($driver, 'openFileHandle', $path, '<', \$fh);
+  sysread($fh, $content, $filesize);
+  &FileUtils::_callFunction($driver, 'closeFileHandle', \$fh);
+  return $content;
+}
+## fileGetContents()
+
+
+## @function filePutContents()
+#
+sub filePutContents
+{
+  my ($path, $str) = @_;
+  my $driver = &FileUtils::_determineDriver($path);
+  my $fh;
+  &FileUtils::_callFunction($driver, 'openFileHandle', $path, '>', \$fh);
+  print $fh $str;
+  &FileUtils::_callFunction($driver, 'closeFileHandle', \$fh);
+  return 1;
+}
+## filePutContents(path, str)
+
+
+## @function sanitizePath()
+#
+sub sanitizePath
+{
+  my ($path) = @_;
+  # fortunately filename concatenate will perform all the double slash removal,
+  # end slash removal we need, and in a protocol aware fashion
+  return &filenameConcatenate($path);
+}
+## sanitizePath()
+
+
+################################################################################
+
+
+## @function canRead()
+#
+sub canRead
+{
+  my ($path) = @_;
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'fileTest', $path, '-R');
+}
+## canRead()
+
+
+## @function closeFileHandle()
+#
+sub closeFileHandle
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'closeFileHandle', @_);
+}
+## closeFileHandle()
+
+# /**
+#  */
+sub copyFiles
+{
+  return &transferFiles(@_, 'COPY');
+}
+# /** copyFiles() **/
+
+# /**
+#  */
+sub directoryExists
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'fileTest', $path, '-d');
+}
+# /** directoryExists($path) **/
+
+# /** @function file_exists($path)
+#  *  Determine if the given file path exists on the target filesystem
+#  *  @param path - the path to the file to test
+#  *  @return true if the file exists, false otherwise
+sub fileExists
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  my $result = &FileUtils::_callFunction($driver, 'fileTest', $path, '-f');
+  return $result;
+}
+# /** fileExists(path) **/
+
+# /** @function filenameConcatenate(<rest>)
+#  */
+sub filenameConcatenate
+{
+  my $first_path_part = shift(@_);
+  my $path = '';
+  if (defined $first_path_part)
+  {
+    my $driver = &FileUtils::_determineDriver($first_path_part);
+    $path = &FileUtils::_callFunction($driver, 'filenameConcatenate', $first_path_part, @_);
+  }
+  return $path;
+}
+# /** filenameConcatenate(<rest>) **/
+
+# /**
+#  */
+sub fileSize
+{
+  my $path = shift(@_);
+  my $driver = &_determineDriver($path);
+  return &_callFunction($driver, 'fileSize', $path);
+}
+# /** fileSize() **/
+
+# /**
+#  */
+sub hardLink
+{
+  my $src_file = shift(@_);
+  my $dst_file = shift(@_);
+  my $src_driver = &FileUtils::_determineDriver($src_file);
+  my $dst_driver = &FileUtils::_determineDriver($dst_file);
+  # you can only symbolic link within the same file system - always
+  if ($src_driver eq 'LocalFS' && $src_driver eq $dst_driver)
+  {
+    &FileUtils::_callFunction($src_driver, 'linkFile', 'HARD', $src_file, $dst_file);
+  }
+  # substitute a copy
+  elsif ($src_driver ne 'LocalFS')
+  {
+    &printWarning('Cannot symbolic link on non-local file systems - copying instead: ' . $src_file . ' => ' . $dst_file);
+    &transferFiles($src_file, $dst_file, 'COPY');
+  }
+  else
+  {
+    &printWarning('Cannot symbolic link between file systems - copying instead: ' . $src_file . ' => ' . $dst_file);
+    &transferFiles($src_file, $dst_file, 'COPY');
+  }
+}
+# /** hardLink() **/
+
+## @function isFilenameAbolsute()
+#
+# Determine if the given path is an absolute path (as compared to relative)
+#
+sub isFilenameAbsolute
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'isFilenameAbsolute', $path, '-l');
+}
+## isFilenameAbsolute()
+
+## @function isSymbolicLink()
+#
+# Determine if the given path is a symbolic link
+#
+sub isSymbolicLink
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'fileTest', $path, '-l');
+}
+## isSymbolicLink()
+
+# /** @function makeAllDirectories
+#  */
+sub makeAllDirectories
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'makeAllDirectories', $path);
+}
+# /** makeAllDirectories() **/
+
+# /**
+#  */
+sub makeDirectory
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  # check if the directory already exists - in which case our job is done :)
+  my $result = &FileUtils::_callFunction($driver, 'fileTest', $path, '-d');
+  # not yet - better try and create it then
+  if (!$result)
+  {
+    $result = &FileUtils::_callFunction($driver, 'makeDirectory', $path);
+  }
+  return $result;
+}
+# /** makeDirectory(path) **/
+
+# /**
+#  */
+sub modificationTime
+{
+  my $path = shift(@_);
+  my $driver = &_determineDriver($path);
+  return &_callFunction($driver, 'modificationTime', $path);
+}
+# /** modificationTime() **/
+
+# /**
+#  */
+sub moveFiles
+{
+  return &transferFiles(@_, 'MOVE');
+}
+# /** moveFiles() **/
+
+# /**
+#  */
+sub openFileHandle
+{
+  my $path = shift(@_);
+  # I'll set mode to read by default, as that is less destructive to precious
+  # files on your system...
+  my $mode = shift(@_) || '<';
+  # the all important determining of the driver by protocol
+  my $driver = &FileUtils::_determineDriver($path);
+  # the function call will return true on success, with the reference to the
+  # file handle hopefully populated with a lovely file descriptor
+  return &FileUtils::_callFunction($driver, 'openFileHandle', $path, $mode, @_);
+}
+# /** openFileHandle($file_handle_ref, $path, $mode) **/
+
+## @function pathExists()
+#
+# Determine if a certain path exists on the file system - regardless of whether
+# it is a file or a directory (the equivalent of -e)
+#
+sub pathExists
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'fileTest', $path, '-e');
+}
+## pathExists()
+
+
+## @function readDirectory()
+#
+# Provide a function to return the files within a directory that is aware
+# of protocols other than file://
+# @param $dirname  the full path to the directory
+#
+sub readDirectory
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  return &FileUtils::_callFunction($driver, 'readDirectory', $path);
+}
+## readDirectory()
+
+
+## @function removeFiles()
+#
+# Removes files (but not directories)
+# @param files - An array of filepaths to remove
+#
+sub removeFiles
+{
+  my (@files) = @_;
+  my $num_removed = 0;
+  # Remove the files
+  foreach my $path (@files)
+  {
+    my $driver = &FileUtils::_determineDriver($path);
+    if (&FileUtils::_callFunction($driver, 'removeFiles', $path))
+    {
+      $num_removed++;
+    }
+  }
+  # Check to make sure all of them were removed
+  if ($num_removed != scalar(@files))
+  {
+    &printError('Not all files were removed');
+    $num_removed = 0;
+  }
+  return $num_removed;
+}
+## removeFile(files)
+
+
+## @function removeFilesFiltered()
+#
+sub removeFilesFiltered
+{
+  my $maybe_paths = shift(@_);
+  my @paths = (ref $maybe_paths eq "ARRAY") ? @$maybe_paths : ($maybe_paths);
+  my $num_removed = 0;
+  foreach my $path (@paths)
+  {
+    my $driver = &FileUtils::_determineDriver($path);
+    $num_removed += &FileUtils::_callFunction($driver, 'removeFilesFiltered', $path, @_);
+  }
+  return $num_removed;
+}
+## removeFilesFiltered()
+
+
+## @function removeFilesRecursive()
+#
+# The equivalent of "rm -rf" with all the dangers therein
+#
+sub removeFilesRecursive
+{
+  my $maybe_paths = shift(@_);
+  my @paths = (ref $maybe_paths eq "ARRAY") ? @$maybe_paths : ($maybe_paths);
+  my $num_removed = 0;
+  foreach my $path (@paths)
+  {
+    my $driver = &FileUtils::_determineDriver($path);
+    $num_removed += &FileUtils::_callFunction($driver, 'removeFilesRecursive', $path);
+  }
+  return $num_removed;
+}
+## removeFilesRecursive()
+
+
+## @function softLink()
+#
+sub softLink
+{
+  my $src_file = shift(@_);
+  my $dst_file = shift(@_);
+  my $src_driver = &FileUtils::_determineDriver($src_file);
+  my $dst_driver = &FileUtils::_determineDriver($dst_file);
+  # you can only symbolic link within the same file system - always
+  if ($src_driver eq 'LocalFS' && $src_driver eq $dst_driver)
+  {
+    &FileUtils::_callFunction($src_driver, 'linkFile', 'SOFT', $src_file, $dst_file, @_);
+  }
+  # substitute a copy
+  elsif ($src_driver ne 'LocalFS')
+  {
+    &printWarning('Cannot symbolic link on non-local file systems - copying instead: ' . $src_file . ' => ' . $dst_file);
+    &transferFiles($src_file, $dst_file, 'COPY');
+  }
+  else
+  {
+    &printWarning('Cannot symbolic link between file systems - copying instead: ' . $src_file . ' => ' . $dst_file);
+    &transferFiles($src_file, $dst_file, 'COPY');
+  }
+}
+## softLink()
+
+
+## @function supportsSymbolicLink()
+#
+sub supportsSymbolicLink
+{
+  my $path = shift(@_);
+  my $driver = &FileUtils::_determineDriver($path);
+  &FileUtils::_callFunction($driver, 'supportsSymbolicLink');
+}
+## supportsSymbolicLink()
+
+
+## @function synchronizeDirectory()
+#
+sub synchronizeDirectory
+{
+  my $fromdir = shift(@_);
+  my $driver = &FileUtils::_determineDriver($fromdir);
+  &FileUtils::_callFunction($driver, 'synchronizeDirectory', $fromdir, @_);
+}
+## synchronizeDirectory()
+
+
+## @function transferFiles()
+# @param paths - one or more source paths
+# @param dst_path - the destination path
+# @param mode - copy or move
+#
+sub transferFiles
+{
+  my $transfer_mode = pop(@_);
+  my $dst_path = pop(@_);
+  my (@src_paths) = @_;
+  &_prettyPrint(0, @src_paths, $dst_path, $transfer_mode) unless (!$debug);
+  my $result = 0;
+  my $dst_driver = &_determineDriver($dst_path);
+  if (scalar (@src_paths) == 0)
+  {
+    &printError('No destination directory given');
+  }
+  elsif ((scalar (@src_paths) > 1) && (!&directoryExists($dst_path)))
+  {
+    &printError('If multiple source files are given the destination must be a directory');
+  }
+  else
+  {
+    foreach my $src_path (@src_paths)
+    {
+      my $src_driver = &_determineDriver($src_path);
+      if ($src_driver eq 'LocalFS')
+      {
+        # Local to local
+        if ($dst_driver eq 'LocalFS')
+        {
+          $result += &_callFunction($src_driver, 'transferFile', $transfer_mode, $src_path, $dst_path);
+        }
+        # Local to X
+        else
+        {
+          $result += &_callFunction($dst_driver, 'transferFileFromLocal', $transfer_mode, $src_path, $dst_path);
+        }
+      }
+      # X to Local
+      elsif ($dst_driver eq 'LocalFS')
+      {
+        $result += &_callFunction($src_driver, 'transferFileToLocal', $transfer_mode, $src_path, $dst_path);
+      }
+      # X to X
+      elsif ($src_driver eq $dst_driver)
+      {
+        $result += &_callFunction($src_driver, 'transferFile', $transfer_mode, $src_path, $dst_path);
+      }
+      # X to Y... not supported
+      else
+      {
+        &printError('transfer between two non-local file systems not supported');
+      }
+    }
+    $result = (scalar(@src_paths) == $result);
+  }
+  return $result;
+}
+## transferFiles()
+
+1;
Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm	(revision 27384)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm	(revision 27384)
@@ -0,0 +1,1 @@
+link /home/jmt12/Dropbox/Karearea-Backup/gsdl2/ext/parallel-building/perllib/FileUtils/HDFSShell.pm
Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm	(revision 27384)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm	(revision 27384)
@@ -0,0 +1,1 @@
+link /home/jmt12/Dropbox/Karearea-Backup/gsdl2/ext/parallel-building/perllib/FileUtils/HDThriftFS.pm
Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS/ThriftFH.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS/ThriftFH.pm	(revision 27384)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS/ThriftFH.pm	(revision 27384)
@@ -0,0 +1,284 @@
+###########################################################################
+#
+# HDThriftFS.pm -- file functions acting upon a HDFS via thrift
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2013 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+################################################################################
+## An Object wrapped around a HDThrift file handle usable as a Perl Handle    ##
+################################################################################
+
+package FileUtils::HDThriftFS::ThriftFH;
+
+require Tie::Handle;
+
+use base 'Tie::Handle';
+use Devel::Peek;
+use MIME::Base91;
+
+my $debug_encoding = 0;
+
+sub TIEHANDLE
+{
+  my $class = shift;
+  my $thrift_client = shift;
+  my $self = {};
+  $self->{'buffer_length'} = 4096; # 4k blocks
+  $self->{'client'} = $thrift_client;
+  $self->{'fh'} = 0;
+  $self->{'file_length'} = 0;
+  $self->{'mode'} = 'r';
+  $self->{'read_offset'} = 0; # A read offset
+  return bless $self, $class;
+}
+
+sub WRITE
+{
+  my $self = shift;
+  my ($scalar, $length, $offset) = @_;
+  print STDERR "ThriftFH::WRITE() - implement me!\n";
+}
+
+sub PRINT
+{
+  my $self = shift;
+  my $result = 0;
+  # only available in write or append modes
+  if ($self->{'mode'} eq 'w' || $self->{'mode'} eq 'a')
+  {
+    foreach my $decoded_buffer (@_)
+    {
+      if ($debug_encoding)
+      {
+        print STDERR "Print String: \n=== START ===\n"; Dump($decoded_buffer); print STDERR "\n=== END ===\n\n";
+      }
+      # We now need to Base91 encode everything sent through to Thrift Server
+      my $encoded_buffer = MIME::Base91::encode($decoded_buffer);
+      if ($debug_encoding)
+      {
+        print STDERR "Encoded String: \n=== START ===\n"; Dump($encoded_buffer); print STDERR "\n=== END ===\n\n";
+      }
+      $self->{'client'}->write($self->{'fh'}, $encoded_buffer);
+    }
+    $result = 1;
+  }
+  return $result;
+}
+
+sub PRINTF
+{
+  my $self = shift;
+  my $format = shift;
+  print STDERR "ThriftFH::PRINTF() - implement me!\n";
+}
+
+sub READ
+{
+  my $self = shift;
+  my ($scalar_ref, $length, $offset) = @_;
+  my $bytes_read = 0;
+  if (!$self->{'eof'} && $self->{'mode'} eq 'r')
+  {
+    my $encoded_buffer = $self->{'client'}->read($self->{'fh'}, $offset, $length);
+    if ($debug_encoding)
+    {
+      print STDERR "Read String: \n=== START ===\n"; Dump($encoded_buffer); print STDERR "\n=== END ===\n\n";
+    }
+    my $decoded_buffer = MIME::Base91::decode($encoded_buffer);
+    if ($debug_encoding)
+    {
+      print STDERR "Decoded String: \n=== START ===\n"; Dump($decoded_buffer); print STDERR "\n=== END ===\n\n";
+    }
+    $bytes_read = scalar($decoded_buffer);
+    # the only way I can see to modify the callers version of $scalar
+    $_[0] = $decoded_buffer;
+  }
+  return $bytes_read;
+}
+
+sub READLINE
+{
+  my $self = shift;
+  #rint "ThriftFH:READLINE()\n";
+  #rint " - file_length=" . $self->{'file_length'} . "\n";
+  my $line = undef;
+  # out of file? out of lines!
+  # not in read mode - can't read!
+  if (!$self->{'eof'} && $self->{'mode'} eq 'r')
+  {
+    my $found_line = 0;
+    my $search_offset = $self->{'read_offset'};
+    my $search_eof = $self->{'eof'};
+    # grab a 'buffer' of data starting at my current read offset
+    #rint "->read(fh, $search_offset, " . $self->{'buffer_length'} . ")\n";
+    my $encoded_buffer = $self->{'client'}->read($self->{'fh'}, $search_offset, $self->{'buffer_length'});
+    my $decoded_buffer = MIME::Base91::decode($encoded_buffer);
+    #rint " - buffer=|" . $decoded_buffer . "|\n";
+    if (length($decoded_buffer) < $self->{'buffer_length'} || ($self->{'read_offset'} + length($decoded_buffer)) < $self->{'file_length'})
+    {
+      $search_eof = 1;
+    }
+    while (!$search_eof && index($decoded_buffer, "\n") == -1)
+    {
+      #rint " * no newline found yet - filling buffer...\n";
+      $search_offset = $self->{'read_offset'} + length($decoded_buffer);
+      #rint "->read(fh, $search_offset, " . $self->{'buffer_length'} . ")\n";
+      my $more_encoded_buffer = $self->{'client'}->read($self->{'fh'}, $search_offset, $self->{'buffer_length'});
+      my $more_decoded_buffer = MIME::Base91::decode($more_encoded_buffer);
+      #rint " - more_buffer=|" . $more_decoded_buffer . "|\n";
+      $decoded_buffer .= $more_decoded_buffer;
+      #rint " - buffer=|" . $decoded_buffer. "|\n";
+      # if I read less than I asked for, or my next search offset is beyond the
+      # end of the file, then I've run out of 'file'
+      if (length($more_decoded_buffer) < $self->{'buffer_length'} || ($self->{'read_offset'} + length($decoded_buffer)) < $self->{'file_length'})
+      {
+        $search_eof = 1;
+        #rint " * found eof!\n";
+      }
+    }
+    # if buffer contains newline, then we only return a fragment of buffer and
+    # update the offset
+    my $newline_offset = index($decoded_buffer, "\n");
+    if ($newline_offset > -1)
+    {
+      #rint " * found newline - returning part of buffer\n";
+      $line = substr($decoded_buffer, 0, $newline_offset + 1);
+      # rest of buffer ignored
+    }
+    # otherwise we return all of buffer and mark the file as eof
+    else
+    {
+      #rint " * no newline found - returning all of buffer\n";
+      $line = $decoded_buffer;
+      $self->{'eof'} = 1; # out of file
+    }
+    # move the read pointer by however much we read (in either case)
+    $self->{'read_offset'} += length($line);
+    #rint " => |" . $line . "|\n\n";
+  }
+  return $line;
+}
+
+sub GETC
+{
+  my $self = shift;
+  print STDERR "ThriftFH::GETC() - implement me!\n";
+  return '';
+}
+
+sub CLOSE
+{
+  my $self = shift;
+  $self->{'client'}->close($self->{'fh'});
+  $self->{'fh'} = 0;
+  return 1;
+}
+
+
+## @function open()
+#
+sub OPEN
+{
+  my $self = shift;
+  my $path = shift;
+  my $mode = shift;
+  if (!defined $mode)
+  {
+    $mode = '<'; # Default to read as that's least destructive
+  }
+  #rint STDERR "DEBUG: ThriftFH::OPEN(self, $path, $mode)\n";
+  # write mode always creates a new file - clobbering any existing
+  if ($mode eq 'w' || $mode eq '>')
+  {
+    # if file already exists, create() will (eventually) fail - so we need to
+    # ensure it doesn't already exist beforehand
+    if ($self->{'client'}->exists($path))
+    {
+      $self->{'client'}->rm ($path);
+    }
+    $self->{'fh'} = $self->{'client'}->create($path);
+    $self->{'mode'} = 'w'; # writing
+  }
+  else
+  {
+    # open will fail if the file doesn't already exist - so we may
+    # need to create it beforehand
+    if (!$self->{'client'}->exists($path))
+    {
+      $self->{'fh'} = $self->{'client'}->create($path);
+      $self->{'file_length'} = 0;
+    }
+    else
+    {
+      $self->{'fh'} = $self->{'client'}->open($path);
+      # for reading we need to know the file size so as to avoid Thrift throwing
+      # exceptions when we try to read past the end
+      my $file_stat = $self->{'client'}->stat($path);
+      $self->{'file_length'} = $file_stat->{length};
+    }
+    $self->{'mode'} = 'r'; # reading
+  }
+  return $self;
+}
+## open()
+
+
+sub BINMODE
+{
+  print STDERR "ThriftFH::BINMODE() - implement me!\n";
+}
+
+sub EOF
+{
+  my $self = shift;
+  return $self->{'eof'};
+}
+
+sub TELL
+{
+  print STDERR "ThriftFH::TELL() - implement me!\n";
+}
+
+sub SEEK
+{
+  my $self = shift;
+  my ($offset, $whence) = @_;
+  print STDERR "ThriftFH::SEEK() - implement me!\n";
+}
+
+sub DESTROY
+{
+  my $self = shift;
+  if ($self->{'fh'} != 0)
+  {
+    $self->{'client'}->close($self->{'fh'});
+    $self->{'fh'} = 0;
+  }
+}
+
+sub UNTIE
+{
+  my $self = shift;
+}
+
+1;
Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/LocalFS.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/LocalFS.pm	(revision 27384)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/LocalFS.pm	(revision 27384)
@@ -0,0 +1,1 @@
+link /home/jmt12/Dropbox/Karearea-Backup/gsdl2/ext/parallel-building/perllib/FileUtils/LocalFS.pm
