Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm	(revision 27386)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm	(revision 27386)
@@ -0,0 +1,497 @@
+###############################################################################
+#
+# HDFSShell.pm -- file functions acting upon a HDFS via the CLI hadoop
+# application
+#
+# A component of the Greenstone digital library software from the New Zealand
+# Digital Library Project at the University of Waikato, New Zealand.
+#
+# Copyright (C) 2013 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 675 Mass
+# Ave, Cambridge, MA 02139, USA.
+#
+###############################################################################
+
+package FileUtils::HDFSShell;
+
+# Pragma
+use strict;
+
+# Modules
+use DateTime;
+
+# Configuration
+my $debug = 0;
+
+################################################################################
+######################### Private Functions & Variables ########################
+################################################################################
+
+## @function _executeHDFSCommand()
+#
+# Executes a HDFS command without caring about the resulting output
+# while still reacting appropriately to failed executions.
+#
+sub _executeHDFSCommand
+{
+  my $return_result = shift(@_);
+  if ($return_result != 0 && $return_result != 1)
+  {
+    &FileUtils::printError('Unexpected value for return_result argument - should be 0 or 1: ' . $return_result, 1);
+  }
+  my $command = &_generateHDFSCommand(@_);
+  my $result = `$command 2>&1`;
+  my $return_value = $?;
+  &_printDebug(' -> util::executeHDFSCommand() => |' . $result . '| [' . $return_value . ']');
+  # sometimes we may want the actual resulting output returned, for
+  # instance when parsing ls
+  if ($return_result)
+  {
+    $return_value = $result;
+  }
+  return $return_value;
+}
+## _executeHDFSCommand()
+
+
+## @function _generateHDFSCommand()
+#
+sub _generateHDFSCommand
+{
+  my $action = shift(@_);
+  my @args = @_;
+  my $arguments = '';
+  foreach my $path (@args)
+  {
+    # Replace the prefix with one HDFS Shell understands
+    $path =~ s/HDFSShell:/hdfs:/;
+    # special case for standard streams
+    if ($path eq '-')
+    {
+      $arguments .= '- ';
+    }
+    else
+    {
+      $arguments .= '"' . $path . '" ';
+    }
+  }
+  my $command = 'hadoop fs -' . $action . ' ' . $arguments;
+  &_printDebug(' -> _generateHDFSCommand("' . $action . '", ...) => |' . $command . '|');
+  return $command;
+}
+## _generateHDFSCommand()
+
+
+## @function _printDebug()
+#
+sub _printDebug
+{
+  my ($message) = @_;
+  if ($debug)
+  {
+    print STDERR '[DEBUG] ' . $message . "\n";
+  }
+}
+## _printDebug()
+
+
+################################################################################
+############################### Public Functions ###############################
+################################################################################
+
+
+## @function closeFileHandle()
+#
+sub closeFileHandle
+{
+  my $fh_ref = shift(@_);
+  close($$fh_ref);
+  return 1;
+}
+## closeFileHandle()
+
+
+## @function fileSize()
+#
+sub fileSize
+{
+  my ($path) = @_;
+  my $file_stats = &fileStats($path);
+  return $file_stats->{'filesize'};
+}
+## fileSize()
+
+
+## @function fileStats()
+#
+sub fileStats
+{
+  my ($path) = @_;
+  my $stats = {};
+  my $result = &_executeHDFSCommand(1, 'ls', $path);
+  # - parse the results
+  if ($result =~ /([ds\-][rwx\-]+)\s+(\d+)\s+([^\s]+)\s+([^\s]+)\s+(\d+)\s+(\d\d\d\d-\d\d-\d\d)\s+(\d\d:\d\d)\s+([^\s]+)$/)
+  {
+    $stats->{'filename'} = $8;
+    $stats->{'replicas'} = $2;
+    $stats->{'filesize'} = $5;
+    $stats->{'modification_date'} = $6;
+    $stats->{'modification_time'} = $7;
+    $stats->{'permissions'} = $1;
+    $stats->{'userid'} = $3;
+    $stats->{'groupid'} = $4;
+  }
+  else
+  {
+    &FileUtils::printError('Failed to parse -ls result: ' . $result, 1);
+  }
+  return $stats;
+}
+## fileStats()
+
+
+## @function fileTest()
+#
+sub fileTest
+{
+  my ($filename_full_path, $test_op) = @_;
+  # Sanity tests
+  # Special case: HDFS doesn't support symlinking - swap for -e instead
+  if (!defined $test_op || $test_op eq '-l')
+  {
+    $test_op = '-e';
+  }
+  my $retval = -1; # cmd return fails > 0 are errors
+  # Special case: the easiest way to support -f is to run a -e followed by a -d
+  # (which should fail for files)
+  if ($test_op eq '-f')
+  {
+    my $retval1 = &_executeHDFSCommand(0, 'test -e', $filename_full_path);
+    if ($retval1 == 0)
+    {
+      my $retval2 = &_executeHDFSCommand(0, 'test -d', $filename_full_path);
+      if ($retval2 > 0)
+      {
+        $retval = 0;
+      }
+    }
+  }
+  # very limited test op support for HDFS
+  elsif ($test_op ne '-d' && $test_op ne '-e' && $test_op ne '-z')
+  {
+    &FileUtils::printError('Unknown or unsupported test mode: ' . $test_op);
+  }
+  else
+  {
+    $retval = &_executeHDFSCommand(0, 'test ' . $test_op, $filename_full_path);
+  }
+  return ($retval == 0 ? 1 : 0);
+}
+## fileTest()
+
+
+## @function filenameConcatenate()
+#
+sub filenameConcatenate
+{
+  my $protocol = shift(@_);
+  my $filename = join('/', @_);
+  # remove repeated slashes
+  $filename =~ s/[\/]+/\//g;
+  # append protocol (which may cause multiple slashes)
+  $filename = $protocol . '/' . $filename;
+  # strip any trailing slashes
+  $filename =~ s/[\\\/]$//;
+  return $filename;
+}
+## filenameConcatenate()
+
+
+## @function makeDirectory()
+#
+sub makeDirectory
+{
+  my ($dir) = @_;
+  my $result = &_executeHDFSCommand(0, 'mkdir', $dir);
+  # HDFSShell mkdir returns 0 on success, -1 on failure
+  return ($result == 0 ? 1 : 0);
+}
+## makeDirectory()
+
+
+## @function modificationTime()
+#
+sub modificationTime
+{
+  my ($path) = @_;
+  my $file_stats = &fileStats($path);
+  my $mod_date = $file_stats->{'modification_date'};
+  $mod_date =~ /(\d\d\d\d)-(\d\d)-(\d\d)/;
+  my $mod_year = $1;
+  my $mod_month = $2;
+  my $mod_day = $3;
+  my $mod_time = $file_stats->{'modification_time'};
+  $mod_time =~ /(\d\d):(\d\d)/;
+  my $mod_hour = $1;
+  my $mod_minute = $2;
+  my $mod_datetime = DateTime->new(year      => $mod_year,
+                                   month     => $mod_month,
+                                   day       => $mod_day,
+                                   hour      => $mod_hour,
+                                   minute    => $mod_minute,
+                                   time_zone => 'local');
+  return $mod_datetime->epoch();
+}
+## modificationTime()
+
+
+## @function openFileHandle()
+#
+sub openFileHandle
+{
+  my ($path, $mode, $fh_ref) = @_;
+  if ($mode eq '>>' || $mode eq 'a')
+  {
+    &FileUtils::printError('Append (>>) mode not supported', 1);
+  }
+  elsif ($mode eq '>' || $mode eq 'w')
+  {
+    # the put command fails if the file already exists
+    if (&fileTest($path, '-e'))
+    {
+      &removeFiles($path);
+    }
+    open($$fh_ref, '| ' . &_generateHDFSCommand('put', '-', $path)) or &FileUtils::printError('Failed to open pipe to HDFS (put) for writing: ' . $path, 1);
+  }
+  else
+  {
+    open($$fh_ref, &_generateHDFSCommand('cat', $path) . ' |') or &FileUtils::printError('Failed to open pipe to HDFS (cat) for reading: ' . $path, 1);
+  }
+  return 1;
+}
+## openFileHandle()
+
+
+## @function readDirectory()
+#
+sub readDirectory
+{
+  my ($path) = @_;
+  my @files;
+  my $result = &_executeHDFSCommand(1, 'ls', $path);
+  my @lines = split(/\r?\n/, $result);
+  foreach my $line (@lines)
+  {
+    if ($line =~ /\/([^\/]+)$/)
+    {
+      my $file = $1;
+      push(@files, $file);
+    }
+  }
+  return \@files;
+}
+## readDirectory()
+
+
+## @function removeFiles()
+#
+sub removeFiles
+{
+  my ($path, $including_dir) = @_;
+  my $result;
+  if (defined $including_dir && $including_dir)
+  {
+    $result = &_executeHDFSCommand(0, 'rmr', $path);
+  }
+  else
+  {
+    $result = &_executeHDFSCommand(0, 'rm', $path);
+  }
+  # HDFSShell mkdir returns 0 on success, -1 on failure
+  return ($result == 0 ? 1 : 0);
+}
+## removeFiles()
+
+
+## @function removeFilesFiltered()
+#
+sub removeFilesFiltered
+{
+  my ($paths, $accept_re, $reject_re) = @_;
+  # Perform a depth first, recursive, removal of files and directories that
+  # match the given accept and reject patterns
+  my @paths_array = (ref $paths eq "ARRAY") ? @$paths : ($paths);
+  my $num_removed = 0;
+  foreach my $path (@paths_array)
+  {
+    # remove trailing slashes
+    $path =~ s/[\/\\]+$//;
+    if (!&fileTest($path, '-e'))
+    {
+      &FileUtils::printError('path does not exist: ' . $path);
+    }
+    elsif (&fileTest($path, '-d'))
+    {
+      my @files = @{&readDirectory($path)};
+      foreach my $file (@files)
+      {
+        my $child_path = $path . '/' . $file;
+        $num_removed += &removeFilesFiltered($child_path, $accept_re, $reject_re);
+      }
+      if (!defined $accept_re && !defined $reject_re)
+      {
+        # remove this directory
+        my $result = &removeFiles($path, 1);
+        if ($result != 1)
+        {
+          &FileUtils::printError('could not remove directory: ' . $path);
+        }
+        else
+        {
+          $num_removed++;
+        }
+      }
+    }
+    else
+    {
+      if (defined $reject_re && ($path =~ m/$reject_re/))
+      {
+        next;
+      }
+      if ((!defined $accept_re) || ($path =~ m/$accept_re/))
+      {
+        # remove this file
+        my $result = &removeFiles($path);
+        if ($result != 1)
+        {
+          &FileUtils::printError('could not remove file: ' . $path);
+        }
+        else
+        {
+          $num_removed++;
+        }
+      }
+    }
+  }
+  return $num_removed;
+}
+## removeFilesFiltered()
+
+
+## @function removeFilesRecursive()
+#
+sub removeFilesRecursive
+{
+  my ($path) = @_;
+  # use the more general removeFilesFiltered() function with no accept
+  # or reject expressions
+  return &removeFilesFiltered($path, undef, undef);
+}
+## removeFilesRecursive()
+
+
+## @function supportsSymbolicLink
+#
+sub supportsSymbolicLink
+{
+  return 0;
+}
+## supportsSymbolicLink()
+
+
+## @function transferFile()
+#
+sub transferFile
+{
+  my ($mode, $src, $dst) = @_;
+  my $result;
+  if ($mode eq 'COPY')
+  {
+    $result = &_executeHDFSCommand(0, 'cp', $src, $dst);
+  }
+  else
+  {
+    $result = &_executeHDFSCommand(0, 'mv', $src, $dst);
+  }
+  # HDFSShell mkdir returns 0 on success, -1 on failure
+  return ($result == 0 ? 1 : 0);
+}
+## transferFile()
+
+
+## @function transferFileFromLocal()
+#
+sub transferFileFromLocal
+{
+  my ($mode, $src, $dst) = @_;
+  if (!-f $src)
+  {
+    &FileUtils::printError('Source file (during ' . $mode . ') doesn\'t exists: ' . $src);
+  }
+  if (&fileTest($dst, '-d'))
+  {
+    my ($filename) = $src =~ /([^\\\/]+)$/;
+    $dst .= '/' . $filename;
+  }
+  if (&fileTest($dst, '-f'))
+  {
+    &FileUtils::printError('Destination file (during ' . $mode . ') already exists: ' . $dst);
+  }
+  my $result = &_executeHDFSCommand(0, 'put', $src, $dst);
+  my $remove_result = 1;
+  if ($mode eq 'MOVE')
+  {
+    unlink($src);
+    # failed to delete somehow
+    if (-f $src)
+    {
+      $remove_result = 0;
+    }
+  }
+  return ($result == 0 && $remove_result ? 1 : 0);
+}
+## transferFileFromLocal()
+
+
+## @function transferFileToLocal()
+#
+sub transferFileToLocal
+{
+  my ($mode, $src, $dst) = @_;
+  if (!&fileTest($src, '-f'))
+  {
+    &FileUtils::printError('Source file (during ' . $mode . ') does not exist: ' . $src);
+  }
+  if (-d $dst)
+  {
+    my ($filename) = $src =~ /([^\\\/]+)$/;
+    $dst .= '/' . $filename;
+  }
+  if (-e $dst)
+  {
+    &FileUtils::printError('Destination file (during ' . $mode . ') already exists: ' . $dst);
+  }
+  my $result = &_executeHDFSCommand(0, 'get', $src, $dst);
+  my $remove_result = 1;
+  if ($mode eq 'MOVE')
+  {
+    $remove_result = &removeFiles($src);
+  }
+  return ($result == 0 && $remove_result ? 1 : 0);
+}
+## transferFileToLocal()
+
+
+1;
Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm	(revision 27386)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDThriftFS.pm	(revision 27386)
@@ -0,0 +1,624 @@
+################################################################################
+#
+# HDThriftFS.pm -- file functions acting upon a HDFS via thrift
+#
+# A component of the Greenstone digital library software from the New Zealand
+# Digital Library Project at the University of Waikato, New Zealand.
+#
+# Copyright (C) 2013 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 675 Mass
+# Ave, Cambridge, MA 02139, USA.
+#
+###############################################################################
+
+# Thrift acts as client-server 'relay' between the Perl code and the HDFS.  It
+# allows for persistant connections and so is significantly faster than
+# repeatedly starting Hadoop's Java application over and over. In order to
+# connect to the Thrift server this code needs to know the host and port the
+# server may be found on - information currently hard-coded near the top of the
+# script. There are also a number of Perl module API 'bindings' generated by
+# the Thrift compilation process... currently located within the packages of
+# the Parallel Processing extension.  Note that I make use of some tie() magic
+# so as to provide calling code with 'file handle'-like objects to interact
+# with (print, readline etc), so that is pretty cool.
+
+package FileUtils::HDThriftFS;
+
+# Pragma
+use strict;
+
+# Setup Environment
+BEGIN
+{
+  die "GEXTPARALLELBUILDING not set\n" unless defined $ENV{'GEXTPARALLELBUILDING'};
+  die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+  die "PERL_VERSION not set\n" unless defined $ENV{'PERL_VERSION'};
+  # Bit::Vector and Thrift modules
+  unshift (@INC, $ENV{'GEXTPARALLELBUILDING'} . '/' . $ENV{'GSDLOS'} . '/lib/perl/' . $ENV{'PERL_VERSION'});
+  # ThriftFS Perl API
+  unshift (@INC, $ENV{'GEXTPARALLELBUILDING'} . '/packages/hadoop-1.1.0/contrib/thriftfs/gen-perl');
+}
+
+# Modules - Core
+use Devel::Peek;
+use MIME::Base64;
+use POSIX qw(floor);
+use Symbol;
+use Thrift::Socket;
+use Thrift::BufferedTransport;
+use Thrift::BinaryProtocol;
+# Modules - Thrift
+use HadoopFS::FileSystem;
+# Modules - Greenstone
+use FileUtils::HDThriftFS::ThriftFH;
+use MIME::Base91;
+
+# Configuration
+my $host = "localhost";
+my $port = 58660;
+my $debug_encoding = 0;
+
+# Globals
+my $transport;
+my $thrift_client;
+
+
+## @function END()
+#
+# Ensure the transport layer, if open, is properly closed
+#
+END
+{
+  if (defined $transport)
+  {
+    $transport->close();
+  }
+}
+## END()
+
+
+## @function _establishClient()
+#
+sub _establishClient
+{
+  if (!defined $thrift_client)
+  {
+    ###rint "Create Thrift Client to $host:$port\n";
+    my $socket = Thrift::Socket->new($host, $port);
+    $socket->setSendTimeout(10000);
+    $socket->setRecvTimeout(20000);
+
+    $transport = Thrift::BufferedTransport->new($socket);
+    my $protocol = Thrift::BinaryProtocol->new($transport);
+    $thrift_client = HadoopFS::FileSystemClient->new($protocol);
+
+    eval { $transport->open(); };
+    if ($@)
+    {
+      &_printError('Unable to connect: ' . $@->{message}, 1);
+    }
+  }
+}
+## _establishClient()
+
+
+## @function _generateHDFSPath()
+#
+sub _generateHDFSPath
+{
+  my ($path) = @_;
+  if (ref($path) ne 'HadoopFS::Pathname')
+  {
+    if ($path !~ /HDThriftFS:\/\//)
+    {
+      &_printError('Not a valid thrift URI: ' . $path);
+    }
+    else
+    {
+      $path = HadoopFS::Pathname->new( { pathname => substr($path, 13) } );
+    }
+  }
+  return $path;
+}
+## _generateHDFSPath()
+
+
+################################################################################
+################################################################################
+################################################################################
+
+
+## @function closeFileHandle()
+#
+sub closeFileHandle
+{
+  my $fh_ref = shift(@_);
+  close($$fh_ref);
+  untie($$fh_ref);
+  return 1;
+}
+## closeFileHandle()
+
+
+## @function fileSize()
+#
+sub fileSize
+{
+  my ($path, $test_op) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # - convert the path into a proper thrift path object
+  $path = &_generateHDFSPath($path);
+  my $file_stat = $thrift_client->stat($path);
+  return $file_stat->{length};
+}
+## fileSize()
+
+
+## @function fileTest()
+#
+sub fileTest
+{
+  my ($raw_path, $test_op) = @_;
+  my $result = 0;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # - convert the path into a proper thrift path object
+  my $path = &_generateHDFSPath($raw_path);
+  # note: symbolic linking not supported within HDFS
+  if (!defined $test_op || $test_op eq '-l')
+  {
+    $test_op = '-e';
+  }
+  if ($test_op eq '-d')
+  {
+    if ($thrift_client->exists($path))
+    {
+      my $file = $thrift_client->stat($path);
+      if ($file->{'isdir'})
+      {
+        $result = 1;
+      }
+    }
+  }
+  elsif ($test_op eq '-e')
+  {
+    if ($thrift_client->exists($path))
+    {
+      $result = 1;
+    }
+  }
+  elsif ($test_op eq '-f')
+  {
+    if ($thrift_client->exists($path))
+    {
+      my $file = $thrift_client->stat($path);
+      if (!$file->{'isdir'})
+      {
+        $result = 1;
+      }
+    }
+  }
+  else
+  {
+    &FileUtils::printError('Unknown or unsupported test mode: ' . $test_op);
+  }
+  return $result;
+}
+## fileTest()
+
+
+## @function filenameConcatenate()
+#
+sub filenameConcatenate
+{
+  my $protocol = shift(@_);
+  my $filename = join('/', @_);
+  # remove repeated slashes
+  $filename =~ s/[\/]+/\//g;
+  # append protocol (which may cause multiple slashes)
+  $filename = $protocol . '/' . $filename;
+  # strip any trailing slashes
+  $filename =~ s/[\\\/]$//;
+  return $filename;
+}
+## filenameConcatenate()
+
+
+## @function makeDirectory()
+#
+sub makeDirectory
+{
+  my ($raw_path) = @_;
+  my $result = 0;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # - convert the path into a proper thrift path object
+  my $path = &_generateHDFSPath($raw_path);
+  if (!&fileTest($path, '-d'))
+  {
+    # - create the directory
+    $thrift_client->mkdirs($path);
+  }
+  # - check that it exists
+  return (&fileTest($path, '-d'));
+}
+## makeDirectory()
+
+
+## @function modificationTime()
+#
+sub modificationTime
+{
+  my ($path) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # - convert the path into a proper thrift path object
+  $path = &_generateHDFSPath($path);
+  my $file_stat = $thrift_client->stat($path);
+  return floor($file_stat->{modificationTime} / 1000);
+}
+## modificationTime()
+
+
+## @function openFileHandle()
+#
+sub openFileHandle
+{
+  my ($raw_path, $mode, $fh_ref) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  #rint STDERR "DEBUG: openFileHandle($raw_path, $mode, fh_ref)\n";
+  my $path = &_generateHDFSPath($raw_path);
+  my $fh = gensym();
+  tie(*$fh, "FileUtils::HDThriftFS::ThriftFH", $thrift_client);
+  open($fh, $path, $mode) or die("Failed to open thriftfs");
+  $$fh_ref = $fh;
+  return 1;
+}
+## openFileHandle()
+
+
+## @function readDirectory()
+#
+sub readDirectory
+{
+  my ($raw_path) = @_;
+  my @files;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  my $path = &_generateHDFSPath($raw_path);
+  my $raw_files = $thrift_client->listStatus($path);
+  if ($raw_files && @{$raw_files} > 0)
+  {
+    foreach my $file_stat (@{$raw_files})
+    {
+      my $file_path = $file_stat->{'path'};
+      my ($filename) = $file_path =~ /([^\\\/]+)$/;
+      push(@files, $filename);
+    }
+  }
+  return \@files;
+}
+## readDirectory()
+
+
+## @function removeFiles()
+#
+sub removeFiles
+{
+  my ($path, $recursive) = @_;
+  my $result = 0;
+  if (!defined $recursive)
+  {
+    $recursive = 0;
+  }
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # - convert the path into a proper thrift path object as necessary
+  $path = &_generateHDFSPath($path);
+  if ($thrift_client->exists($path) && ($recursive || &fileTest($path, '-f')))
+  {
+    $thrift_client->rm($path, $recursive);
+    $result = !$thrift_client->exists($path);
+  }
+  return $result;
+}
+## removeFiles()
+
+
+## @function removeFilesFiltered()
+#
+sub removeFilesFiltered
+{
+  my ($paths, $accept_re, $reject_re) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # Perform a depth first, recursive, removal of files and directories that
+  # match the given accept and reject patterns
+  my @paths_array = (ref $paths eq "ARRAY") ? @$paths : ($paths);
+  my $num_removed = 0;
+  foreach my $raw_path (@paths_array)
+  {
+    # remove trailing slashes
+    $raw_path =~ s/[\/\\]+$//;
+    my $path = &_generateHDFSPath($raw_path);
+    if (!$thrift_client->exists($path))
+    {
+      print STDERR "HDThriftFS::removeFilesFiltered() path does not exist: " . $raw_path . "\n";
+    }
+    elsif (&fileTest($path, '-d'))
+    {
+      my @files = @{&readDirectory($path)};
+      foreach my $file (@files)
+      {
+        my $child_path = $raw_path . '/' . $file;
+        $num_removed += &removeFilesFiltered($child_path, $accept_re, $reject_re);
+      }
+      if (!defined $accept_re && !defined $reject_re)
+      {
+        # remove this directory - non-recursively so that the command fails
+        # if there are (somehow) still files contained within
+        $thrift_client->rm($path, 0);
+        if ($thrift_client->exists($path))
+        {
+          print STDERR "HDThriftFS::removeFilesFiltered() couldn't remove directory: " . $raw_path . "\n";
+        }
+        else
+        {
+          $num_removed++;
+        }
+      }
+    }
+    else
+    {
+      if (defined $reject_re && ($raw_path =~ m/$reject_re/))
+      {
+        next;
+      }
+      if ((!defined $accept_re) || ($raw_path =~ m/$accept_re/))
+      {
+        # remove this file
+        $thrift_client->rm($path, 0);
+        if ($thrift_client->exists($path))
+        {
+          print STDERR "HDThriftFS::removeFilesFiltered() couldn't remove file: " . $raw_path . "\n";
+        }
+        else
+        {
+          $num_removed++;
+        }
+      }
+    }
+  }
+  return $num_removed;
+}
+## removeFilesFiltered()
+
+
+## @function removeFilesRecursive()
+#
+sub removeFilesRecursive
+{
+  my ($path) = @_;
+  # use the more general removeFilesFiltered() function with no accept or reject
+  # expressions
+  return &removeFilesFiltered($path, undef, undef);
+}
+## removeFilesRecursive()
+
+
+## @function supportsSymbolicLink
+#
+sub supportsSymbolicLink
+{
+  return 0;
+}
+## supportsSymbolicLink()
+
+
+## @function transferFile()
+#
+sub transferFile
+{
+  my ($mode, $src, $dst) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  #rint STDERR "transferFile($mode, $src, $dst)\n";
+  my $src_path = &_generateHDFSPath($src);
+  my $dst_path = &_generateHDFSPath($dst);
+  if (&fileTest($dst_path, '-d'))
+  {
+    my ($filename) = $src =~ /([^\\\/]+)$/;
+    $dst .= '/' . $filename;
+    $dst_path = &_generateHDFSPath($dst);
+  }
+  if (!$thrift_client->exists($src_path))
+  {
+    &FileUtils::printError('Source file (during ' . $mode . ') does not exist: ' . $src);
+    return 0;
+  }
+  if ($thrift_client->exists($dst_path))
+  {
+    &FileUtils::printError('Destination file (during ' . $mode . ') already exists: ' . $dst);
+    return 0;
+  }
+  # what happens next depends on the mode, and is either very easy or really
+  # hard
+  if ($mode eq 'MOVE')
+  {
+    $thrift_client->rename($src_path, $dst_path);
+  }
+  elsif ($mode eq 'COPY')
+  {
+    # Open the src file for reading
+    #rint STDERR "DEBUG: FHIN opened (should be 'r'): $src\n";
+    my $fhin = $thrift_client->open($src_path);
+    # Create the dst file for writing
+    #rint STDERR "DEBUG: FHOUT created (should be 'w'): $dst\n";
+    my $fhout = $thrift_client->create($dst_path);
+    # Read all of src file writing to dst file
+    # - this is where things have the potential to go wrong, as it doesn't seem
+    #   thrift supports writing bytes
+    # - only strings. May need to see if I can make Perl behave using black
+    #   magic flags (marking string as binary etc) It'll work fine for text
+    #   files though
+    my $data = undef;
+    my $offset = 0;
+    my $length = 4096;
+    # Read 4K blocks at a time
+    while ($data = $thrift_client->read($fhin, $offset, $length))
+    {
+      $thrift_client->write($fhout, $data);
+      $offset += $length;
+      if (length ($data) < $length)
+      {
+        last;
+      }
+    }
+    # Close files
+    $thrift_client->close($fhout);
+    $thrift_client->close($fhin);
+  }
+  my $result = ($thrift_client->exists($dst_path));
+  #rint STDERR "transferFile() => $result\n";
+  return $result;
+}
+## transferFile()
+
+
+## @function transferFileFromLocal()
+#
+sub transferFileFromLocal
+{
+  my ($mode, $src, $dst) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # destination is remote
+  my $dst_path = &_generateHDFSPath($dst);
+  if (&fileTest($dst_path, '-d'))
+  {
+    my ($filename) = $src =~ /([^\\\/]+)$/;
+    $dst .= '/' . $filename;
+    $dst_path = &_generateHDFSPath($dst);
+  }
+  # can't replace - if the file already exists
+  if (&fileTest($dst_path, '-f'))
+  {
+    &FileUtils::printError('Destination file (during ' . $mode . ') already exists: ' . $dst);
+    return 0;
+  }
+  # copy the file
+  my $fhin;
+  open($fhin, '<:raw', $src) or die("Failed to open file for reading: " . $src . " (" . $! . ")");
+  my $decoded = '';
+  my $fhout = $thrift_client->create($dst_path);
+  while (read($fhin, $decoded, 4096))
+  {
+    if ($debug_encoding)
+    {
+      print STDERR "Writing Data: \n=== START ===\n"; Dump($decoded); print STDERR "\n=== END ===\n\n";
+    }
+    # Base64 encode to protect binary
+    #my $encoded = encode_base64($decoded);
+    # Base91 encode to protect binary - we add a Byte Order Marker so the
+    # Thrift Server can detect the need to decode the string sent
+    my $encoded = MIME::Base91::encode($decoded);
+    if ($debug_encoding)
+    {
+      print STDERR "Encoded: \n=== START ===\n"; Dump($encoded); print STDERR "\n=== END ===\n\n";
+    }
+    $thrift_client->write($fhout, $encoded);
+  }
+  close($fhin);
+  $thrift_client->close($fhout);
+  # in general, the transfer has worked if the destination file exists
+  my $result = $thrift_client->exists($dst_path);
+  # if moving, remove the source file from the local filesystem
+  if ($mode eq 'MOVE')
+  {
+    unlink($src);
+    # update result to reflect if we successfully removed the src file
+    $result = $result && (!-f $src);
+  }
+  return $result
+}
+## transferFileFromLocal()
+
+
+## @function transferFileToLocal()
+#
+sub transferFileToLocal
+{
+  my ($mode, $src, $dst) = @_;
+  # ensure we have a connection to the thrift server
+  &_establishClient();
+  # source is remote
+  my $src_path = &_generateHDFSPath($src);
+  if (!$thrift_client->exists($src_path))
+  {
+    &FileUtils::printError('Source file (during ' . $mode . ') does not exist: ' . $src);
+    return 0;
+  }
+  if (-d $dst)
+  {
+    my ($filename) = $src =~ /([^\\\/]+)$/;
+    $dst .= '/' . $filename;
+  }
+  if (-e $dst)
+  {
+    &FileUtils::printError('Destination file (during ' . $mode . ') already exists: ' . $dst);
+    return 0;
+  }
+  # open local file
+  my $fhout;
+  my $encoded = undef;
+  my $offset = 0;
+  my $length = 4096; # Read 4K blocks at a time
+  open($fhout, '>:raw', $dst) or die("Failed to open file for writing: " . $dst);
+  my $fhin = $thrift_client->open($src_path);
+  while ($encoded = $thrift_client->read($fhin, $offset, $length))
+  {
+    if ($debug_encoding)
+    {
+      print STDERR "Reading Data: \n=== START ===\n"; Dump($encoded); print STDERR "\n=== END ===\n\n";
+    }
+    #my $decoded = decode_base64($encoded);
+    my $decoded = MIME::Base91::decode($encoded);
+    if ($debug_encoding)
+    {
+      print STDERR "Decoded: \n=== START ===\n"; Dump($decoded); print STDERR "\n=== END ===\n\n";
+    }
+    print $fhout $decoded;
+    last if (length ($encoded) < $length);
+    $offset += $length;
+  }
+  close($fhout);
+  $thrift_client->close($fhin);
+  # in general, the transfer has worked if the destination file exists
+  my $result = (-f $dst);
+  # if moving, remove the source file from the HDFS filesystem
+  if ($mode eq 'MOVE')
+  {
+    $thrift_client->rm($src_path, 0);
+    # update result to reflect if we successfully removed the src file
+    $result = $result && !$thrift_client->exists($src_path);
+  }
+  return $result;
+}
+## transferFileToLocal()
+
+
+1;
Index: /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/LocalFS.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/LocalFS.pm	(revision 27386)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/LocalFS.pm	(revision 27386)
@@ -0,0 +1,591 @@
+###############################################################################
+#
+# LocalFS.pm -- file functions acting upon the local filesystem
+#
+# A component of the Greenstone digital library software from the New Zealand
+# Digital Library Project at the University of Waikato, New Zealand.
+#
+# Copyright (C) 2013 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 675 Mass
+# Ave, Cambridge, MA 02139, USA.
+#
+###############################################################################
+
+package FileUtils::LocalFS;
+
+# Pragma
+use strict;
+
+
+## @function closeFileHandle
+#
+sub closeFileHandle
+{
+  my $fh_ref = shift(@_);
+  close($$fh_ref);
+  return 1;
+}
+## closeFileHandle()
+
+
+## @function filenameConcatenate()
+#
+sub filenameConcatenate
+{
+  my $first_file = shift(@_);
+  my (@filenames) = @_;
+
+  #   Useful for debugging
+  #     -- might make sense to call caller(0) rather than (1)??
+  #   my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
+  #   print STDERR "Calling method: $cfilename:$cline $cpackage->$csubr\n";
+
+  # If first_file is not null or empty, then add it back into the list
+  if (defined $first_file && $first_file =~ /\S/) {
+    unshift(@filenames, $first_file);
+  }
+
+  my $filename = join("/", @filenames);
+
+  # remove duplicate slashes and remove the last slash
+  if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+    $filename =~ s/[\\\/]+/\\/g;
+  } else {
+    $filename =~ s/[\/]+/\//g;
+    # DB: want a filename abc\de.html to remain like this
+  }
+  $filename =~ s/[\\\/]$//;
+
+  return $filename;
+}
+## filenameConcatenate()
+
+
+## @function fileSize()
+#
+sub fileSize
+{
+  my ($filename_full_path) = @_;
+  return -s $filename_full_path;
+}
+## fileStatus()
+
+
+## @function fileTest()
+#
+sub fileTest
+{
+  my $filename_full_path = shift(@_);
+  # By default tests for existance of file or directory (-e)
+  # Can be made more specific by providing second parameter (e.g. -f or -d)
+  my $test_op = shift(@_) || '-e';
+
+  my $exists = 0;
+
+  if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
+    require Win32;
+    my $filename_short_path = Win32::GetShortPathName($filename_full_path);
+    if (!defined $filename_short_path) {
+      # Was probably still in UTF8 form (not what is needed on Windows)
+      my $unicode_filename_full_path = eval "decode(\"utf8\",\$filename_full_path)";
+      if (defined $unicode_filename_full_path) {
+        $filename_short_path = Win32::GetShortPathName($unicode_filename_full_path);
+      }
+    }
+    $filename_full_path = $filename_short_path;
+  }
+
+  if (defined $filename_full_path) {
+    $exists = eval "($test_op \$filename_full_path)";
+  }
+
+  # The eval may result in exists being undefined, but we need to return
+  # something
+  return ($exists || 0);
+}
+## fileTest()
+
+
+## @function linkFile()
+#
+sub linkFile
+{
+  my ($mode, $src, $dest, $ensure_paths_absolute) = @_;
+
+  # remove trailing slashes from source and destination files
+  $src =~ s/[\\\/]+$//;
+  $dest =~ s/[\\\/]+$//;
+
+  # Ensure file paths are absolute IF requested to do so
+  # Soft_linking didn't work for relative paths
+  if($mode eq 'HARD' || (defined $ensure_paths_absolute && $ensure_paths_absolute))
+  {
+    # We need to ensure that the src file is the absolute path
+    # See http://perldoc.perl.org/File/Spec.html
+    if(!File::Spec->file_name_is_absolute( $src ))
+    {
+      # it's relative
+      $src = File::Spec->rel2abs($src); # make absolute
+    }
+    # Might as well ensure that the destination file's absolute path is used
+    if(!File::Spec->file_name_is_absolute( $dest ))
+    {
+      $dest = File::Spec->rel2abs($dest); # make absolute
+    }
+  }
+
+  # a few sanity checks
+  if (!-e $src)
+  {
+    &FileUtils::printError('Source file does not exist: ' . $src);
+    return 0;
+  }
+
+  my $dest_dir = &File::Basename::dirname($dest);
+  if (!-e $dest_dir)
+  {
+    &makeAllDirectories($dest_dir);
+  }
+
+  if ($ENV{'GSDLOS'} =~ /^windows$/i)
+  {
+    # symlink not supported on windows
+    &FileUtils::printWarning('Symlink not supported on windows');
+  }
+  elsif ($mode eq 'HARD')
+  {
+    if (!eval {link($src, $dest)})
+    {
+      &FileUtils::printWarning('Unable to create hard link: ' . $dest);
+    }
+  }
+  elsif ($mode eq 'SOFT')
+  {
+    if (!eval {symlink($src, $dest)})
+    {
+      &FileUtils::printWarning('Unable to create soft link: ' . $src);
+    }
+  }
+  else
+  {
+    &FileUtils::printError('Unknown mode requested: ' . $mode);
+  }
+  if (!-e $dest)
+  {
+    &_printWarning('linkFile', 'Link failed. Attempting to copy instead.');
+    &File::Copy::copy ($src, $dest);
+  }
+  return (-e $dest);
+}
+# /** linkFile() **/
+
+## @function makeAllDirectories()
+#
+# in case anyone cares - I did some testing (using perls Benchmark module)
+# on this subroutine against File::Path::mkpath (). mk_all_dir() is apparently
+# slightly faster (surprisingly) - Stefan.
+#
+sub makeAllDirectories
+{
+  my ($dir) = @_;
+
+  # use / for the directory separator, remove duplicate and
+  # trailing slashes
+  $dir=~s/[\\\/]+/\//g;
+  $dir=~s/[\\\/]+$//;
+
+  # ensure the directory doesn't already exist
+  if (-e $dir)
+  {
+    return 0;
+  }
+
+  # make sure the cache directory exists
+  my $dirsofar = "";
+  my $first = 1;
+  foreach my $dirname (split ("/", $dir))
+  {
+    $dirsofar .= "/" unless $first;
+    $first = 0;
+
+    $dirsofar .= $dirname;
+
+    next if $dirname =~ /^(|[a-z]:)$/i;
+    if (!-e $dirsofar)
+    {
+      my $store_umask = umask(0002);
+      my $mkdir_ok = mkdir ($dirsofar, 0777);
+      umask($store_umask);
+      if (!$mkdir_ok)
+      {
+        &FileUtils::printError('Could not create directory: ' . $dirsofar);
+        return 0;
+      }
+    }
+  }
+  return (-e $dir);
+}
+## makeAllDirectories()
+
+
+## @function makeDirectory()
+#
+sub makeDirectory
+{
+  my $dir = shift(@_);
+  my $store_umask = umask(0002);
+  my $mkdir_ok = mkdir ($dir, 0777);
+  umask($store_umask);
+  return $mkdir_ok;
+}
+## makeDirectory()
+
+
+## @function modificationTime()
+#
+sub modificationTime
+{
+  my $path = shift(@_);
+  my @file_status = stat($path);
+  return $file_status[9];
+}
+## modificationTime()
+
+## @function openFileHandle()
+#
+sub openFileHandle
+{
+  my $path = shift(@_);
+  my $mode = shift(@_);
+  my $fh_ref = shift(@_);
+  my $encoding = shift(@_);
+  my $mode_symbol;
+  if ($mode eq 'w' || $mode eq '>')
+  {
+    $mode_symbol = '>';
+    $mode = 'writing';
+  }
+  elsif ($mode eq 'a' || $mode eq '>>')
+  {
+    $mode_symbol = '>>';
+    $mode = 'appending';
+  }
+  else
+  {
+    $mode_symbol = '<';
+    $mode = 'reading';
+  }
+  if (defined $encoding)
+  {
+    $mode_symbol .= ':' . $encoding;
+  }
+  if (!open($$fh_ref, $mode_symbol, $path))
+  {
+    &FileUtils::printError('Failed to open file for ' . $mode . ': ' . $path, 1);
+  }
+  return 1;
+}
+## openFileHandle()
+
+# /**
+#  */
+sub readDirectory
+{
+  my $path = shift(@_);
+  opendir(DH, $path) or &FileUtils::printError('Failed to open directory for reading: ' . $path, 1);
+  my @files = readdir(DH);
+  close(DH);
+  return \@files;
+}
+# /** readDirectory() **/
+
+
+## @function removeFiles()
+#
+sub removeFiles
+{
+  my $file = shift(@_);
+  my $result = 0;
+  if (!-e $file && !-l $file)
+  {
+    &FileUtils::printError('File does not exist: ' . $file);
+  }
+  elsif ((!-f $file) && (!-l $file))
+  {
+    &FileUtils::printError('Not a regular file: ' . $file);
+  }
+  else
+  {
+    $result = unlink($file);
+    if (!$result)
+    {
+      &FileUtils::printError('Failed to remove file: ' . $file);
+    }
+  }
+  return $result;
+}
+## removeFiles()
+
+
+## @function removeFilesFiltered()
+#
+# recursive removal
+#
+sub removeFilesFiltered
+{
+  my ($files, $file_accept_re, $file_reject_re) = @_;
+  #   my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(2);
+  #   my ($lcfilename) = ($cfilename =~ m/([^\\\/]*)$/);
+  #   print STDERR "** Calling method (2): $lcfilename:$cline $cpackage->$csubr\n";
+  my @files_array = (ref $files eq "ARRAY") ? @$files : ($files);
+
+  my $num_removed = 0;
+
+  foreach my $file (@files_array)
+  {
+    # remove trailing slashes
+    $file =~ s/[\/\\]+$//;
+
+    if (!-e $file)
+    {
+      print STDERR "util::filtered_rm_r $file does not exist\n";
+    }
+    # don't recurse down symbolic link
+    elsif ((-d $file) && (!-l $file))
+    {
+      # get the contents of this directory
+      if (!opendir (INDIR, $file))
+      {
+        print STDERR "util::filtered_rm_r could not open directory $file\n";
+      }
+      else
+      {
+        my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+        closedir (INDIR);
+
+        # remove all the files in this directory
+        map {$_="$file/$_";} @filedir;
+        $num_removed += &FileUtils::LocalFS::removeFilesFiltered(\@filedir, $file_accept_re, $file_reject_re);
+
+        if (!defined $file_accept_re && !defined $file_reject_re)
+        {
+          # remove this directory
+          if (!rmdir $file)
+          {
+            print STDERR "util::filtered_rm_r couldn't remove directory $file\n";
+          }
+          else
+          {
+            $num_removed++;
+          }
+        }
+      }
+    }
+    else
+    {
+      next if (defined $file_reject_re && ($file =~ m/$file_reject_re/));
+
+      if ((!defined $file_accept_re) || ($file =~ m/$file_accept_re/))
+      {
+        # remove this file
+        $num_removed += &removeFiles($file);
+      }
+    }
+  }
+  return $num_removed;
+}
+## removeFilesFiltered()
+
+
+## @function removeFilesRecursive()
+#
+sub removeFilesRecursive
+{
+  my $path = shift(@_);
+  # use the more general (but reterospectively written filteredRemove()
+  # function with no accept or reject expressions
+  return FileUtils::LocalFS::removeFilesFiltered($path, undef, undef);
+}
+## removeFilesRecursive()
+
+
+## @function supportsSymbolicLink
+#
+sub supportsSymbolicLink
+{
+  return 1;
+}
+## supportsSymbolicLink()
+
+
+## @function synchronizeDirectory()
+#
+# updates a copy of a directory in some other part of the filesystem
+# verbosity settings are: 0=low, 1=normal, 2=high
+# both $fromdir and $todir should be absolute paths
+#
+sub synchronizeDirectory
+{
+  my ($fromdir, $todir, $verbosity) = @_;
+  $verbosity = 1 unless defined $verbosity;
+
+  # use / for the directory separator, remove duplicate and
+  # trailing slashes
+  $fromdir=~s/[\\\/]+/\//g;
+  $fromdir=~s/[\\\/]+$//;
+  $todir=~s/[\\\/]+/\//g;
+  $todir=~s/[\\\/]+$//;
+
+  &mk_all_dir ($todir);
+
+  # get the directories in ascending order
+  if (!opendir (FROMDIR, $fromdir))
+  {
+    print STDERR "util::cachedir could not read directory $fromdir\n";
+    return;
+  }
+  my @fromdir = grep (!/^\.\.?$/, sort(readdir (FROMDIR)));
+  closedir (FROMDIR);
+
+  if (!opendir (TODIR, $todir))
+  {
+    print STDERR "util::cacedir could not read directory $todir\n";
+    return;
+  }
+  my @todir = grep (!/^\.\.?$/, sort(readdir (TODIR)));
+  closedir (TODIR);
+
+  my $fromi = 0;
+  my $toi = 0;
+
+  while ($fromi < scalar(@fromdir) || $toi < scalar(@todir))
+  {
+    #	print "fromi: $fromi toi: $toi\n";
+
+    # see if we should delete a file/directory
+    # this should happen if the file/directory
+    # is not in the from list or if its a different
+    # size, or has an older timestamp
+    if ($toi < scalar(@todir))
+    {
+      if (($fromi >= scalar(@fromdir)) ||
+          ($todir[$toi] lt $fromdir[$fromi] || 
+           ($todir[$toi] eq $fromdir[$fromi] && 
+            &differentfiles("$fromdir/$fromdir[$fromi]","$todir/$todir[$toi]",
+                            $verbosity))))
+      {
+        # the files are different
+        &rm_r("$todir/$todir[$toi]");
+        splice(@todir, $toi, 1); # $toi stays the same
+      }
+      elsif ($todir[$toi] eq $fromdir[$fromi])
+      {
+        # the files are the same
+        # if it is a directory, check its contents
+        if (-d "$todir/$todir[$toi]")
+        {
+          &synchronizeDirectory("$fromdir/$fromdir[$fromi]", "$todir/$todir[$toi]", $verbosity);
+        }
+
+        $toi++;
+        $fromi++;
+        next;
+      }
+    }
+
+    # see if we should insert a file/directory
+    # we should insert a file/directory if there
+    # is no tofiles left or if the tofile does not exist
+    if ($fromi < scalar(@fromdir) && ($toi >= scalar(@todir) ||
+                                      $todir[$toi] gt $fromdir[$fromi]))
+    {
+      &cp_r ("$fromdir/$fromdir[$fromi]", "$todir/$fromdir[$fromi]");
+      splice (@todir, $toi, 0, $fromdir[$fromi]);
+      $toi++;
+      $fromi++;
+    }
+  }
+}
+## synchronizeDirectory()
+
+
+# /**
+#  */
+sub transferFile
+{
+  my ($mode, $file, $dest) = @_;
+  # remove trailing slashes from source and destination files
+  $file =~ s/[\\\/]+$//;
+  $dest =~ s/[\\\/]+$//;
+  my $tempdest = $dest;
+  if (!-e $file)
+  {
+    &FileUtils::printError('File does not exist: ' . $file);
+  }
+  else
+  {
+    if (-d $tempdest)
+    {
+      my ($filename) = $file =~ /([^\\\/]+)$/;
+      $tempdest .= "/$filename";
+    }
+    # start by processing any move request
+    if ($mode eq 'MOVE')
+    {
+      &File::Copy::move($file, $tempdest);
+    }
+    # now if we were instead doing a copy, or if the move request above failed
+    # for some reason, we process a copy request
+    if ($mode eq 'COPY' || !-e $tempdest)
+    {
+      &File::Copy::copy($file, $tempdest);
+    }
+    # finally, we check if a successful move command has somehow left the origin
+    # file lying around (rename partially succeeded - for instance when moving
+    # hardlinks)
+    if ($mode eq 'MOVE' && -e $tempdest && -e $file)
+    {
+      unlink($file);
+    }
+  }
+  # Have we successfully moved the file?
+  my $result = 0;
+  if (-e $tempdest)
+  {
+    if ($mode eq 'MOVE')
+    {
+      if (-e $file)
+      {
+        &FileUtils::printError('Failed to remove original file during move: ' . $file);
+      }
+      else
+      {
+        $result = 1;
+      }
+    }
+    else
+    {
+      $result = 1;
+    }
+  }
+  else
+  {
+    &FileUtils::printError('Failed to move/copy file: ' . $file);
+  }
+  return $result;
+}
+# /** moveFile() **/
+
+1;
