Index: /gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl	(revision 27436)
+++ /gs2-extensions/parallel-building/trunk/src/bin/script/test_fileutils.pl	(revision 27436)
@@ -0,0 +1,775 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+no warnings 'once';
+
+# Configure
+my $user = 'jmt12';
+my $hadoop_path = '/mnt/sdb1/jmt12/gsdl2-svn/ext/parallel-building/packages/hadoop-1.1.0';
+my $redirect_errors = 0;
+my $display_errors = 0;
+
+my $test_localfs = 0;
+my $test_hdthriftfs = 1;
+my $test_hdfsshell = 0;
+
+# Globals
+my $base_path;
+my $test_count;
+my $pass_count;
+my $skip_count;
+my $start_time;
+my $total_time;
+my $error_messages;
+
+my $sample_str1 = "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet,\nconsectetur,\nadipisci velit...\n";
+# This was going to be used for append (since Thrift has an append function)
+# but it turns out append is dependant upon HDFS being configured to allow
+# append - and mine isn't. Left here for those whom don't speak latin!
+my $sample_str2 = "[Translation]\nThere is no one who loves pain itself,\nwho seeks after it and wants to have it,\nsimply because it is pain...";
+
+# Requires setup.bash to have been sourced
+BEGIN
+{
+  die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+  die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+  die "Not supported under windows\n" unless $ENV{'GSDLOS'} !~ /^windows$/i;
+  # Ensure Greenstone Perl locations are in INC
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
+  if (defined $ENV{'GSDLEXTS'})
+  {
+    my @extensions = split(/:/, $ENV{'GSDLEXTS'});
+    foreach my $e (@extensions)
+    {
+      my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
+      unshift (@INC, $ext_prefix . '/perllib');
+      unshift (@INC, $ext_prefix . '/perllib/cpan');
+    }
+  }
+}
+
+use Cwd;
+use DateTime;
+use Devel::Peek;
+
+use FileUtils;
+
+# populate globals
+$base_path = getcwd();
+
+print "\n";
+print "============================= Test FileUtils ============================\n";
+print "Test the FileUtils module to ensure it correctly manages local and HDFS\n";
+print "based files. HDFS support is provided by two different drivers: HDFSShell\n";
+print "uses repeated calls to the CLI Hadoop program, while HDThriftFS creates a\n";
+print "client socket connection to a running Thrift server.\n";
+print "=========================================================================\n";
+print "\n";
+
+if ($redirect_errors)
+{
+  open SAVEERR, ">&STDERR";
+  close STDERR;
+  open STDERR, ">", \$error_messages or die "What the hell?\n";
+}
+
+if ($test_localfs)
+{
+  $start_time = time();
+  print "A. Local File System\n";
+  $test_count = 0;
+  $pass_count = 0;
+  $skip_count = 0;
+  my $test_dir = '/tmp/fileutils';
+  # From this test on you are safe to assume the testing directory exists
+  &testMakeDirectory($test_dir);
+  &testFilePutContents($test_dir); # creates alpha.txt
+  &testFileGetContents($test_dir); # removes alpha.txt
+  &testFileExists($test_dir);
+  &testDirectoryExists($test_dir);
+  &testFilePermissions($test_dir);
+  &testFilenameConcatenate('/home/gsdl/collect/test/etc/collect.cfg', # target path
+                           '/', 'home','gsdl','collect','test','etc','collect.cfg', #parts
+                          );
+  &testOpenFileHandle($test_dir);
+  &testFileSize($test_dir);
+  &testModificationTime($test_dir);
+  &testDifferentFiles($test_dir);
+  &testReadDirectory($test_dir);
+  &testTransferFiles($test_dir, 'move');
+  &testTransferFiles($test_dir, 'copy');
+  &testLinkFile($test_dir, $base_path, 'soft');
+  &testLinkFile($test_dir, $base_path, 'hard');
+  $skip_count += &skipAction(\$test_count, "Binary file transfer test (roundtrip)");
+  &testRemoveFiles($test_dir);
+  &testRemoveFilesFiltered($test_dir);
+  &testRemoveFilesRecursive($test_dir);
+  print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
+  $total_time = time() - $start_time;
+  print "Time: " . $total_time . " seconds\n\n";
+  # Full cleanup before we start the next round of tests
+  &FileUtils::removeFilesRecursive($test_dir);
+}
+
+if ($test_hdthriftfs)
+{
+  $start_time = time();
+  print "B. HDFS using Thrift\n";
+  my $thrift_fileutils_path = 'HDThriftFS:///user/' . $user. '/fileutils';
+  $test_count = 0;
+  $pass_count = 0;
+  $skip_count = 0;
+  &testFilenameConcatenate('HDThriftFS:///home/gsdl/collect/test/etc/collect.cfg', # target path
+                           'HDThriftFS://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
+                          );
+  &testMakeDirectory($thrift_fileutils_path);
+  &testFilePutContents($thrift_fileutils_path); # Leaves alpha.txt
+  &testFileGetContents($thrift_fileutils_path); # Removes alpha.txt
+  &testFileExists($thrift_fileutils_path);
+  &testDirectoryExists($thrift_fileutils_path);
+  &testFilePermissions($thrift_fileutils_path);
+  &testOpenFileHandle($thrift_fileutils_path);
+  &testFileSize($thrift_fileutils_path);
+  &testModificationTime($thrift_fileutils_path);
+  &testDifferentFiles($thrift_fileutils_path);
+  &testReadDirectory($thrift_fileutils_path);
+  &testTransferFiles($thrift_fileutils_path, 'move', '/tmp');
+  &testTransferFiles($thrift_fileutils_path, 'copy', '/tmp');
+  &testLinkFile($thrift_fileutils_path, '', 'soft');
+  &testLinkFile($thrift_fileutils_path, '', 'hard');
+  &testBinaryTransfer($thrift_fileutils_path, $base_path);
+  &testRemoveFiles($thrift_fileutils_path);
+  &testRemoveFilesFiltered($thrift_fileutils_path);
+  &testRemoveFilesRecursive($thrift_fileutils_path);
+  print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
+  $total_time = time() - $start_time;
+  print "Time: " . $total_time . " seconds\n\n";
+  # Cleanup
+  &FileUtils::removeFilesRecursive($thrift_fileutils_path);
+}
+
+if ($test_hdfsshell)
+{
+  $start_time = time();
+  print "C. HDFS using Shell\n";
+  my $hdfs_fileutils_path = 'HDFSShell:///user/' . $user. '/fileutils';
+  $test_count = 0;
+  $pass_count = 0;
+  $skip_count = 0;
+  &testFilenameConcatenate('HDFSShell:///home/gsdl/collect/test/etc/collect.cfg', # target path
+                           'HDFSShell://', 'home','gsdl','collect','test','etc','collect.cfg', #parts
+                          );
+  &testMakeDirectory($hdfs_fileutils_path);
+  &testFilePutContents($hdfs_fileutils_path); # Leaves alpha.txt
+  &testFileGetContents($hdfs_fileutils_path); # Removes alpha.txt
+  &testFileExists($hdfs_fileutils_path);
+  &testDirectoryExists($hdfs_fileutils_path);
+  &testFilePermissions($hdfs_fileutils_path);
+  &testOpenFileHandle($hdfs_fileutils_path);
+  &testFileSize($hdfs_fileutils_path);
+  &testModificationTime($hdfs_fileutils_path);
+  &testDifferentFiles($hdfs_fileutils_path);
+  &testReadDirectory($hdfs_fileutils_path);
+  &testTransferFiles($hdfs_fileutils_path, 'move', '/tmp');
+  &testTransferFiles($hdfs_fileutils_path, 'copy', '/tmp');
+  &testLinkFile($hdfs_fileutils_path, '', 'soft');
+  &testLinkFile($hdfs_fileutils_path, '', 'hard');
+  &testBinaryTransfer($hdfs_fileutils_path, $base_path);
+  &testRemoveFiles($hdfs_fileutils_path);
+  &testRemoveFilesFiltered($hdfs_fileutils_path);
+  &testRemoveFilesRecursive($hdfs_fileutils_path);
+  print "Result: Passed " . $pass_count . "/" . ($test_count - $skip_count) . " (skipped " . $skip_count . ")\n";
+  $total_time = time() - $start_time;
+  print "Time: " . $total_time . " seconds\n\n";
+  # Cleanup
+  &FileUtils::removeFilesRecursive($hdfs_fileutils_path);
+}
+
+if ($redirect_errors)
+{
+  close STDERR;
+  open STDERR, ">&SAVEERR";
+  if ($display_errors)
+  {
+    print "Error Messages\n";
+    print $error_messages;
+    print "\n\n";
+  }
+}
+
+exit;
+
+# /** A test that doesn't actually test anything, but always skips. Used as a
+#  *  placeholder where certain tests aren't applicable.
+#  */
+sub skipAction
+{
+  my ($test_count_ref, $description) = @_;
+  $$test_count_ref++;
+  printf("% 2d. %s: Skipped\n", $$test_count_ref, $description);
+  return 1;
+}
+
+sub subtestAction
+{
+  my ($expected_result, $actual_result, $debug) = @_;
+  if (defined $debug && $debug)
+  {
+    print "subtestAction(expected:$expected_result, actual:$actual_result)\n";
+  }
+  my $result = 'Fail';
+  if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
+  {
+    $result = 'Pass';
+  }
+  return ('Pass' eq $result);
+}
+
+sub testAction
+{
+  my ($test_count_ref, $description, $expected_result, $actual_result) = @_;
+  $$test_count_ref++;
+  my $result = 'Fail';
+  if (defined $actual_result && (('' . $expected_result) eq ('' . $actual_result)))
+  {
+    $result = 'Pass';
+  }
+  else
+  {
+    $result .= ' (' . $expected_result . ' != ' . $actual_result . ')';
+  }
+  printf("% 2d. %s: %s\n", $$test_count_ref, $description, $result);
+  return ('Pass' eq $result);
+}
+
+################################################################################
+##### TESTS
+################################################################################
+
+## @function testBinaryTransfer()
+#
+sub testBinaryTransfer
+{
+  my ($dir, $base_dir) = @_;
+  # setup
+  my $local_image_path = $base_path . '/../../resources/aurora_australus.jpg';
+  my $thrift_copy_path = $dir . '/test_binary_transfer.jpg';
+  my $local_copy_path = '/tmp/test_binary_transfer.jpg';
+  # actions - roundtrip copy of files
+  &FileUtils::copyFiles($local_image_path, $thrift_copy_path);
+  &FileUtils::copyFiles($thrift_copy_path, $local_copy_path);
+  # test
+  my $result = `diff $local_image_path $local_copy_path`;
+  $pass_count += &testAction(\$test_count, "Binary file transfer test (roundtrip)", '', $result);
+  # cleanup
+  &FileUtils::removeFiles($thrift_copy_path, $local_copy_path);
+}
+## testBinaryTransfer()
+
+
+## @function testDifferentFiles
+#
+sub testDifferentFiles
+{
+  my ($dir) = @_;
+  my $patha = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($patha, 'alpha');
+  my $pathb = $dir . '/beta.txt';
+  &FileUtils::filePutContents($pathb, 'beta');
+  $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to itself", 0, &FileUtils::differentFiles($patha, $patha));
+  $pass_count += &testAction(\$test_count, "differentFiles() comparing a directory to itself", 0, &FileUtils::differentFiles($dir, $dir));
+  $pass_count += &testAction(\$test_count, "differentFiles() comparing different files", 1, &FileUtils::differentFiles($patha, $pathb));
+  $pass_count += &testAction(\$test_count, "differentFiles() comparing a file to a directory", 1, &FileUtils::differentFiles($patha, $dir));
+  # -cleanup
+  &FileUtils::removeFiles($patha, $pathb);
+}
+# /** testDifferentFiles() **/
+
+sub testDirectoryExists
+{
+  my ($patha, $pathb, $pathc, $pathd) = @_;
+  $pass_count += &testAction(\$test_count, "directoryExists() for existing directory", 1, &FileUtils::directoryExists($patha));
+  $pass_count += &testAction(\$test_count, "directoryExists() for existing file (should fail)", 0, &FileUtils::directoryExists($pathb));
+  $pass_count += &testAction(\$test_count, "directoryExists() for directory that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathc));
+  $pass_count += &testAction(\$test_count, "directoryExists() for file that doesn't exist (should fail)", 0, &FileUtils::directoryExists($pathd));
+}
+
+
+## @function testFileExists()
+#
+sub testFileExists
+{
+  my ($dir) = @_;
+  my $alpha_path = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($alpha_path, 'alpha');
+  $pass_count += &testAction(\$test_count, "fileExists() for existing file", 1, &FileUtils::fileExists($alpha_path));
+  my $beta_path = $dir . '/beta.txt';
+  $pass_count += &testAction(\$test_count, "fileExists() for file that doesn't exist (should fail)", 0, &FileUtils::fileExists($beta_path));
+}
+## testFileExists()
+
+
+## @function testFileGetContents()
+#
+sub testFileGetContents
+{
+  my ($dir) = @_;
+  my $apath = $dir . '/alpha.txt';
+  $pass_count += &testAction(\$test_count, "fileGetContents() to read an entire file as a string", $sample_str1, &FileUtils::fileGetContents($apath));
+  &FileUtils::removeFiles($apath); # just have to hope this works
+}
+## testFileGetContents()
+
+
+## @function testFilePermissions()
+#
+sub testFilePermissions
+{
+  my ($dir) = @_;
+  my $patha = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($patha, 'alpha');
+  $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file I own", 1, &FileUtils::canRead($patha));
+  $pass_count += &testAction(\$test_count, "canRead() testing read permission on a file in my group", 1, &FileUtils::canRead($patha, 'johnsmith', 'supergroup'));
+  $pass_count += &testAction(\$test_count, "canRead() testing global access read permission on a file", 1, &FileUtils::canRead($patha, 'johnsmith', 'othergroup'));
+
+  $skip_count += &skipAction(\$test_count, "canWrite() testing write permission on a file");
+  $skip_count += &skipAction(\$test_count, "canExecute() testing execute permission on a file");
+  # - cleanup
+  &FileUtils::removeFiles($patha);
+}
+## testFilePermissions()
+
+
+## @function testFilePutContents()
+#
+sub testFilePutContents
+{
+  my ($dir) = @_;
+  my $apath = $dir . '/alpha.txt';
+  $pass_count += &testAction(\$test_count, "filePutContents() to write a string direct to a file", 1, &FileUtils::filePutContents($apath, $sample_str1));
+}
+## testFilePutContents()
+
+
+## @function testFileSize()
+#
+sub testFileSize
+{
+  my ($dir) = @_;
+  my $apath = $dir . '/alpha.txt';
+  my $astr = 'alpha';
+  &FileUtils::filePutContents($apath, $astr);
+  $pass_count += &testAction(\$test_count, "fileSize() to determine a file's size in bytes", length($astr), &FileUtils::fileSize($apath));
+  &FileUtils::removeFiles($apath);
+}
+## testFileSize()
+
+
+sub testFilenameConcatenate
+{
+  my $target = shift(@_);
+  $pass_count += &testAction(\$test_count, "filenameConcatenate() to combine file paths", $target, &FileUtils::filenameConcatenate(@_));
+}
+
+sub testLinkFile
+{
+  my ($dir, $base_path, $mode) = @_;
+  if (!&FileUtils::supportsSymbolicLink($dir))
+  {
+    &FileUtils::printWarning('Symbolic links not supported so files will be copied instead');
+  }
+
+  # - init
+  my $sub_pass_count = 0;
+  my $patha = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($patha, 'alpha');
+  my $pathb = $dir . '/beta.txt';
+  # - subtests
+  if ($mode eq 'soft')
+  {
+    $sub_pass_count += &subtestAction(1, &FileUtils::softLink($patha, $pathb));
+    # If the driver doesn't support symbolic links, then the following function
+    # will actually end up triggering a '-e' - which is true even for hardlinks
+    $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
+  }
+  else
+  {
+    $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($patha, $pathb));
+    if (&FileUtils::supportsSymbolicLink($pathb))
+    {
+      $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
+    }
+    # If the driver doesn't support symbolic links, then the following function
+    # will actually end up triggering a '-e' - which is true even for hardlinks
+    else
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
+    }
+  }
+  $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
+  $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
+  $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
+  # - test
+  $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file using relative path", 5, $sub_pass_count);
+  # - cleanup
+  &FileUtils::removeFiles($pathb);
+
+  # - init
+  if ($base_path ne '')
+  {
+    my $depth = split(/[\/\\]/, $base_path) - 1;
+    my $rel_prefix = '..';
+    for (my $i = 1; $i < $depth; $i++)
+    {
+      $rel_prefix .= '/..';
+    }
+    my $rel_patha = $rel_prefix . $patha;
+    my $rel_pathb = $rel_prefix . $pathb;
+    $sub_pass_count = 0;
+    # - subtests
+    if ($mode eq 'soft')
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::softLink($rel_patha, $rel_pathb));
+      $sub_pass_count += &subtestAction(1, &FileUtils::isSymbolicLink($pathb));
+    }
+    else
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::hardLink($rel_patha, $rel_pathb));
+      $sub_pass_count += &subtestAction(0, &FileUtils::isSymbolicLink($pathb));
+    }
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($patha));
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($pathb));
+    $sub_pass_count += &subtestAction('alpha', &FileUtils::fileGetContents($pathb));
+    # - test
+    $pass_count += &testAction(\$test_count, $mode . "Link() " . $mode . " symbolic linking a single file forcing absolute path", 5, $sub_pass_count);
+    # - cleanup
+    &FileUtils::removeFiles($pathb, $patha);
+  }
+  else
+  {
+    $skip_count += &skipAction(\$test_count, $mode . "Link() relative paths not supported in this filesystem");
+  }
+}
+
+sub testMakeDirectory
+{
+  my ($path) = @_;
+  $pass_count += &testAction(\$test_count, 'makeDirectory() to create a new directory', 1, &FileUtils::makeDirectory($path));
+  $pass_count += &testAction(\$test_count, 'makeDirectory() for an existing directory', 1, &FileUtils::makeDirectory($path));
+}
+
+sub testModificationTime
+{
+  my ($dir) = @_;
+  my $patha = $dir . '/alpha.txt';
+  my $exact_modification_time = time();
+  #rint STDERR "Exact Modification Time:  $exact_modification_time\n";
+  &FileUtils::filePutContents($patha, 'Hello World');
+  my $file_modification_time = &FileUtils::modificationTime($patha);
+  #rint STDERR "File Modification Time:   $file_modification_time\n";
+  # Try to find an exact match first
+  if ($exact_modification_time == $file_modification_time)
+  {
+    $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (exact)', 1, 1);
+  }
+  # Failing that, we can try a rounded match (for file systems that
+  # don't store modification times down to the second i.e. HDFSShell)
+  else
+  {
+    $skip_count += &skipAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (exact)');
+    my ($mod_second, $mod_minute, $mod_hour, $mod_day, $mod_month, $mod_year) = localtime($exact_modification_time);
+    $mod_year += 1900; # initially years since 1900, silly computer scientists
+    $mod_month += 1; # in the range [0..11], silly computer scientists
+    my $mod_datetime = DateTime->new(year      => $mod_year,
+                                     month     => $mod_month,
+                                     day       => $mod_day,
+                                     hour      => $mod_hour,
+                                     minute    => $mod_minute,
+                                     second    => 0,
+                                     time_zone => 'local');
+    my $start_modification_time = $mod_datetime->epoch();
+    #rint STDERR "Start Modification Time: $start_modification_time\n";
+    if ($start_modification_time == $file_modification_time)
+    {
+      $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (rounded)', 1, 1);
+    }
+    # And finally we just accept any time that lies within the same minute
+    else
+    {
+      $skip_count += &skipAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (rounded)');
+      $mod_datetime->add( minutes => 1 );
+      my $end_modification_time = $mod_datetime->epoch();
+      #rint STDERR "End Modification Time: $end_modification_time\n";
+      $pass_count += &testAction(\$test_count, 'modificationTime() to determine the last modified time as a linux epoc (approximate)', 1, ($start_modification_time <= $file_modification_time && $file_modification_time <= $end_modification_time ? 1 : 0));
+    }
+  }
+  # cleanup
+  &FileUtils::removeFiles($patha);
+}
+
+sub testOpenFileHandle
+{
+  my ($dir) = @_;
+  my $path = $dir . '/alpha.txt';
+  my $dest_str;
+  my $file_handle;
+  # - writing
+  $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for writing", 1, &FileUtils::openFileHandle($path, 'w', \$file_handle));
+  print $file_handle $sample_str1;
+  $pass_count += &testAction(\$test_count, "closeFileHandle() to close the writable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
+  # - reading
+  $pass_count += &testAction(\$test_count, "openFileHandle() to open a file handle for reading", 1, &FileUtils::openFileHandle($path, 'r', \$file_handle));
+  while (my $line = <$file_handle>)
+  {
+    $dest_str .= $line;
+  }
+  $pass_count += &testAction(\$test_count, "comparing string written to string read", $sample_str1, $dest_str);
+  $pass_count += &testAction(\$test_count, "closeFileHandle() to close the readable file handle", 1, &FileUtils::closeFileHandle($path, \$file_handle));
+  # - cleanup
+  &FileUtils::removeFiles($path);
+}
+
+sub testReadDirectory
+{
+  my ($dir) = @_;
+  my $apath = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($apath, 'alpha');
+  my $bpath = $dir . '/beta.txt';
+  &FileUtils::filePutContents($bpath, 'beta');
+  my $dpath = $dir . '/delta.txt';
+  &FileUtils::filePutContents($dpath, 'delta');
+  my $file_count = 3;
+  # - read the directory, which will only contain the files we just put there
+  my @files = @{&FileUtils::readDirectory($dir)};
+  my $result_count = 0;
+  foreach my $file (@files)
+  {
+    if ($file !~ /^\./)
+    {
+      $result_count++;
+    }
+  }
+  $pass_count += &testAction(\$test_count, "readDirectory() listing files", $file_count, $result_count);
+  &FileUtils::removeFiles($apath, $bpath, $dpath);
+}
+
+## @function testRemoveFiles()
+#
+sub testRemoveFiles
+{
+  my ($dir) = @_;
+
+  my $patha = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($patha, 'alpha');
+  my $pathb = $dir . '/beta.txt';
+  &FileUtils::softLink($patha, $pathb);
+  my $pathd = $dir . '/delta.txt';
+  &FileUtils::filePutContents($pathd, 'delta');
+  my $pathe = $dir . '/epsilon.txt';
+  &FileUtils::filePutContents($pathe, 'epsilon');
+  my $pathg = $dir . '/gamma.txt';
+  &FileUtils::filePutContents($pathg, 'gamma');
+  my $patht = $dir . '/tau.txt';
+  my $dirm = $dir .'/mu';
+  &FileUtils::makeDirectory($dirm);
+  # - tests
+  $pass_count += &testAction(\$test_count, "removeFiles() for existing file", 1, &FileUtils::removeFiles($patha));
+  $pass_count += &testAction(\$test_count, "removeFiles() for soft symbolic link", 1, &FileUtils::removeFiles($pathb));
+  $pass_count += &testAction(\$test_count, "removeFiles() for multiple existing files", 3, &FileUtils::removeFiles($pathd, $pathe, $pathg));
+  $pass_count += &testAction(\$test_count, "removeFiles() for file that doesn't exist (should fail)", 0, &FileUtils::removeFiles($patht));
+  $pass_count += &testAction(\$test_count, "removeFiles() for existing directory (should fail)", 0, &FileUtils::removeFiles($dirm));
+  # - cleanup
+  &FileUtils::removeFilesRecursive($dirm);
+}
+## testRemoveFiles()
+
+
+## @function testRemoveFilesFiltered()
+#
+sub testRemoveFilesFiltered
+{
+  my ($dir) = @_;
+  # - setup
+  my $patha = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($patha, 'alpha');
+  &FileUtils::filePutContents($dir . '/beta.tmp','beta');
+  my $pathd = $dir . '/delta.txt';
+  &FileUtils::filePutContents($pathd, 'delta');
+  &FileUtils::filePutContents($dir . '/gamma.tmp','gamma');
+  &FileUtils::filePutContents($dir . '/epsilon.txt','epsilon');
+  &FileUtils::filePutContents($dir . '/eta.txt','eta');
+  # - tests
+  $pass_count += &testAction(\$test_count, "filteredRemove() with accept regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.tmp$', undef));
+  $pass_count += &testAction(\$test_count, "filteredRemove() with accept and reject regular expression", 2, &FileUtils::removeFilesFiltered($dir, '\.txt$', '(alpha|delta)'));
+  # - cleanup
+  &FileUtils::removeFiles($patha, $pathd);
+}
+## testRemoveFilesFiltered()
+
+
+## @function testRemoveFilesRecursive()
+#
+sub testRemoveFilesRecursive
+{
+  my ($dir) = @_;
+  # - setup
+  my $dira = $dir . '/alpha';
+  &FileUtils::makeDirectory($dira);
+  &FileUtils::filePutContents($dira . '/alpha.txt', 'alpha');
+  &FileUtils::filePutContents($dira . '/beta.txt', 'beta');
+  my $dirb = $dira . '/beta';
+  &FileUtils::makeDirectory($dirb);
+  &FileUtils::filePutContents($dirb . '/gamma.txt', 'gamma');
+  # - test
+  $pass_count += &testAction(\$test_count, "removeFilesRecursive() for directory containing files", 5, &FileUtils::removeFilesRecursive($dira));
+}
+## testRemoveFilesRecursive()
+
+
+## @function testTransferFiles()
+#
+sub testTransferFiles
+{
+  my ($dir, $mode, $local_dir) = @_;
+  my $verb = 'moving';
+  if ($mode eq 'copy')
+  {
+    $verb = 'copying';
+  }
+  # Move a single local file to a new local location
+  # - setup
+  my $apath = $dir . '/alpha.txt';
+  &FileUtils::filePutContents($apath,'alpha');
+  my $bpath = $dir . '/beta.txt';
+  my $sub_pass_count = 0;
+  # - subtests
+  if ($mode eq 'move')
+  {
+    $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $bpath));
+    $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
+  }
+  else
+  {
+    $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $bpath));
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
+  }
+  $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($bpath));
+
+  # - test
+  $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " a single file", 3, $sub_pass_count);
+  # - cleanup
+  &FileUtils::removeFiles($bpath);
+
+  # Move multiple local files into a single local directory
+  # - setup
+  my $dpath = $dir . '/delta.txt';
+  &FileUtils::filePutContents($dpath, 'delta');
+  my $gpath = $dir . '/gamma.txt';
+  &FileUtils::filePutContents($gpath, 'gamma');
+  my $move_dir = $dir . '/movedir';
+  &FileUtils::makeDirectory($move_dir);
+  $sub_pass_count = 0;
+  # - subtests
+  if ($mode eq 'move')
+  {
+    $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($dpath, $gpath, $move_dir));
+    $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($dpath));
+    $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($gpath));
+  }
+  else
+  {
+    $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($dpath, $gpath, $move_dir));
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($dpath));
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($gpath));
+  }
+  $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/delta.txt'));
+  $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($move_dir . '/gamma.txt'));
+  # - test
+  $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a directory", 5, $sub_pass_count);
+  # - cleanup
+  if ($mode ne 'move')
+  {
+    &FileUtils::removeFiles($dpath, $gpath);
+  }
+
+  # Moving multiple files to file
+  if ($mode eq 'move')
+  {
+    $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::moveFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
+  }
+  else
+  {
+    $pass_count += &testAction(\$test_count, $mode . "Files() " . $verb . " multiple files to a file (should fail)", 0, &FileUtils::copyFiles($move_dir . '/delta.txt', $move_dir . '/gamma.txt', $bpath));
+  }
+
+  # - cleanup
+  &FileUtils::removeFilesRecursive($move_dir);
+
+  if (defined $local_dir && -d $local_dir)
+  {
+    &FileUtils::filePutContents($apath, 'alpha');
+    my $zpath = $local_dir . '/fileutils-' . time() . '.txt';
+    # - init
+    $sub_pass_count = 0;
+    # - subtests
+    if ($mode eq 'move')
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($apath, $zpath));
+      $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($apath));
+    }
+    else
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($apath, $zpath));
+      $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
+    }
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
+    # - test
+    $pass_count += &testAction(\$test_count, $mode . "Files() from non-local file to local one", 3, $sub_pass_count);
+
+    # If we are copying, we'll need to remove alpha, otherwise the round trip
+    # back to HDFS will fail
+    if ($mode eq 'copy')
+    {
+      &FileUtils::removeFiles($apath);
+    }
+
+    # - init
+    $sub_pass_count = 0;
+    # - subtests
+    if ($mode eq 'move')
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::moveFiles($zpath, $apath));
+      $sub_pass_count += &subtestAction(0, &FileUtils::fileExists($zpath));
+    }
+    else
+    {
+      $sub_pass_count += &subtestAction(1, &FileUtils::copyFiles($zpath, $apath));
+      $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($zpath));
+    }
+    $sub_pass_count += &subtestAction(1, &FileUtils::fileExists($apath));
+
+    # - test
+    $pass_count += &testAction(\$test_count, $mode . "Files() from local file to non-local one", 3, $sub_pass_count);
+    # - cleanup
+    if ($mode eq 'move')
+    {
+      &FileUtils::removeFiles($apath);
+    }
+    else
+    {
+      &FileUtils::removeFiles($apath, $zpath);
+    }
+  }
+  else
+  {
+    # Move a single non-local file to a new local location
+    $skip_count += &skipAction(\$test_count, $mode . "Files() from non-local file to local one");
+    # Move a single local file to a new non-local location
+    $skip_count += &skipAction(\$test_count, $mode . "Files() from local file to non-local one");
+  }
+}
+## testTransferFiles()
+
+1;
