Index: /main/trunk/greenstone2/perllib/FileUtils.pm
===================================================================
--- /main/trunk/greenstone2/perllib/FileUtils.pm	(revision 27306)
+++ /main/trunk/greenstone2/perllib/FileUtils.pm	(revision 27306)
@@ -0,0 +1,1201 @@
+###########################################################################
+#
+# FileUtils.pm -- functions for dealing with files. Skeleton for more
+# advanced system using dynamic class cloading available in extensions.
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2013 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package FileUtils;
+
+# Pragma
+use strict;
+use warnings;
+
+# Greenstone modules
+use util;
+
+################################################################################
+# util::cachedir()             => FileUtils::synchronizeDirectories()
+# util::cp()                   => FileUtils::copyFiles()
+# util::cp_r()                 => FileUtils::copyFilesRecursive()
+# util::cp_r_nosvn()           => FileUtils::copyFilesRecursiveNoSVN()
+# util::cp_r_toplevel()        => FileUtils::copyFilesRecursiveTopLevel()
+# util::different_files()      => FileUtils::differentFiles()
+# util::dir_exists()           => FileUtils::directoryExists()
+# util::fd_exists()            => FileUtils::fileTest()
+# util::file_exists()          => FileUtils::fileExists()
+# util::filename_cat()         => FileUtils::filenameConcatenate()
+# util::filename_is_absolute() => FileUtils::isFilenameAbsolute()
+# util::filtered_rm_r()        => FileUtils::removeFilesFiltered()
+# util::hard_link()            => FileUtils::hardLink()
+# util::mk_all_dir()           => FileUtils::makeAllDirectories()
+# util::mv()                   => FileUtils::moveFiles()
+# util::mk_dir()               => FileUtils::makeDirectory()
+# util::mv_dir_contents()      => FileUtils::moveDirectoryContents()
+# util::rm()                   => FileUtils::removeFiles()
+# util::rm_r()                 => FileUtils::removeFilesRecursive()
+# util::soft_link()            => FileUtils::softLink()
+
+################################################################################
+# Note: there are lots of functions involving files/directories/paths
+# etc found in utils.pm that are not represented here. My intention
+# was to just have those functions that need to be dynamic based on
+# filesystem, or need some rejigging to be filesystem aware. There is
+# an argument, I guess, for moving some of the other functions here so
+# that they are nicely encapsulated - but the question is what to do
+# with functions like filename_within_directory_url_format() which is
+# more URL based than file based.
+################################################################################
+
+## @function differentFiles()
+#
+# this function returns -1 if either file is not found assumes that
+# $file1 and $file2 are absolute file names or in the current
+# directory $file2 is allowed to be newer than $file1
+#
+sub differentFiles
+{
+  my ($file1, $file2, $verbosity) = @_;
+  $verbosity = 1 unless defined $verbosity;
+
+  $file1 =~ s/\/+$//;
+  $file2 =~ s/\/+$//;
+
+  my ($file1name) = $file1 =~ /\/([^\/]*)$/;
+  my ($file2name) = $file2 =~ /\/([^\/]*)$/;
+
+  return -1 unless (-e $file1 && -e $file2);
+  if ($file1name ne $file2name)
+  {
+    print STDERR "filenames are not the same\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  my @file1stat = stat ($file1);
+  my @file2stat = stat ($file2);
+
+  if (-d $file1)
+  {
+    if (! -d $file2)
+    {
+      print STDERR "one file is a directory\n" if ($verbosity >= 2);
+      return 1;
+    }
+    return 0;
+  }
+
+  # both must be regular files
+  unless (-f $file1 && -f $file2)
+  {
+    print STDERR "one file is not a regular file\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  # the size of the files must be the same
+  if ($file1stat[7] != $file2stat[7])
+  {
+    print STDERR "different sized files\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  # the second file cannot be older than the first
+  if ($file1stat[9] > $file2stat[9])
+  {
+    print STDERR "file is older\n" if ($verbosity >= 2);
+    return 1;
+  }
+
+  return 0;
+}
+## differentFiles()
+
+## @function directoryExists()
+#
+sub directoryExists
+{
+  my ($filename_full_path) = @_;
+  return &fileTest($filename_full_path, '-d');
+}
+## directoryExists()
+
+## @function closeFileHandle
+#
+sub closeFileHandle
+{
+  my ($path, $fh_ref) = @_;
+  close($$fh_ref);
+}
+## closeFileHandle()
+
+## @function copyFiles()
+#
+# copies a file or a group of files
+#
+sub copyFiles
+{
+  my $dest = pop (@_);
+  my (@srcfiles) = @_;
+
+  # remove trailing slashes from source and destination files
+  $dest =~ s/[\\\/]+$//;
+  map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+  # a few sanity checks
+  if (scalar (@srcfiles) == 0)
+  {
+    print STDERR "FileUtils::copyFiles() no destination directory given\n";
+    return;
+  }
+  elsif ((scalar (@srcfiles) > 1) && (!-d $dest))
+  {
+    print STDERR "FileUtils::copyFiles() if multiple source files are given the destination must be a directory\n";
+    return;
+  }
+
+  # copy the files
+  foreach my $file (@srcfiles)
+  {
+    my $tempdest = $dest;
+    if (-d $tempdest)
+    {
+      my ($filename) = $file =~ /([^\\\/]+)$/;
+      $tempdest .= "/$filename";
+    }
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::copyFiles() $file does not exist\n";
+    }
+    elsif (!-f $file)
+    {
+      print STDERR "FileUtils::copyFiles() $file is not a plain file\n";
+    }
+    else
+    {
+      &File::Copy::copy ($file, $tempdest);
+    }
+  }
+}
+## copyFiles()
+
+## @function copyFilesRecursive()
+#
+# recursively copies a file or group of files syntax: cp_r
+# (sourcefiles, destination directory) destination must be a directory
+# to copy one file to another use cp instead
+#
+sub copyFilesRecursive
+{
+  my $dest = pop (@_);
+  my (@srcfiles) = @_;
+
+  # a few sanity checks
+  if (scalar (@srcfiles) == 0)
+  {
+    print STDERR "FileUtils::copyFilesRecursive() no destination directory given\n";
+    return;
+  }
+  elsif (-f $dest)
+  {
+    print STDERR "FileUtils::copyFilesRecursive() destination must be a directory\n";
+    return;
+  }
+
+  # create destination directory if it doesn't exist already
+  if (! -d $dest)
+  {
+    my $store_umask = umask(0002);
+    mkdir ($dest, 0777);
+    umask($store_umask);
+  }
+
+  # copy the files
+  foreach my $file (@srcfiles)
+  {
+
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::copyFilesRecursive() $file does not exist\n";
+    }
+    elsif (-d $file)
+    {
+      # make the new directory
+      my ($filename) = $file =~ /([^\\\/]*)$/;
+      $dest = &filenameConcatenate($dest, $filename);
+      my $store_umask = umask(0002);
+      mkdir ($dest, 0777);
+      umask($store_umask);
+
+      # get the contents of this directory
+      if (!opendir (INDIR, $file))
+      {
+        print STDERR "FileUtils::copyFilesRecursive() could not open directory $file\n";
+      }
+      else
+      {
+        my @filedir = readdir (INDIR);
+        closedir (INDIR);
+        foreach my $f (@filedir)
+        {
+          next if $f =~ /^\.\.?$/;
+          # copy all the files in this directory
+          my $ff = &filenameConcatenate($file, $f);
+          &copyFilesRecursive($ff, $dest);
+        }
+      }
+
+    }
+    else
+    {
+      &copyFiles($file, $dest);
+    }
+  }
+}
+## copyFilesRecursive()
+
+## @function copyFilesRecursiveNoSVN()
+#
+# recursively copies a file or group of files, excluding SVN
+# directories, with syntax: cp_r (sourcefiles, destination directory)
+# destination must be a directory - to copy one file to another use cp
+# instead
+#
+# this should be merged with copyFilesRecursive() at some stage - jmt12
+#
+sub copyFilesRecursiveNoSVN
+{
+  my $dest = pop (@_);
+  my (@srcfiles) = @_;
+
+  # a few sanity checks
+  if (scalar (@srcfiles) == 0)
+  {
+    print STDERR "FileUtils::copyFilesRecursiveNoSVN() no destination directory given\n";
+    return;
+  }
+  elsif (-f $dest)
+  {
+    print STDERR "FileUtils::copyFilesRecursiveNoSVN() destination must be a directory\n";
+    return;
+  }
+
+  # create destination directory if it doesn't exist already
+  if (! -d $dest)
+  {
+    my $store_umask = umask(0002);
+    mkdir ($dest, 0777);
+    umask($store_umask);
+  }
+
+  # copy the files
+  foreach my $file (@srcfiles)
+  {
+    if (!-e $file)
+    {
+      print STDERR "copyFilesRecursiveNoSVN() $file does not exist\n";
+    }
+    elsif (-d $file)
+    {
+      # make the new directory
+      my ($filename) = $file =~ /([^\\\/]*)$/;
+      $dest = &filenameConcatnate($dest, $filename);
+      my $store_umask = umask(0002);
+      mkdir ($dest, 0777);
+      umask($store_umask);
+
+      # get the contents of this directory
+      if (!opendir (INDIR, $file))
+      {
+        print STDERR "copyFilesRecursiveNoSVN() could not open directory $file\n";
+      }
+      else
+      {
+        my @filedir = readdir (INDIR);
+        closedir (INDIR);
+        foreach my $f (@filedir)
+        {
+          next if $f =~ /^\.\.?$/;
+          next if $f =~ /^\.svn$/;
+          # copy all the files in this directory
+          my $ff = &filenameConcatenate($file, $f);
+          # util.pm version incorrectly called cp_r here - jmt12
+          &recursiveCopyNoSVN($ff, $dest);
+        }
+      }
+    }
+    else
+    {
+      &copyFiles($file, $dest);
+    }
+  }
+}
+## copyFilesRecursiveNoSVN()
+
+## @function copyFilesRecursiveTopLevel()
+#
+# copies a directory and its contents, excluding subdirectories, into a new directory
+#
+# another candidate for merging in with copyFilesRecursive() - jmt12
+#
+sub copyFilesRecursiveTopLevel
+{
+  my $dest = pop (@_);
+  my (@srcfiles) = @_;
+
+  # a few sanity checks
+  if (scalar (@srcfiles) == 0)
+  {
+    print STDERR "FileUtils::copyFilesRecursiveTopLevel() no destination directory given\n";
+    return;
+  }
+  elsif (-f $dest)
+  {
+    print STDERR "FileUtils::copyFilesRecursiveTopLevel() destination must be a directory\n";
+    return;
+  }
+
+  # create destination directory if it doesn't exist already
+  if (! -d $dest)
+  {
+    my $store_umask = umask(0002);
+    mkdir ($dest, 0777);
+    umask($store_umask);
+  }
+
+  # copy the files
+  foreach my $file (@srcfiles)
+  {
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::copyFilesRecursiveTopLevel() $file does not exist\n";
+    }
+    elsif (-d $file)
+    {
+      # make the new directory
+      my ($filename) = $file =~ /([^\\\/]*)$/;
+      $dest = &filenameConcatenate($dest, $filename);
+      my $store_umask = umask(0002);
+      mkdir ($dest, 0777);
+      umask($store_umask);
+
+      # get the contents of this directory
+      if (!opendir (INDIR, $file))
+      {
+        print STDERR "FileUtils::copyFilesRecursiveTopLevel() could not open directory $file\n";
+      }
+      else
+      {
+        my @filedir = readdir (INDIR);
+        closedir (INDIR);
+        foreach my $f (@filedir)
+        {
+          next if $f =~ /^\.\.?$/;
+
+          # copy all the files in this directory, but not directories
+          my $ff = &filenameConcatenate($file, $f);
+          if (-f $ff)
+          {
+            &copyFiles($ff, $dest);
+            #&cp_r ($ff, $dest);
+          }
+        }
+      }
+    }
+    else
+    {
+      &copyFiles($file, $dest);
+    }
+  }
+}
+## copyFilesRecursiveTopLevel()
+
+## @function fileExists()
+#
+sub fileExists
+{
+  my ($filename_full_path) = @_;
+  return &fileTest($filename_full_path, '-f');
+}
+## fileExists()
+
+## @function filenameConcatenate()
+#
+sub filenameConcatenate
+{
+  my $first_file = shift(@_);
+  my (@filenames) = @_;
+
+  #   Useful for debugging
+  #     -- might make sense to call caller(0) rather than (1)??
+  #   my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
+  #   print STDERR "Calling method: $cfilename:$cline $cpackage->$csubr\n";
+
+  # If first_file is not null or empty, then add it back into the list
+  if (defined $first_file && $first_file =~ /\S/)
+  {
+    unshift(@filenames, $first_file);
+  }
+
+  my $filename = join("/", @filenames);
+
+  # remove duplicate slashes and remove the last slash
+  if ($ENV{'GSDLOS'} =~ /^windows$/i)
+  {
+    $filename =~ s/[\\\/]+/\\/g;
+  }
+  else
+  {
+    $filename =~ s/[\/]+/\//g;
+    # DB: want a filename abc\de.html to remain like this
+  }
+  $filename =~ s/[\\\/]$//;
+
+  return $filename;
+}
+## filenameConcatenate()
+
+## @function filePutContents()
+#
+# Create a file and write the given string directly to it
+# @param $path the full path of the file to write as a String
+# @param $content the String to be written to the file
+#
+sub filePutContents
+{
+  my ($path, $content) = @_;
+  if (open(FOUT, '>:utf8', $path))
+  {
+    print FOUT $content;
+    close(FOUT);
+  }
+  else
+  {
+    die('Error! Failed to open file for writing: ' . $path . "\n");
+  }
+}
+## filePutContents()
+
+## @function fileSize()
+#
+sub fileSize
+{
+  my $path = shift(@_);
+  my $file_size = -s $path;
+  return $file_size;
+}
+## fileSize()
+
+## @function fileTest()
+#
+sub fileTest
+{
+  my $filename_full_path = shift @_;
+  my $test_op = shift @_ || "-e";
+
+  # By default tests for existance of file or directory (-e)
+  # Can be made more specific by providing second parameter (e.g. -f or -d)
+
+  my $exists = 0;
+
+  if ($ENV{'GSDLOS'} =~ m/^windows$/i)
+  {
+    require Win32;
+    my $filename_short_path = Win32::GetShortPathName($filename_full_path);
+    if (!defined $filename_short_path)
+    {
+      # Was probably still in UTF8 form (not what is needed on Windows)
+      my $unicode_filename_full_path = eval "decode(\"utf8\",\$filename_full_path)";
+      if (defined $unicode_filename_full_path)
+      {
+        $filename_short_path = Win32::GetShortPathName($unicode_filename_full_path);
+      }
+    }
+    $filename_full_path = $filename_short_path;
+  }
+
+  if (defined $filename_full_path)
+  {
+    $exists = eval "($test_op \$filename_full_path)";
+  }
+
+  return $exists;
+}
+## fileTest()
+
+## @function hardLink()
+# make hard link to file if supported by OS, otherwise copy the file
+#
+sub hardLink
+{
+  my ($src, $dest, $verbosity) = @_;
+
+  # remove trailing slashes from source and destination files
+  $src =~ s/[\\\/]+$//;
+  $dest =~ s/[\\\/]+$//;
+
+  ##    print STDERR "**** src = ", unicode::debug_unicode_string($src),"\n";
+  # a few sanity checks
+  if (-e $dest)
+  {
+    # destination file already exists
+    return;
+  }
+  elsif (!-e $src)
+  {
+    print STDERR "FileUtils::hardLink() source file \"" . $src . "\" does not exist\n";
+    return 1;
+  }
+  elsif (-d $src)
+  {
+    print STDERR "FileUtils::hardLink() source \"" . $src . "\" is a directory\n";
+    return 1;
+  }
+
+  my $dest_dir = &File::Basename::dirname($dest);
+  if (!-e $dest_dir)
+  {
+    &makeAllDirectories($dest_dir);
+  }
+
+  if (!link($src, $dest))
+  {
+    if ((!defined $verbosity) || ($verbosity>2))
+    {
+      print STDERR "FileUtils::hardLink(): unable to create hard link. ";
+      print STDERR " Copying file: $src -> $dest\n";
+    }
+    &File::Copy::copy ($src, $dest);
+  }
+  return 0;
+}
+## hardLink()
+
+## @function isDirectoryEmpty()
+#
+# A method to check if a directory is empty (note that an empty
+# directory still has non-zero size!!!).  Code is from
+# http://episteme.arstechnica.com/eve/forums/a/tpc/f/6330927813/m/436007700831
+#
+sub isDirectoryEmpty
+{
+  my ($path) = @_;
+  opendir DIR, $path;
+  while(my $entry = readdir DIR)
+  {
+    next if($entry =~ /^\.\.?$/);
+    closedir DIR;
+    return 0;
+  }
+  closedir DIR;
+  return 1;
+}
+## isDirectoryEmpty()
+
+## @function isFilenameAbsolute()
+#
+sub isFilenameAbsolute
+{
+  my ($filename) = @_;
+  if ($ENV{'GSDLOS'} =~ /^windows$/i)
+  {
+    return ($filename =~ m/^(\w:)?\\/);
+  }
+  return ($filename =~ m/^\//);
+}
+# isFilenameAbsolute()
+
+## @function isSymbolicLink
+#
+# Determine if a given path is a symbolic link (soft link)
+#
+sub isSymbolicLink
+{
+  my $path = shift(@_);
+  my $is_soft_link = -l $path;
+  return $is_soft_link;
+}
+## isSymbolicLink()
+
+## @function makeAllDirectories()
+#
+# in case anyone cares - I did some testing (using perls Benchmark module)
+# on this subroutine against File::Path::mkpath (). mk_all_dir() is apparently
+# slightly faster (surprisingly) - Stefan.
+#
+sub makeAllDirectories
+{
+  my ($dir) = @_;
+
+  # use / for the directory separator, remove duplicate and
+  # trailing slashes
+  $dir=~s/[\\\/]+/\//g;
+  $dir=~s/[\\\/]+$//;
+
+  # make sure the cache directory exists
+  my $dirsofar = "";
+  my $first = 1;
+  foreach my $dirname (split ("/", $dir))
+  {
+    $dirsofar .= "/" unless $first;
+    $first = 0;
+
+    $dirsofar .= $dirname;
+
+    next if $dirname =~ /^(|[a-z]:)$/i;
+    if (!-e $dirsofar)
+    {
+      my $store_umask = umask(0002);
+      my $mkdir_ok = mkdir ($dirsofar, 0777);
+      umask($store_umask);
+      if (!$mkdir_ok)
+      {
+        print STDERR "FileUtils::makeAllDirectories() could not create directory $dirsofar\n";
+        return;
+      }
+    }
+  }
+}
+## makeAllDirectories()
+
+## @function makeDirectory()
+#
+sub makeDirectory
+{
+  my ($dir) = @_;
+
+  my $store_umask = umask(0002);
+  my $mkdir_ok = mkdir ($dir, 0777);
+  umask($store_umask);
+
+  if (!$mkdir_ok)
+  {
+    print STDERR "FileUtils::makeDirectory() could not create directory $dir\n";
+    return;
+  }
+}
+## makeDirectory()
+
+## @function modificationTime()
+#
+sub modificationTime
+{
+  my $path = shift(@_);
+  my @file_status = stat($path);
+  return $file_status[9];
+}
+## modificationTime()
+
+## @function moveDirectoryContents()
+#
+# Move the contents of source directory into target directory (as
+# opposed to merely replacing target dir with the src dir) This can
+# overwrite any files with duplicate names in the target but other
+# files and folders in the target will continue to exist
+#
+sub moveDirectoryContents
+{
+  my ($src_dir, $dest_dir) = @_;
+
+  # Obtain listing of all files within src_dir
+  # Note that readdir lists relative paths, as well as . and ..
+  opendir(DIR, "$src_dir");
+  my @files= readdir(DIR);
+  close(DIR);
+
+  my @full_path_files = ();
+  foreach my $file (@files)
+  {
+    # process all except . and ..
+    unless($file eq "." || $file eq "..")
+    {
+      my $dest_subdir = &filenameConcatenate($dest_dir, $file); # $file is still a relative path
+
+      # construct absolute paths
+      $file = &filenameConcatenate($src_dir, $file); # $file is now an absolute path
+
+      # Recurse on directories which have an equivalent in target dest_dir
+      # If $file is a directory that already exists in target $dest_dir,
+      # then a simple move operation will fail (definitely on Windows).
+      if(-d $file && -d $dest_subdir)
+      {
+        #print STDERR "**** $file is a directory also existing in target, its contents to be copied to $dest_subdir\n";
+        &moveDirectoryContents($file, $dest_subdir);
+
+        # now all content is moved across, delete empty dir in source folder
+        if(&isDirectoryEmpty($file))
+        {
+          if (!rmdir $file)
+          {
+            print STDERR "ERROR. FileUtils::moveDirectoryContents() couldn't remove directory $file\n";
+          }
+        }
+        # error
+        else
+        {
+          print STDERR "ERROR. FileUtils::moveDirectoryContents(): subfolder $file still non-empty after moving contents to $dest_subdir\n";
+        }
+      }
+      # process files and any directories that don't already exist with a simple move
+      else
+      {
+        push(@full_path_files, $file);
+      }
+    }
+  }
+
+  # create target toplevel folder or subfolders if they don't exist
+  if(!&directoryExists($dest_dir))
+  {
+    &makeDirectory($dest_dir);
+  }
+
+  #print STDERR "@@@@@ Copying files |".join(",", @full_path_files)."| to: $dest_dir\n";
+
+  # if non-empty, there's something to copy across
+  if(@full_path_files)
+  {
+    &moveFiles(@full_path_files, $dest_dir);
+  }
+}
+## moveDirectoryContents()
+
+## @function moveFiles()
+#
+# moves a file or a group of files
+#
+sub moveFiles
+{
+  my $dest = pop (@_);
+  my (@srcfiles) = @_;
+
+  # remove trailing slashes from source and destination files
+  $dest =~ s/[\\\/]+$//;
+  map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+  # a few sanity checks
+  if (scalar (@srcfiles) == 0)
+  {
+    print STDERR "FileUtils::moveFiles() no destination directory given\n";
+    return;
+  }
+  elsif ((scalar (@srcfiles) > 1) && (!-d $dest))
+  {
+    print STDERR "FileUtils::moveFiles() if multiple source files are given the destination must be a directory\n";
+    return;
+  }
+
+  # move the files
+  foreach my $file (@srcfiles)
+  {
+    my $tempdest = $dest;
+    if (-d $tempdest)
+    {
+      my ($filename) = $file =~ /([^\\\/]+)$/;
+      $tempdest .= "/$filename";
+    }
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::moveFiles() $file does not exist\n";
+    }
+    else
+    {
+      if(!rename ($file, $tempdest))
+      {
+        print STDERR "**** Failed to rename $file to $tempdest\n";
+        &File::Copy::copy($file, $tempdest);
+        &removeFiles($file);
+      }
+      # rename (partially) succeeded) but srcfile still exists after rename
+      elsif(-e $file)
+      {
+        #print STDERR "*** srcfile $file still exists after rename to $tempdest\n";
+        if(!-e $tempdest)
+        {
+          print STDERR "@@@@ ERROR: $tempdest does not exist\n";
+        }
+        # Sometimes the rename operation fails (as does
+        # File::Copy::move).  This turns out to be because the files
+        # are hardlinked.  Need to do a copy-delete in this case,
+        # however, the copy step is not necessary: the srcfile got
+        # renamed into tempdest, but srcfile itself still exists,
+        # delete it.  &File::Copy::copy($file, $tempdest);
+        &removeFiles($file);
+      }
+    }
+  }
+}
+## moveFiles()
+
+## @function openFileHandle()
+#
+sub openFileHandle
+{
+  my $path = shift(@_);
+  my $mode = shift(@_);
+  my $fh_ref = shift(@_);
+  my $encoding = shift(@_);
+  my $mode_symbol = '<';
+  if ($mode eq 'w')
+  {
+    $mode_symbol = '>';
+  }
+  if (defined $encoding)
+  {
+    $mode_symbol .= ':' . $encoding;
+  }
+  if (!open($$fh_ref, $mode_symbol, $path))
+  {
+    $mode = 'writing';
+    if ($mode eq '>')
+    {
+      $mode = 'reading';
+    }
+    die("Error! Failed to open file for " . $mode . ": " . $path . "\n");
+  }
+  return 1;
+}
+## openFileHandle()
+
+## @function readDirectory()
+#
+sub readDirectory
+{
+  my $path = shift(@_);
+  my @files;
+  if (opendir(DH, $path))
+  {
+    @files = readdir(DH);
+    close(DH);
+  }
+  else
+  {
+    die("Error! Failed to open directory to list files: " . $path . "\n");
+  }
+  return \@files;
+}
+## readDirectory()
+
+## @function removeFiles()
+#
+# removes files (but not directories)
+#
+sub removeFiles
+{
+  my (@files) = @_;
+  my @filefiles = ();
+
+  # make sure the files we want to delete exist
+  # and are regular files
+  foreach my $file (@files)
+  {
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::removeFiles() $file does not exist\n";
+    }
+    elsif ((!-f $file) && (!-l $file))
+    {
+      print STDERR "FileUtils::removeFiles() $file is not a regular (or symbolic) file\n";
+    }
+    else
+    {
+      push (@filefiles, $file);
+    }
+  }
+
+  # remove the files
+  my $numremoved = unlink @filefiles;
+
+  # check to make sure all of them were removed
+  if ($numremoved != scalar(@filefiles))
+  {
+    print STDERR "FileUtils::removeFiles() Not all files were removed\n";
+  }
+}
+## removeFiles()
+
+## @function removeFilesDebug()
+#
+# removes files (but not directories) - can rename this to the default
+# "rm" subroutine when debugging the deletion of individual files.
+# Unused?
+#
+sub removeFilesDebug
+{
+  my (@files) = @_;
+  my @filefiles = ();
+
+  # make sure the files we want to delete exist
+  # and are regular files
+  foreach my $file (@files)
+  {
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::removeFilesDebug() " . $file . " does not exist\n";
+    }
+    elsif ((!-f $file) && (!-l $file))
+    {
+      print STDERR "FileUtils::removeFilesDebug() " . $file . " is not a regular (or symbolic) file\n";
+    }
+    # debug message
+    else
+    {
+      unlink($file) or warn "Could not delete file " . $file . ": " . $! . "\n";
+    }
+  }
+}
+## removeFilesDebug()
+
+## @function removeFilesFiltered()
+#
+sub removeFilesFiltered
+{
+  my ($files,$file_accept_re,$file_reject_re) = @_;
+
+  #   my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(2);
+  #   my ($lcfilename) = ($cfilename =~ m/([^\\\/]*)$/);
+  #   print STDERR "** Calling method (2): $lcfilename:$cline $cpackage->$csubr\n";
+
+  my @files_array = (ref $files eq "ARRAY") ? @$files : ($files);
+
+  # recursively remove the files
+  foreach my $file (@files_array)
+  {
+    $file =~ s/[\/\\]+$//; # remove trailing slashes
+
+    if (!-e $file)
+    {
+      print STDERR "FileUtils::removeFilesFiltered() $file does not exist\n";
+    }
+    # don't recurse down symbolic link
+    elsif ((-d $file) && (!-l $file))
+    {
+      # get the contents of this directory
+      if (!opendir (INDIR, $file))
+      {
+        print STDERR "FileUtils::removeFilesFiltered() could not open directory $file\n";
+      }
+      else
+      {
+        my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+        closedir (INDIR);
+
+        # remove all the files in this directory
+        map {$_="$file/$_";} @filedir;
+        &removeFilesFiltered(\@filedir,$file_accept_re,$file_reject_re);
+
+        if (!defined $file_accept_re && !defined $file_reject_re)
+        {
+          # remove this directory
+          if (!rmdir $file)
+          {
+            print STDERR "FileUtils::removeFilesFiltered() couldn't remove directory $file\n";
+          }
+        }
+      }
+    }
+    else
+    {
+      next if (defined $file_reject_re && ($file =~ m/$file_reject_re/));
+
+      if ((!defined $file_accept_re) || ($file =~ m/$file_accept_re/))
+      {
+        # remove this file
+        &removeFiles($file);
+      }
+    }
+  }
+}
+## removeFilesFiltered()
+
+## @function removeFilesRecursive()
+#
+# The equivalent of "rm -rf" with all the dangers therein
+#
+sub removeFilesRecursive
+{
+  my (@files) = @_;
+
+  # use the more general (but reterospectively written) function
+  # filtered_rm_r function() with no accept or reject expressions
+  &removeFilesFiltered(\@files,undef,undef);
+}
+## removeFilesRecursive()
+
+## @function sanitizePath()
+#
+sub sanitizePath
+{
+  my ($path) = @_;
+
+  # fortunately filename concatenate will perform all the double slash
+  # removal and end slash removal we need, and in a protocol aware
+  # fashion
+  return &filenameConcatenate($path);
+}
+## sanitizePath()
+
+## @function softLink()
+#
+# make soft link to file if supported by OS, otherwise copy file
+#
+sub softLink
+{
+  my ($src, $dest, $ensure_paths_absolute) = @_;
+
+  # remove trailing slashes from source and destination files
+  $src =~ s/[\\\/]+$//;
+  $dest =~ s/[\\\/]+$//;
+
+  # Ensure file paths are absolute IF requested to do so
+  # Soft_linking didn't work for relative paths
+  if(defined $ensure_paths_absolute && $ensure_paths_absolute)
+  {
+    # We need to ensure that the src file is the absolute path
+    # See http://perldoc.perl.org/File/Spec.html
+    # it's relative
+    if(!File::Spec->file_name_is_absolute( $src ))
+    {
+      $src = File::Spec->rel2abs($src); # make absolute
+    }
+    # Might as well ensure that the destination file's absolute path is used
+    if(!File::Spec->file_name_is_absolute( $dest ))
+    {
+      $dest = File::Spec->rel2abs($dest); # make absolute
+    }
+  }
+
+  # a few sanity checks
+  if (!-e $src)
+  {
+    print STDERR "FileUtils::softLink() source file $src does not exist\n";
+    return 0;
+  }
+
+  my $dest_dir = &File::Basename::dirname($dest);
+  if (!-e $dest_dir)
+  {
+    &makeAllDirectories($dest_dir);
+  }
+
+  if ($ENV{'GSDLOS'} =~ /^windows$/i)
+  {
+    # symlink not supported on windows
+    &File::Copy::copy ($src, $dest);
+  }
+  elsif (!eval {symlink($src, $dest)})
+  {
+    print STDERR "FileUtils::softLink(): unable to create soft link.\n";
+    return 0;
+  }
+  return 1;
+}
+## softLink()
+
+## @function synchronizeDirectory()
+#
+# updates a copy of a directory in some other part of the filesystem
+# verbosity settings are: 0=low, 1=normal, 2=high
+# both $fromdir and $todir should be absolute paths
+#
+sub synchronizeDirectory
+{
+  my ($fromdir, $todir, $verbosity) = @_;
+  $verbosity = 1 unless defined $verbosity;
+
+  # use / for the directory separator, remove duplicate and
+  # trailing slashes
+  $fromdir=~s/[\\\/]+/\//g;
+  $fromdir=~s/[\\\/]+$//;
+  $todir=~s/[\\\/]+/\//g;
+  $todir=~s/[\\\/]+$//;
+
+  &makeAllDirectories($todir);
+
+  # get the directories in ascending order
+  if (!opendir (FROMDIR, $fromdir))
+  {
+    print STDERR "FileUtils::synchronizeDirectory() could not read directory $fromdir\n";
+    return;
+  }
+  my @fromdir = grep (!/^\.\.?$/, sort(readdir (FROMDIR)));
+  closedir (FROMDIR);
+
+  if (!opendir (TODIR, $todir))
+  {
+    print STDERR "FileUtils::synchronizeDirectory() could not read directory $todir\n";
+    return;
+  }
+  my @todir = grep (!/^\.\.?$/, sort(readdir (TODIR)));
+  closedir (TODIR);
+
+  my $fromi = 0;
+  my $toi = 0;
+
+  while ($fromi < scalar(@fromdir) || $toi < scalar(@todir))
+  {
+    #	print "fromi: $fromi toi: $toi\n";
+
+    # see if we should delete a file/directory
+    # this should happen if the file/directory
+    # is not in the from list or if its a different
+    # size, or has an older timestamp
+    if ($toi < scalar(@todir))
+    {
+      if (($fromi >= scalar(@fromdir)) || ($todir[$toi] lt $fromdir[$fromi] || ($todir[$toi] eq $fromdir[$fromi] && &differentFiles("$fromdir/$fromdir[$fromi]","$todir/$todir[$toi]", $verbosity))))
+      {
+
+        # the files are different
+        &removeFilesRecursive("$todir/$todir[$toi]");
+        splice(@todir, $toi, 1); # $toi stays the same
+
+      }
+      elsif ($todir[$toi] eq $fromdir[$fromi])
+      {
+        # the files are the same
+        # if it is a directory, check its contents
+        if (-d "$todir/$todir[$toi]")
+        {
+          &synchronizeDirectory("$fromdir/$fromdir[$fromi]", "$todir/$todir[$toi]", $verbosity);
+        }
+
+        $toi++;
+        $fromi++;
+        next;
+      }
+    }
+
+    # see if we should insert a file/directory
+    # we should insert a file/directory if there
+    # is no tofiles left or if the tofile does not exist
+    if ($fromi < scalar(@fromdir) && ($toi >= scalar(@todir) || $todir[$toi] gt $fromdir[$fromi]))
+    {
+      &cp_r ("$fromdir/$fromdir[$fromi]", "$todir/$fromdir[$fromi]");
+      splice (@todir, $toi, 0, $fromdir[$fromi]);
+
+      $toi++;
+      $fromi++;
+    }
+  }
+}
+## synchronizeDirectory()
+
+1;
Index: /main/trunk/greenstone2/perllib/basebuilder.pm
===================================================================
--- /main/trunk/greenstone2/perllib/basebuilder.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/basebuilder.pm	(revision 27306)
@@ -35,4 +35,5 @@
 use plugin;
 use util;
+use FileUtils;
 
 
@@ -248,10 +249,10 @@
     if (!$self->{'debug'} && !$self->{'keepold'}) {
 	# remove any old builds
-	&util::rm_r($self->{'build_dir'});
-	&util::mk_all_dir($self->{'build_dir'});
+	&FileUtils::removeFilesRecursive($self->{'build_dir'});
+	&FileUtils::makeAllDirectories($self->{'build_dir'});
         
 	# make the text directory
 	my $textdir = "$self->{'build_dir'}/text";
-	&util::mk_all_dir($textdir);
+	&FileUtils::makeAllDirectories($textdir);
     }
 
@@ -404,8 +405,8 @@
     print STDERR "BuildDir: $self->{'build_dir'}\n";
 
-    my $textdir = &util::filename_cat($self->{'build_dir'}, "text");
-    my $assocdir = &util::filename_cat($self->{'build_dir'}, "assoc");
-    &util::mk_all_dir ($textdir);
-    &util::mk_all_dir ($assocdir);
+    my $textdir = &FileUtils::filenameConcatenate($self->{'build_dir'}, "text");
+    my $assocdir = &FileUtils::filenameConcatenate($self->{'build_dir'}, "assoc");
+    &FileUtils::makeAllDirectories ($textdir);
+    &FileUtils::makeAllDirectories ($assocdir);
 
     # Get info database file path
@@ -541,5 +542,5 @@
 
     # get the text directory
-    &util::mk_all_dir ($self->{'build_dir'});
+    &FileUtils::makeAllDirectories ($self->{'build_dir'});
 
     # store the build date
@@ -596,9 +597,9 @@
     
     # write out the earliestDatestamp information needed for OAI
-    my $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives");
+    my $archivedir = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, "archives");
     if(!-d $archivedir) {
-	$archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "export");
-    }
-    my $earliestDatestampFile = &util::filename_cat ($archivedir, "earliestDatestamp");
+	$archivedir = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, "export");
+    }
+    my $earliestDatestampFile = &FileUtils::filenameConcatenate($archivedir, "earliestDatestamp");
     my $earliestDatestamp = 0;
     if (open(FIN,"<$earliestDatestampFile")) {
@@ -620,9 +621,9 @@
 
     if ($gs_mode eq "gs2") {
-      &colcfg::write_build_cfg(&util::filename_cat($self->{'build_dir'},"build.cfg"), $build_cfg);
+      &colcfg::write_build_cfg(&FileUtils::filenameConcatenate($self->{'build_dir'},"build.cfg"), $build_cfg);
     }
     if ($gs_mode eq "gs3") {
 
-      &colcfg::write_build_cfg_xml(&util::filename_cat($self->{'build_dir'}, "buildConfig.xml"), $build_cfg, $self->{'collect_cfg_preserve'});
+      &colcfg::write_build_cfg_xml(&FileUtils::filenameConcatenate($self->{'build_dir'}, "buildConfig.xml"), $build_cfg, $self->{'collect_cfg_preserve'});
     }    
 
@@ -766,9 +767,9 @@
     }
     
-    my $buildconfigfile = &util::filename_cat($self->{'build_dir'}, $buildconfigfilename);
+    my $buildconfigfile = &FileUtils::filenameConcatenate($self->{'build_dir'}, $buildconfigfilename);
     
     if (!-e $buildconfigfile) {
 	# try the index dir - but do we know where it is?? try here
-	$buildconfigfile  = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "index", $buildconfigfilename);
+	$buildconfigfile  = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, "index", $buildconfigfilename);
 	if (!-e $buildconfigfile) {
 	    #we cant find a config file - just ignore the field list
Index: /main/trunk/greenstone2/perllib/basebuildproc.pm
===================================================================
--- /main/trunk/greenstone2/perllib/basebuildproc.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/basebuildproc.pm	(revision 27306)
@@ -39,4 +39,5 @@
 no strict 'refs';
 use util;
+use FileUtils;
 
 BEGIN {
@@ -78,5 +79,5 @@
 	# For incremental building need to seed num_docs etc from values
 	# stored in build.cfg (if present)
-	$buildconfigfile = &util::filename_cat($build_dir, "build.cfg");
+	$buildconfigfile = &FileUtils::filenameConcatenate($build_dir, "build.cfg");
 	if (-e $buildconfigfile) {
 	    $found_num_data = 1;
@@ -84,5 +85,5 @@
 	else {
 	    # try the index dir
-	    $buildconfigfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
+	    $buildconfigfile = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, 
 						   "index", "build.cfg");
 	    if (-e $buildconfigfile) {
@@ -835,10 +836,10 @@
 	# dir, otherwise it is relative to the HASH... directory
 	if ($assoc_file->[1] =~ m@^[/\\]@) {
-	    $afile = &util::filename_cat($self->{'assocdir'}, $assoc_file->[1]);
+	    $afile = &FileUtils::filenameConcatenate($self->{'assocdir'}, $assoc_file->[1]);
 	} else {
-	    $afile = &util::filename_cat($self->{'assocdir'}, $archivedir, $assoc_file->[1]);
+	    $afile = &FileUtils::filenameConcatenate($self->{'assocdir'}, $archivedir, $assoc_file->[1]);
 	}
 	
-	&util::hard_link ($assoc_file->[0], $afile, $self->{'verbosity'});
+	&FileUtils::hardLink($assoc_file->[0], $afile, $self->{'verbosity'});
     }
 }
@@ -849,7 +850,7 @@
     my ($archivedir, $edit_mode) = @_;
 
-    my $assoc_dir = &util::filename_cat($self->{'assocdir'}, $archivedir);
+    my $assoc_dir = &FileUtils::filenameConcatenate($self->{'assocdir'}, $archivedir);
     if (-d $assoc_dir) {
-	&util::rm_r($assoc_dir);
-    }
-}
+	&FileUtils::removeFilesRecursive($assoc_dir);
+    }
+}
Index: /main/trunk/greenstone2/perllib/colcfg.pm
===================================================================
--- /main/trunk/greenstone2/perllib/colcfg.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/colcfg.pm	(revision 27306)
@@ -31,4 +31,7 @@
 use cfgread;
 use gsprintf 'gsprintf';
+use util;
+use FileUtils;
+
 use strict;
 
@@ -268,5 +271,5 @@
     # gsdl/collect/COLLECTION/custom/COLLECTION/etc/custom.cfg file. This
     # customization was added for DLC by Stefan, 30/6/2007.
-    my $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "custom", $ENV{'GSDLCOLLECTION'}, "etc", "custom.cfg");
+    my $configfilename = &FileUtils::filenameConcatenate ($ENV{'GSDLCOLLECTDIR'}, "custom", $ENV{'GSDLCOLLECTION'}, "etc", "custom.cfg");
 
     if (-e $configfilename) {
@@ -275,5 +278,5 @@
 
     # Check if there is a collectionConfig.xml file. If there is one, it's gs3
-    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collectionConfig.xml");    
+    $configfilename = &FileUtils::filenameConcatenate ($ENV{'GSDLCOLLECTDIR'}, "etc", "collectionConfig.xml");    
     if (-e $configfilename) {
         return ($configfilename, "gs3");
@@ -281,5 +284,5 @@
 
     # If we get to here we check if there is a collect.cfg file in the usual place, i.e. it is gs2.
-    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
+    $configfilename = &FileUtils::filenameConcatenate ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
     if (-e $configfilename) {
         return ($configfilename, "gs2");
@@ -304,5 +307,5 @@
 
     if($gs_version eq "gs2") {
-	$configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "custom", $ENV{'GSDLCOLLECTION'}, "etc", "custom.cfg");
+	$configfilename = &FileUtils::filenameConcatenate ($ENV{'GSDLCOLLECTDIR'}, "custom", $ENV{'GSDLCOLLECTION'}, "etc", "custom.cfg");
     
 	if (-e $configfilename) {
@@ -313,5 +316,5 @@
     # Check if there is a collectionConfig.xml file if it's gs3
     if($gs_version eq "gs3") {
-	$configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collectionConfig.xml");    
+	$configfilename = &FileUtils::filenameConcatenate ($ENV{'GSDLCOLLECTDIR'}, "etc", "collectionConfig.xml");    
 	if (-e $configfilename) {
 	    return $configfilename;
@@ -321,5 +324,5 @@
     # Check if there is a collect.cfg file in the usual place for gs2.
     if($gs_version eq "gs2") {
-	$configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
+	$configfilename = &FileUtils::filenameConcatenate ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
 	if (-e $configfilename) {
 	    return $configfilename;
Index: /main/trunk/greenstone2/perllib/dbutil/gdbm.pm
===================================================================
--- /main/trunk/greenstone2/perllib/dbutil/gdbm.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/dbutil/gdbm.pm	(revision 27306)
@@ -29,4 +29,6 @@
 use strict;
 
+use util;
+use FileUtils;
 use dbutil::gdbmtxtgz;
 
@@ -40,5 +42,5 @@
   my $opt_append = shift(@_);
 
-  my $txt2db_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",$ENV{'GSDLOS'}, "txt2db" . &util::get_os_exe());
+  my $txt2db_exe = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin",$ENV{'GSDLOS'}, "txt2db" . &util::get_os_exe());
   my $infodb_file_handle = undef;
   my $cmd = "\"$txt2db_exe\"";
@@ -84,5 +86,5 @@
   my $infodb_file_extension = ".gdb";
   my $infodb_file_name = &util::get_dirsep_tail($collection_name) . $infodb_file_extension;
-  return &util::filename_cat($infodb_directory_path, $infodb_file_name);
+  return &FileUtils::filenameConcatenate($infodb_directory_path, $infodb_file_name);
 }
 
Index: /main/trunk/greenstone2/perllib/doc.pm
===================================================================
--- /main/trunk/greenstone2/perllib/doc.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/doc.pm	(revision 27306)
@@ -37,4 +37,5 @@
 use unicode;
 use util;
+use FileUtils;
 use ghtml;
 use File::stat;
@@ -89,4 +90,7 @@
 	$self->{'terse_source_path'} = $source_filename;
 
+        # Use the FileUtil library methods as they are aware of more special
+        # cases such as HDFS [jmt12]
+ 	##if (&util::file_exists($source_filename)) {
 	if (-e $source_filename) {
 	    # See if we can do better for Windows with a filename
@@ -241,5 +245,5 @@
 #    print STDERR "-> $srcfilename -> ";
 #    $srcfilename = &util::rename_file($srcfilename.$suffix, $rename_method);
-#    $source_filename = &util::filename_cat($dirname, $srcfilename);
+#    $source_filename = &FileUtils::filenameConcatenate($dirname, $srcfilename);
 #    print STDERR "$source_filename\n";
 	
@@ -257,5 +261,5 @@
 #    print STDERR "-> $srcfilename -> ";
     $srcfilename = &util::rename_file($srcfilename.$suffix, $rename_method);
-    $source_filename = &util::filename_cat($dirname, $srcfilename);
+    $source_filename = &FileUtils::filenameConcatenate($dirname, $srcfilename);
 
     return $source_filename;
@@ -304,6 +308,6 @@
     }
 
-    if (!&util::filename_is_absolute($filename)) {
-	$filename = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},$filename);
+    if (!&FileUtils::isFilenameAbsolute($filename)) {
+	$filename = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'},$filename);
     }
 
@@ -367,5 +371,5 @@
     my $osexe = &util::get_os_exe();
 
-    my $hashfile_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",
+    my $hashfile_exe = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin",
 					   $ENV{'GSDLOS'},"hashfile$osexe");
 
Index: /main/trunk/greenstone2/perllib/gsprintf.pm
===================================================================
--- /main/trunk/greenstone2/perllib/gsprintf.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/gsprintf.pm	(revision 27306)
@@ -32,4 +32,5 @@
 use unicode;
 use util;
+use FileUtils;
 
 @gsprintf::EXPORT_OK = qw(gsprintf); # functions we can export into namespace
@@ -236,5 +237,5 @@
 
     my $primary_resourcebundlefile 
-	= &util::filename_cat($bundle_base,$primary_dir,$resourcename);
+	= &FileUtils::filenameConcatenate($bundle_base,$primary_dir,$resourcename);
 
     my $resourcebundle = read_resource_bundle($primary_resourcebundlefile);
@@ -245,8 +246,8 @@
 	foreach my $e (@extensions) {	
 	    my $ext_base
-		= &util::filename_cat($bundle_base,"ext",$e);
+		= &FileUtils::filenameConcatenate($bundle_base,"ext",$e);
 	    
 	    my $ext_resourcebundlefile 
-		= &util::filename_cat($ext_base,$primary_dir,$resourcename);
+		= &FileUtils::filenameConcatenate($ext_base,$primary_dir,$resourcename);
 	    
 	    # can ignore return value (will be same reference to $resourcebundle)
@@ -258,8 +259,8 @@
 	foreach my $e (@extensions) {	
 	    my $ext_base
-		= &util::filename_cat($ENV{'GSDL3SRCHOME'},"ext",$e);
+		= &FileUtils::filenameConcatenate($ENV{'GSDL3SRCHOME'},"ext",$e);
 	    
 	    my $ext_resourcebundlefile 
-		= &util::filename_cat($ext_base,$primary_dir,$resourcename);
+		= &FileUtils::filenameConcatenate($ext_base,$primary_dir,$resourcename);
 	    
 	    # can ignore return value (will be same reference to $resourcebundle)
Index: /main/trunk/greenstone2/perllib/mgppbuilder.pm
===================================================================
--- /main/trunk/greenstone2/perllib/mgppbuilder.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/mgppbuilder.pm	(revision 27306)
@@ -31,4 +31,5 @@
 use strict; no strict 'refs';
 use util;
+use FileUtils;
 
 
@@ -171,15 +172,15 @@
     my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
     my $exe = &util::get_os_exe ();
-    my $mgpp_passes_exe = &util::filename_cat($exedir, "mgpp_passes$exe");
-    my $mgpp_compression_dict_exe = &util::filename_cat($exedir, "mgpp_compression_dict$exe");
+    my $mgpp_passes_exe = &FileUtils::filenameConcatenate($exedir, "mgpp_passes$exe");
+    my $mgpp_compression_dict_exe = &FileUtils::filenameConcatenate($exedir, "mgpp_compression_dict$exe");
     my $outhandle = $self->{'outhandle'};
 
     my $maxnumeric = $self->{'maxnumeric'};
     
-    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, "text"));
+    &FileUtils::makeAllDirectories (&FileUtils::filenameConcatenate($self->{'build_dir'}, "text"));
 
     my $collect_tail = &util::get_dirsep_tail($self->{'collection'});
-    my $basefilename = &util::filename_cat("text",$collect_tail);
-    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, $basefilename);
+    my $basefilename = &FileUtils::filenameConcatenate("text",$collect_tail);
+    my $fulltextprefix = &FileUtils::filenameConcatenate($self->{'build_dir'}, $basefilename);
     
     my $osextra = "";
@@ -384,11 +385,11 @@
     # get the full index directory path and make sure it exists
     my $indexdir = $self->{'index_mapping'}->{$index};
-    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, $indexdir));
+    &FileUtils::makeAllDirectories (&FileUtils::filenameConcatenate($self->{'build_dir'}, $indexdir));
 
     my $collect_tail = &util::get_dirsep_tail($self->{'collection'});
-    my $fullindexprefix = &util::filename_cat ($self->{'build_dir'}, 
+    my $fullindexprefix = &FileUtils::filenameConcatenate($self->{'build_dir'}, 
 					       $indexdir, 
 					       $collect_tail);
-    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, "text", 
+    my $fulltextprefix = &FileUtils::filenameConcatenate($self->{'build_dir'}, "text", 
 					       $collect_tail);
 
@@ -397,5 +398,5 @@
 
     my $exe = &util::get_os_exe ();
-    my $mgpp_passes_exe = &util::filename_cat($exedir, "mgpp_passes$exe");
+    my $mgpp_passes_exe = &FileUtils::filenameConcatenate($exedir, "mgpp_passes$exe");
 
     # define the section names for mgpasses
@@ -406,11 +407,11 @@
 
     my $mgpp_perf_hash_build_exe = 
-	&util::filename_cat($exedir, "mgpp_perf_hash_build$exe");
+	&FileUtils::filenameConcatenate($exedir, "mgpp_perf_hash_build$exe");
     my $mgpp_weights_build_exe = 
-	&util::filename_cat ($exedir, "mgpp_weights_build$exe");
+	&FileUtils::filenameConcatenate($exedir, "mgpp_weights_build$exe");
     my $mgpp_invf_dict_exe = 
-	&util::filename_cat ($exedir, "mgpp_invf_dict$exe");
+	&FileUtils::filenameConcatenate($exedir, "mgpp_invf_dict$exe");
     my $mgpp_stem_idx_exe =
-	&util::filename_cat ($exedir, "mgpp_stem_idx$exe");
+	&FileUtils::filenameConcatenate($exedir, "mgpp_stem_idx$exe");
 
     my $maxnumeric = $self->{'maxnumeric'};
@@ -595,5 +596,5 @@
 
 	# remove unwanted files
-	my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);
+	my $tmpdir = &FileUtils::filenameConcatenate($self->{'build_dir'}, $indexdir);
 	opendir (DIR, $tmpdir) || die
 	    "mgppbuilder::build_index - couldn't read directory $tmpdir\n";
@@ -604,5 +605,5 @@
 		# delete it!
 		print $outhandle "deleting $file\n" if $self->{'verbosity'} > 2;
-		#&util::rm (&util::filename_cat ($tmpdir, $file));
+		#&util::rm (&FileUtils::filenameConcatenate($tmpdir, $file));
 	    }
 	}
Index: /main/trunk/greenstone2/perllib/plugins/ArchivesInfPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/ArchivesInfPlugin.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugins/ArchivesInfPlugin.pm	(revision 27306)
@@ -31,4 +31,5 @@
 
 use util;
+use FileUtils;
 use doc;
 use PrintInfo;
@@ -120,5 +121,5 @@
 		my $base_dir =$self->{'base_dir'};
 
-		my $doc_filename = &util::filename_cat($base_dir,$doc_file);
+		my $doc_filename = &FileUtils::filenameConcatenate($base_dir,$doc_file);
 
 		my ($doc_tailname, $doc_dirname, $suffix) 
@@ -229,6 +230,6 @@
     
     # see if this has a archives information file within it
-##    my $archive_info_filename = &util::filename_cat($base_dir,$file,"archives.inf");
-    my $archive_info_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", &util::filename_cat($base_dir, $file));
+##    my $archive_info_filename = &FileUtils::filenameConcatenate($base_dir,$file,"archives.inf");
+    my $archive_info_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", &FileUtils::filenameConcatenate($base_dir, $file));
 
     if (-e $archive_info_filename) {
@@ -254,5 +255,5 @@
 	    last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs);
 
-	    my $tmp = &util::filename_cat ($file, $subfile->[0]);
+	    my $tmp = &FileUtils::filenameConcatenate($file, $subfile->[0]);
 	    next if $tmp eq $file;
 
Index: /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 27306)
@@ -39,4 +39,6 @@
 use ghtml;
 use gsprintf 'gsprintf';
+use util;
+use FileUtils;
 
 use PrintInfo;
@@ -447,8 +449,8 @@
 	return if $coverfile eq $filename;
 	
-	if (!&util::fd_exists($coverfile)) {
+	if (!&FileUtils::fileExists($coverfile)) {
 	    $coverfile =~ s/jpg$/JPG/;
 	} 	
-	if (&util::fd_exists($coverfile)) {
+	if (&FileUtils::fileExists($coverfile)) {
 	    &util::block_filename($block_hash,$coverfile);
 	} 
@@ -1418,5 +1420,5 @@
     my $top_section=$doc_obj->get_top_section();
 
-    if (&util::fd_exists($upgraded_filename)) {
+    if (&FileUtils::fileExists($upgraded_filename)) {
 	$doc_obj->associate_source_file($filename);
     	$doc_obj->associate_file($filename, "cover.jpg", "image/jpeg");
@@ -1429,5 +1431,5 @@
 	$upgraded_upper_filename =~ s/jpg$/JPG/;
 
-	if (&util::fd_exists($upgraded_upper_filename)) {
+	if (&FileUtils::fileExists($upgraded_upper_filename)) {
 	    $doc_obj->associate_source_file($upper_filename);
 	    $doc_obj->associate_file($upper_filename, "cover.jpg",
Index: /main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm	(revision 27306)
@@ -34,4 +34,5 @@
 use plugin;
 use util;
+use FileUtils;
 use metadatautil;
 
@@ -294,5 +295,5 @@
     # that it is not explicitly blocked.
     my $dirname = $file;
-    $dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+    $dirname = &FileUtils::filenameConcatenate($base_dir, $file) if $base_dir =~ /\w/;
 
     my $directory_ok = $self->check_directory_path($dirname);
@@ -329,5 +330,5 @@
 
 	my $this_file_base_dir = $base_dir;
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+	my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
 
 	# Recursively read each $raw_subfile
@@ -381,5 +382,5 @@
     } else {
 	$dirname = $file;
-	$dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+	$dirname = &FileUtils::filenameConcatenate($base_dir, $file) if $base_dir =~ /\w/;
     }
 	
@@ -412,5 +413,5 @@
     # Re-order the files in the list so any directories ending with .all are moved to the end
     for (my $i = scalar(@dir) - 1; $i >= 0; $i--) {
-	if (-d &util::filename_cat($dirname, $dir[$i]) && $dir[$i] =~ /\.all$/) {
+	if (-d &FileUtils::filenameConcatenate($dirname, $dir[$i]) && $dir[$i] =~ /\.all$/) {
 	    push(@dir, splice(@dir, $i, 1));
 	}
@@ -461,6 +462,6 @@
 	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
 
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
-	my $raw_full_filename = &util::filename_cat($this_file_base_dir, $raw_file_subfile);
+	my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
+	my $raw_full_filename = &FileUtils::filenameConcatenate($this_file_base_dir, $raw_file_subfile);
 
 	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
@@ -542,7 +543,7 @@
 	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
 
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+	my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
 	my $raw_full_filename 
-	    = &util::filename_cat($this_file_base_dir,$raw_file_subfile);
+	    = &FileUtils::filenameConcatenate($this_file_base_dir,$raw_file_subfile);
 
 	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
@@ -554,5 +555,5 @@
 	if ($raw_subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
 	    require Win32::Shortcut;
-	    my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $raw_subfile));
+	    my $shortcut = new Win32::Shortcut(&FileUtils::filenameConcatenate($dirname, $raw_subfile));
 	    if ($shortcut) {
 		# The file to be processed is now the target of the shortcut
Index: /main/trunk/greenstone2/perllib/plugins/GreenstoneXMLPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/GreenstoneXMLPlugin.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugins/GreenstoneXMLPlugin.pm	(revision 27306)
@@ -36,4 +36,6 @@
 
 use ReadXMLFile;
+use util;
+use FileUtils;
 
 use strict;
@@ -118,5 +120,5 @@
 
 		my $dirname = dirname($filename_full_path);
-		my $full_gsdl_assoc_filename = &util::filename_cat($dirname,$gsdl_assoc_file);
+		my $full_gsdl_assoc_filename = &FileUtils::filenameConcatenate($dirname,$gsdl_assoc_file);
 		if ($self->{'verbosity'}>2) {
 		    print $outhandle "  Storing block list item: $full_gsdl_assoc_filename\n";
@@ -316,5 +318,5 @@
           {
             # And if not, append it so as to make this absolute
-            $assoc_filepath = &util::filename_cat($self->{'base_dir'}, $assoc_filepath);
+            $assoc_filepath = &FileUtils::filenameConcatenate($self->{'base_dir'}, $assoc_filepath);
           }
       }
@@ -331,7 +333,7 @@
         #rint STDERR "mime_type: $mime_type\n";
         #rint STDERR "dir: $dir\n";
-	my $real_dir = &util::filename_cat($assoc_filepath, $assoc_file),
+	my $real_dir = &FileUtils::filenameConcatenate($assoc_filepath, $assoc_file),
 	my $assoc_dir = (defined $dir && $dir ne "") 
-	    ? &util::filename_cat($dir, $assoc_file) : $assoc_file;
+	    ? &FileUtils::filenameConcatenate($dir, $assoc_file) : $assoc_file;
 	$self->{'doc_obj'}->associate_file($real_dir, $assoc_dir, $mime_type);
         #rint STDERR "According to me the real assoc_filepath is: $real_dir\n";
Index: /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm	(revision 27306)
@@ -44,4 +44,5 @@
 use unicode;
 use util;
+use FileUtils;
 use XMLParser;
 
@@ -229,5 +230,5 @@
 	    # Turn relative file path into full path
 	    my $dirname = &File::Basename::dirname($filename_full_path);
-	    $raw_link = &util::filename_cat($dirname, $raw_link);
+	    $raw_link = &FileUtils::filenameConcatenate($dirname, $raw_link);
 	}
 	$raw_link = $self->eval_dir_dots($raw_link);
@@ -336,5 +337,5 @@
 	
 	# set the file to be tidied
-	$input_filename = &util::filename_cat($base_dir,$file) if $base_dir =~ m/\w/;
+	$input_filename = &FileUtils::filenameConcatenate($base_dir,$file) if $base_dir =~ m/\w/;
 	
 	# get the tidied file
@@ -386,5 +387,5 @@
     }
 
-    my $filename = &util::filename_cat($base_dir,$file);
+    my $filename = &FileUtils::filenameConcatenate($base_dir,$file);
     my $upgraded_base_dir = &util::upgrade_if_dos_filename($base_dir);
     my $upgraded_filename = &util::upgrade_if_dos_filename($filename);
@@ -709,5 +710,5 @@
 	if(scalar(@styleTagsText) > 0)
 	{
-		my $css_filename_dirname = &File::Basename::dirname(&util::filename_cat($base_dir, $file));
+		my $css_filename_dirname = &File::Basename::dirname(&FileUtils::filenameConcatenate($base_dir, $file));
 		foreach my $styleText (@styleTagsText)
 		{
@@ -823,5 +824,5 @@
 		my $directory = &File::Basename::dirname($file);
 		
-		my $css_filename = &util::filename_cat($base_dir, $directory, $actual_link);	
+		my $css_filename = &FileUtils::filenameConcatenate($base_dir, $directory, $actual_link);	
 		$self->retrieve_css_associated_files($css_filename);
 	}
@@ -943,5 +944,5 @@
 		$img_url =~ s/^\s*"?([^"]*)"?\s*$/$1/;
 
-		$self->{'css_assoc_files'}->{&util::filename_cat($css_filename_dirname, $img_url)} = $img_url;
+		$self->{'css_assoc_files'}->{&FileUtils::filenameConcatenate($css_filename_dirname, $img_url)} = $img_url;
 	}
 }
@@ -980,5 +981,5 @@
     }
 
-    $filename = &util::filename_cat($base_dir, $filename);
+    $filename = &FileUtils::filenameConcatenate($base_dir, $filename);
 
     if (($self->{'use_realistic_book'}) || ($self->{'old_style_HDL'})) {
@@ -1035,5 +1036,6 @@
     # some special processing if the intended filename was converted to utf8, but
     # the actual file still needs to be renamed
-    if (!&util::fd_exists($filename)) {
+    #if (!&util::fd_exists($filename)) {
+    if (!&FileUtils::fileExists($filename)) {
 	# try the original filename stored in map
 	if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
@@ -1131,10 +1133,10 @@
 	    # Turn url (using /) into file name (possibly using \ on windows)
 	    my @http_dir_split = split('/', $before_hash);
-	    $before_hash = &util::filename_cat(@http_dir_split);
+	    $before_hash = &FileUtils::filenameConcatenate(@http_dir_split);
 	}
 
 	$before_hash = $self->eval_dir_dots($before_hash);
 	
-	my $linkfilename = &util::filename_cat ($base_dir, $before_hash);
+	my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash);
 	
 	my $rl = 0;
@@ -1158,5 +1160,5 @@
 		my @dirs = split /[\/\\]/, $file;
 		my $domname = shift (@dirs);
-		$before_hash = &util::filename_cat($domname, $before_hash);
+		$before_hash = &FileUtils::filenameConcatenate($domname, $before_hash);
 		$before_hash =~ s@\\@/@g; # for windows
 	    }
@@ -1181,5 +1183,5 @@
 		    $before_hash = "/$before_hash"; 
 
-		    $before_hash = &util::filename_cat("",$before_hash);
+		    $before_hash = &FileUtils::filenameConcatenate("",$before_hash);
 		    $before_hash =~ s@^$base_dir/@@;
 		}
@@ -1188,9 +1190,9 @@
 	    # Turn relative file path into full path
 	    my $dirname = &File::Basename::dirname($file);
-	    $before_hash = &util::filename_cat($dirname, $before_hash);
+	    $before_hash = &FileUtils::filenameConcatenate($dirname, $before_hash);
 	    $before_hash = $self->eval_dir_dots($before_hash);	    	    
 	}
 
-	my $linkfilename = &util::filename_cat ($base_dir, $before_hash); 
+	my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash); 
 
 
@@ -1462,5 +1464,5 @@
     # Need to fiddle with number of elements in @eval_dirs if the
     # first one is the empty string.  This is because of a
-    # modification to util::filename_cat that supresses the addition
+    # modification to FileUtils::filenameConcatenate that supresses the addition
     # of a leading '/' character (or \ if windows) (intended to help
     # filename cat with relative paths) if the first entry in the
@@ -1474,5 +1476,5 @@
     }
     
-    my $evaluated_filename = (scalar @eval_dirs > 0) ? &util::filename_cat(@eval_dirs) : "";
+    my $evaluated_filename = (scalar @eval_dirs > 0) ? &FileUtils::filenameConcatenate(@eval_dirs) : "";
     return $evaluated_filename;
 }
@@ -1756,5 +1758,5 @@
     
     # softlink to collection tmp dir
-    my $tmp_dirname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tidytmp");
+    my $tmp_dirname = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, "tidytmp");
     &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
     
@@ -1771,5 +1773,5 @@
 	{
 	    my $folderdirname = $`;
-	    $tmp_dirname = &util::filename_cat($tmp_dirname,$folderdirname);
+	    $tmp_dirname = &FileUtils::filenameConcatenate($tmp_dirname,$folderdirname);
 	    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
 	    $test_dirname = $'; #'
@@ -1777,5 +1779,5 @@
     }
     
-    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
+    my $tmp_filename = &FileUtils::filenameConcatenate($tmp_dirname, "$tailname$suffix");
     
     # tidy or convert the input file if it is a HTML-like file or it is accepted by the process_exp
@@ -1786,5 +1788,5 @@
     	if ($self->{'old_style_HDL'})
     	{
-	    $hdl_output_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
+	    $hdl_output_filename = &FileUtils::filenameConcatenate($tmp_dirname, "$tailname$suffix");
 	    $hdl_output_filename = $self->convert_to_newHDLformat($input_filename,$hdl_output_filename);
     	}
@@ -1797,6 +1799,6 @@
 	foreach my $file (@files) 
 	{
-	    my $src_file = &util::filename_cat($base_dirname,$file);
-	    my $dest_file = &util::filename_cat($tmp_dirname,$file);
+	    my $src_file = &FileUtils::filenameConcatenate($base_dirname,$file);
+	    my $dest_file = &FileUtils::filenameConcatenate($tmp_dirname,$file);
 	    if ((!-e $dest_file) && (!-d $src_file))
 	    {
@@ -1810,5 +1812,5 @@
 	if ($self->{'use_realistic_book'})
 	{
-	    $tidy_output_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
+	    $tidy_output_filename = &FileUtils::filenameConcatenate($tmp_dirname, "$tailname$suffix");
 	    $tidy_output_filename = $self->tmp_tidy_file($hdl_output_filename,$tidy_output_filename);
 	}
@@ -1858,5 +1860,5 @@
 
 	    # get the full path to the image
-	    my $img_file = &util::filename_cat($dirname,$attr->{src});
+	    my $img_file = &FileUtils::filenameConcatenate($dirname,$attr->{src});
 	    
 	    # set the width and height attribute
Index: /main/trunk/greenstone2/perllib/plugins/MetadataXMLPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/MetadataXMLPlugin.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugins/MetadataXMLPlugin.pm	(revision 27306)
@@ -95,4 +95,5 @@
 use extrametautil;
 use util;
+use FileUtils;
 use metadatautil;
 
@@ -216,5 +217,5 @@
     my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
 	
-    my $filename_full_path = &util::filename_cat($base_dir, $file);
+    my $filename_full_path = &FileUtils::filenameConcatenate($base_dir, $file);
     return undef unless $self->can_process_this_file($filename_full_path);    
 
@@ -244,5 +245,5 @@
 	$processor, $gli, $aux) = @_;
 
-    my $filename = &util::filename_cat($base_dir, $file);
+    my $filename = &FileUtils::filenameConcatenate($base_dir, $file);
     return undef unless $self->can_process_this_file($filename);    
 	
Index: /main/trunk/greenstone2/perllib/plugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugout.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugout.pm	(revision 27306)
@@ -31,4 +31,5 @@
 
 require util;
+use FileUtils;
 use gsprintf 'gsprintf';
 
@@ -44,37 +45,108 @@
 my ($verbosity, $outhandle, $failhandle, $globaloptions);
 
+# - significantly rewritten to support plugouts in extensions [jmt12]
 sub load_plugout{
     my ($plugout) = shift @_;
+    my $plugout_name = shift @$plugout;
+    my $plugout_suffix = &FileUtils::filenameConcatenate('perllib', 'plugouts', $plugout_name . '.pm');
+    my $plugout_found = 0;
 
-    my $colplugdir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/plugouts");
-    unshift (@INC, $colplugdir); 
-    
-    my $plugoutname = shift @$plugout;
+    # add collection plugout directory to INC unless it is already there
+    my $colplugdir = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'},'perllib','plugouts');
+    if (-d $colplugdir)
+    {
+      my $found_colplugdir = 0;
+      foreach my $path (@INC)
+      {
+        if ($path eq $colplugdir)
+        {
+          $found_colplugdir = 1;
+          last;
+        }
+      }
+      if (!$found_colplugdir)
+      {
+        unshift (@INC, $colplugdir);
+      }
+    }
 
-    # find the plugout
-    my $customplugname;
+    # To find the plugout we check a number of possible locations
+    # - any plugout found in the collection itself is our first choice, with
+    #   those located in 'custom' having precedence
     if (defined($ENV{'GSDLCOLLECTION'}))
     {
-        $customplugname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "custom", $ENV{'GSDLCOLLECTION'}, 
-						 'perllib', 'plugouts', "${plugoutname}.pm");
+      my $collect_dir = $ENV{'GSDLCOLLECTION'};
+
+      # (needed for Veridian?)
+      my $custom_plugout = &FileUtils::filenameConcatenate($collect_dir, "custom", $collect_dir, $plugout_suffix);
+      if (&FileUtils::fileExists($custom_plugout))
+      {
+        require $custom_plugout;
+        $plugout_found = 1;
+      }
+      else
+      {
+        # typical collection override
+        my $collection_plugout = &FileUtils::filenameConcatenate($collect_dir, $plugout_suffix);
+        if (&FileUtils::fileExists($collection_plugout))
+        {
+          require $custom_plugout;
+          $plugout_found = 1;
+        }
+      }
+    }
+    # - we then search for overridden version provided by any registered GS3
+    #   extensions (check in order of extension definition)
+    if (!$plugout_found && defined $ENV{'GSDL3EXTS'})
+    {
+      my $ext_prefix = &FileUtils::filenameConcatenate($ENV{'GSDL3SRCHOME'}, "ext");
+      my @extensions = split(/:/,$ENV{'GSDL3EXTS'});
+      foreach my $e (@extensions)
+      {
+        my $extension_plugout = &FileUtils::filenameConcatenate($ext_prefix, $e, $plugout_suffix);
+        if (&FileUtils::fileExists($extension_plugout))
+        {
+          require $extension_plugout;
+          $plugout_found = 1;
+        }
+      }
+    }
+    # - similar for GS2 extensions
+    if (!$plugout_found && defined $ENV{'GSDLEXTS'})
+    {
+      my $ext_prefix = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "ext");
+      my @extensions = split(/:/,$ENV{'GSDLEXTS'});
+      foreach my $e (@extensions)
+      {
+        my $extension_plugout = &FileUtils::filenameConcatenate($ext_prefix, $e, $plugout_suffix);
+        if (&FileUtils::fileExists($extension_plugout))
+        {
+          require $extension_plugout;
+          $plugout_found = 1;
+        }
+      }
+    }
+    # - and the default is the main Greenstone version of the plugout
+    if (!$plugout_found)
+    {
+      my $main_plugout = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, $plugout_suffix);
+      if (&FileUtils::fileExists($main_plugout))
+      {
+        require $main_plugout;
+        $plugout_found = 1;
+      }
     }
 
-    my $colplugname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 'perllib', 'plugouts', 
-					  "${plugoutname}.pm");
-    my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'}, 'perllib', 'plugouts',
-					  "${plugoutname}.pm");
-    if (defined($customplugname) && -e $customplugname) { require $customplugname; }
-    elsif (-e $colplugname) { require $colplugname; }
-    elsif (-e $mainplugname) { require $mainplugname; }
-    else {
-	gsprintf($outhandle, "{plugout.could_not_find_plugout}\n",
-		 $plugoutname);
-	    die "\n";
+    # - no plugout found with this name
+    if (!$plugout_found)
+    {
+      gsprintf($outhandle, "{plugout.could_not_find_plugout}\n", $plugout_name);
+      die "\n";
     }
 
-    # create a plugout object
+    # - create a plugout object
     my ($plugobj);
 
-    eval ("\$plugobj = new \$plugoutname([],\$plugout)");
+    eval ("\$plugobj = new \$plugout_name([],\$plugout)");
     die "$@" if $@;
 
Index: /main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 27306)
@@ -36,5 +36,6 @@
 use printusage;
 use parse2;
-
+use util;
+use FileUtils;
 
 # suppress the annoying "subroutine redefined" warning that various
@@ -509,6 +510,6 @@
 	    my $output_dir = $self->get_output_dir();
 	    &util::mk_all_dir ($output_dir) unless -e $output_dir;
-	    my $doc_file = &util::filename_cat ($output_dir, $doc_dir, "doc.xml");
-	    my $short_doc_file = &util::filename_cat ($doc_dir, "doc.xml");
+	    my $doc_file = &FileUtils::filenameConcatenate($output_dir, $doc_dir, "doc.xml");
+	    my $short_doc_file = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");
 	    
 	    if ($gs_count>0)
@@ -590,5 +591,5 @@
 
     if (!defined $self->{'group'} || !$self->{'group'}){
-	&util::mk_all_dir (&util::filename_cat ($working_dir, $doc_dir));
+	&FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir));
     }
 
@@ -619,9 +620,9 @@
        }
    } while ($doc_dir_rest ne "" && 
-	    ((-d &util::filename_cat ($working_dir, "$doc_dir.dir")) || 
+	    ((-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) || 
 	     ($working_info->size() >= 1024 && $doc_dir_num < 2)));
    my $i = 1;
    my $doc_dir_base = $doc_dir;
-   while (-d &util::filename_cat ($working_dir, "$doc_dir.dir")) {
+   while (-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) {
        $doc_dir = "$doc_dir_base-$i";
        $i++;
@@ -642,5 +643,5 @@
     &util::mk_all_dir ($output_dir) unless -e $output_dir;
       
-    my $working_dir = &util::filename_cat($output_dir, $doc_dir);
+    my $working_dir = &FileUtils::filenameConcatenate($output_dir, $doc_dir);
     &util::mk_all_dir ($working_dir) unless -e $working_dir;
 
@@ -662,5 +663,5 @@
      
 	if ($source_filename !~ /^$dirsep_regexp/) {
-	    $source_filename = &util::filename_cat($collect_dir, $source_filename);
+	    $source_filename = &FileUtils::filenameConcatenate($collect_dir, $source_filename);
 	}
     }
@@ -686,8 +687,8 @@
 	if (-e $real_filename) {
 
-	    $filename = &util::filename_cat($working_dir, $afile);
-
-	    &util::hard_link ($real_filename, $filename, $self->{'verbosity'});
-	    
+	    $filename = &FileUtils::filenameConcatenate($working_dir, $afile);
+
+            &FileUtils::hardLink($real_filename, $filename, $self->{'verbosity'});
+
 	    $doc_obj->add_utf8_metadata ($doc_obj->get_top_section(),
 					 "gsdlassocfile",
@@ -815,5 +816,5 @@
 	}
 
-	my $rss_filename = &util::filename_cat($output_dir,"rss-items.rdf");
+	my $rss_filename = &FileUtils::filenameConcatenate($output_dir,"rss-items.rdf");
 	if (open(RSSOUT,">>$rss_filename")) {
 	    print RSSOUT "<item>\n";
@@ -875,10 +876,10 @@
     return unless defined $xslt_file and $xslt_file ne "" and -e $xslt_file;
     
-    my $java_class_path =  &util::filename_cat ($ENV{'GSDLHOME'},"bin","java","ApplyXSLT.jar");
+    my $java_class_path =  &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin","java","ApplyXSLT.jar");
 
     my $mapping_file_path = "";
 
     if ($ENV{'GSDLOS'} eq "windows"){
-	$java_class_path .=";".&util::filename_cat ($ENV{'GSDLHOME'},"bin","java","xalan.jar");
+	$java_class_path .=";".&FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin","java","xalan.jar");
 	# this file:/// bit didn't work for me on windows XP
 	#$xslt_file = "\"file:///".$xslt_file."\"";
@@ -886,5 +887,5 @@
     }
     else{
-	$java_class_path .=":".&util::filename_cat ($ENV{'GSDLHOME'},"bin","java","xalan.jar");
+	$java_class_path .=":".&FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"bin","java","xalan.jar");
     }
 
Index: /main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm	(revision 27306)
@@ -32,4 +32,5 @@
 eval {require bytes};
 use util;
+use FileUtils;
 use BasePlugout;
 use docprint;
@@ -72,5 +73,5 @@
 	&util::mk_all_dir ($output_dir) unless -e $output_dir;
 	
-	my $working_dir = &util::filename_cat ($output_dir, $doc_dir);    
+	my $working_dir = &FileUtils::filenameConcatenate($output_dir, $doc_dir);    
 	&util::mk_all_dir ($working_dir) unless -e $working_dir;
 	
@@ -79,5 +80,5 @@
 	$self->process_metafiles_metadata ($doc_obj);
 
-	my $output_file = util::filename_cat ($working_dir, "doc.xml");
+	my $output_file = &FileUtils::filenameConcatenate($working_dir, "doc.xml");
 	
 	$self->open_xslt_pipe($output_file, $self->{'xslt_file'});
@@ -107,5 +108,5 @@
 	}
 	
-	$self->{'short_doc_file'} = util::filename_cat ($doc_dir, "doc.xml");  
+	$self->{'short_doc_file'} = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");  
 	
 	$self->store_output_info_reference($doc_obj);
Index: /main/trunk/greenstone2/perllib/unicode.pm
===================================================================
--- /main/trunk/greenstone2/perllib/unicode.pm	(revision 27305)
+++ /main/trunk/greenstone2/perllib/unicode.pm	(revision 27306)
@@ -36,4 +36,5 @@
 use strict;
 use util;
+use FileUtils;
 use MIME::Base64; # for base64 encoding
 
@@ -247,5 +248,5 @@
     my $encodename = "$encoding-unicode";
     my $enc_info = $encodings::encodings->{$encoding};
-    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, "mappings",
+    my $mapfile = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "mappings",
 				      "to_uc", $enc_info->{'mapfile'});
     if (!&loadmapencoding ($encodename, $mapfile)) {
@@ -499,5 +500,5 @@
 
     my $enc_info = $encodings::encodings->{$encoding};
-    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, "mappings",
+    my $mapfile = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "mappings",
 				      "from_uc", $enc_info->{'mapfile'});
     if (!&loadmapencoding ($encodename, $mapfile)) {
