Index: /gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl	(revision 27643)
+++ /gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl	(revision 27644)
@@ -6,5 +6,5 @@
 BEGIN
 {
-  die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+  die "GSDLHOME not set\n" unless (defined $ENV{'GSDLHOME'} && $ENV{'GSDLHOME'} ne '');
   die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
   die "GEXTPARALLELBUILDING not set\n" unless defined $ENV{'GEXTPARALLELBUILDING'};
@@ -19,7 +19,10 @@
 my $collection = 'test';
 my $use_thrift = 1;
-my $start_thrift = 1;
+my $start_thrift = 0;
 my $debug = 0;
 my $dry_run = 0;
+my $flush_diskcache = 0;
+my $use_nfs = 0;
+
 my $gsdl_home = $ENV{'GSDLHOME'};
 my $gsdl_hadoop_ext = $ENV{'GEXTPARALLELBUILDING_INSTALLED'};
@@ -35,5 +38,5 @@
 
 # 1. Read and validate parameters
-if (defined $ARGV[0])
+if (defined $ARGV[0] && $ARGV[0] =~ /^[a-z0-9]+$/i)
 {
   $collection = $ARGV[0];
@@ -41,5 +44,6 @@
 else
 {
-  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-dry_run] [-start_thrift] [-disable_thrift] [-refresh_import] \n\n";
+  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-dry_run] [-start_thrift] [-disable_thrift] [-refresh_import] [-flush_diskcache] [-use_nfs]\n\n";
+  exit;
 }
 my $offset = 1;
@@ -62,4 +66,16 @@
     $refresh_import = 1;
   }
+  if ($ARGV[$offset] eq '-flush_diskcache')
+  {
+    $flush_diskcache = 1;
+  }
+  if ($ARGV[$offset] eq '-start_thrift')
+  {
+    $start_thrift = 1;
+  }
+  if ($ARGV[$offset] eq '-use_nfs')
+  {
+    $use_nfs = 1;
+  }
   $offset++;
 }
@@ -68,4 +84,8 @@
 {
   $hdfs_fs_prefix = 'HDFSShell://';
+}
+if ($use_nfs)
+{
+  $hdfs_fs_prefix = '/hdfs';
 }
 
@@ -87,13 +107,34 @@
 }
 # - directories within HDFS
-#my $hdfs_input_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'import');
-my $hdfs_input_dir = &urlCat('hdfs://', 'user', $username, 'gsdl', 'collect', $collection, 'import');
-#my $hdfs_output_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'archives');
-my $hdfs_output_dir = &urlCat('hdfs://', 'user', $username, 'gsdl', 'collect', $collection, 'archives');
+my $hdfs_input_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'import');
+print "HDFS Import Directory: " . $hdfs_input_dir . "\n";
+my $nfs_input_dir = &urlCat('/hdfs', 'user', $username, 'gsdl', 'collect', $collection, 'import');
+if ($use_nfs)
+{
+  print "=> NFS Import Directory: " . $nfs_input_dir . "\n";
+}
+my $hdfs_output_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'archives');
+print "HDFS Archives Directory: " . $hdfs_output_dir . "\n";
+my $nfs_output_dir = &urlCat('/hdfs', 'user', $username, 'gsdl', 'collect', $collection, 'archives');
+if ($use_nfs)
+{
+  print "=> NFS Archives Directory: " . $nfs_output_dir . "\n";
+}
 
 # 2. Copy the import directory into HDFS
 print " * Replicating import directory in HDFS...";
 # - check if import directory already exists
-my $hdfs_import_exists = &hdfsTest('d', 0, $hdfs_input_dir);
+my $hdfs_import_exists = 0;
+if ($use_nfs)
+{
+  if (-d $nfs_input_dir)
+  {
+    $hdfs_import_exists = 1;
+  }
+}
+else
+{
+  $hdfs_import_exists = &hdfsTest('d', 0, $hdfs_input_dir);
+}
 if ($refresh_import || !$hdfs_import_exists)
 {
@@ -101,9 +142,24 @@
   if ($hdfs_import_exists)
   {
-    &hdfsCommand('rmr', $hdfs_input_dir);
+    if ($use_nfs)
+    {
+      &recursiveDelete($nfs_input_dir, '/hdfs');
+    }
+    else
+    {
+      &hdfsCommand('rmr', $hdfs_input_dir);
+    }
   }
   # - now recursively copy the contents of import directory into HDFS ensuring
   #   that relative paths are maintained
-  my $file_count = &recursiveCopy($gs_import_dir, $hdfs_input_dir);
+  my $file_count = 0;
+  if ($use_nfs)
+  {
+    $file_count = &recursiveCopy($gs_import_dir, $nfs_input_dir);
+  }
+  else
+  {
+    $file_count = &recursiveCopy($gs_import_dir, $hdfs_input_dir);
+  }
   &debugPrint($file_count . " files 'putted'");
   print "Done!\n";
@@ -113,4 +169,5 @@
   print "Already exists!\n";
 }
+
 # - clear out the archives regardless
 my $gs_archives_dir = $gs_collection_dir . '/archives';
@@ -119,9 +176,21 @@
 {
   print " * Clearing existing archives directory for this collection... ";
-  &shellCommand('rm -rf "' . $gs_archives_dir . '"');
+  &recursiveDelete($gs_archives_dir, $gsdl_home);
   $deleted_archives = 1;
 }
 mkdir($gs_archives_dir, 0755);
-if (&hdfsTest('d', 0, $hdfs_output_dir))
+my $hdfs_archives_exists = 0;
+if ($use_nfs)
+{
+  if (-d $nfs_output_dir)
+  {
+    $hdfs_archives_exists = 1;
+  }
+}
+else
+{
+  $hdfs_archives_exists = &hdfsTest('d', 0, $hdfs_output_dir)
+}
+if ($hdfs_archives_exists)
 {
   if (!$deleted_archives)
@@ -129,5 +198,12 @@
     print " * Clearing existing archives directory for this collection... ";
   }
-  &hdfsCommand('rmr', $hdfs_output_dir);
+  if ($use_nfs)
+  {
+    &recursiveDelete($nfs_output_dir, '/hdfs');
+  }
+  else
+  {
+    &hdfsCommand('rmr', $hdfs_output_dir);
+  }
   $deleted_archives = 1;
 }
@@ -136,4 +212,5 @@
   print "Done!\n";
 }
+
 # - watch for cached directories for Media based collections
 my $gs_cached_dir = $gs_collection_dir . '/cached';
@@ -141,5 +218,5 @@
 {
   print " * Clearing existing cached media directory for this collection... ";
-  &shellCommand('rm -rf "' . $gs_cached_dir . '"');
+  &recursiveDelete($gs_cached_dir, $gsdl_home);
   print "Done!\n";
 }
@@ -150,5 +227,5 @@
 if (!&dirIsEmpty($gs_logs_dir))
 {
-  &shellCommand('rm -f ' . $gs_logs_dir . '/*.*');
+  &recursiveDelete($gs_logs_dir . '/*.*', $gsdl_home);
 }
 if (!&dirIsEmpty('/tmp/greenstone'))
@@ -156,4 +233,5 @@
   &shellCommand('rm -f /tmp/greenstone/*.*');
   &shellCommand('rm -rf /tmp/gsimport*');
+  &shellCommand('rm -rf /tmp/thrift');
 }
 if ($is_rocks_cluster)
@@ -161,15 +239,19 @@
   &shellCommand('rocks run host "rm -f /tmp/greenstone/*.*"');
   &shellCommand('rocks run host "rm -rf /tmp/gsimport*"');
+  &shellCommand('rocks run host "rm -rf /tmp/thrift"');
 }
 print "Done!\n";
 
 # - flush DNS cache too, so we are playing on a level field
-print " * Flushing disk cache... ";
-&shellCommand('flush_caches.pl');
-if ($is_rocks_cluster)
-{
-  &shellCommand('rocks run host "flush_caches.pl"');
-}
-print "Done!\n";
+if ($flush_diskcache)
+{
+  print " * Flushing disk cache... ";
+  &shellCommand('flush_caches.pl');
+  if ($is_rocks_cluster)
+  {
+    &shellCommand('rocks run host "flush_caches.pl"');
+  }
+  print "Done!\n";
+}
 
 # 3. Special case for *Server type infodbs (namely TDBServer and GDBMServer)
@@ -218,5 +300,5 @@
   {
     print " * Starting Thrift Servers (on compute nodes)... ";
-    &shellCommand('rocks run host "cd ' . $ENV{'GEXTPARALLELBUILDING'} . '/packages/ThriftFS-0.9.0/bin && ./thriftctl.sh start"');
+    print "[DEBUG]\n" . &shellCommand('rocks run host "cd ' . $ENV{'GEXTPARALLELBUILDING'} . '/packages/ThriftFS-0.9.0/bin && ./thriftctl.sh start"') . "\n\n";
   }
   # single server
@@ -227,4 +309,15 @@
   }
   print "Done!\n";
+}
+
+my $actual_archives_dir;
+if ($use_nfs)
+{
+  $actual_archives_dir = $nfs_output_dir;
+}
+else
+{
+  $actual_archives_dir = $hdfs_output_dir;
+  $actual_archives_dir =~ s/hdfs:\/\//$hdfs_fs_prefix/;
 }
 
@@ -236,6 +329,14 @@
 print " * Running import using Hadoop...";
 my $hadoop_log = $gs_results_dir . '/hadoop.log';
-&shellCommand('echo "host:$HDFSHOST" > ' . $hadoop_log);
-my $hadoop_command = $hadoop_exe . ' jar ' . $gsdl_hadoop_ext . '/lib/hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest "' . $gsdl_home . '" "' . $hdfs_fs_prefix . '" "' . $ENV{'HADOOP_PREFIX'} . '" ' . $collection . ' "' .  $hdfs_input_dir . '" "' . $hdfs_output_dir . '" >> ' . $hadoop_log . ' 2>&1';
+&shellCommand('echo "host:' . $ENV{'HDFSHOST'} . '" > ' . $hadoop_log);
+my $hadoop_command = $hadoop_exe . ' jar ' . $gsdl_hadoop_ext . '/lib/hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest ';
+$hadoop_command .= '"' . $gsdl_home . '" '; # Greenstone's home dir
+$hadoop_command .= '"' . $ENV{'HADOOP_PREFIX'} . '" '; # Hadoop's home dir
+$hadoop_command .= $collection . ' '; # The collection name
+$hadoop_command .= '"' . $actual_archives_dir . '" '; # Collection archive dir
+$hadoop_command .= '"' . $hdfs_fs_prefix . '" '; # Prefix for talking to HDFS (driver)
+$hadoop_command .= '"' . $hdfs_input_dir . '" '; # HDFS in
+$hadoop_command .= '"' . $hdfs_output_dir . '" '; # HDFS out
+$hadoop_command .= ' >> ' . $hadoop_log . ' 2>&1'; # Redirect to log
 &shellCommand($hadoop_command);
 print "Done!\n";
@@ -351,6 +452,7 @@
 # /** printUsage() **/
 
-# /**
-#  */
+
+## @function recursiveCopy()
+#
 sub recursiveCopy
 {
@@ -358,5 +460,12 @@
   my $file_count = 0;
   # - create the directory in HDFS
-  &hdfsCommand('mkdir', $hdfs_dir);
+  if ($use_nfs)
+  {
+    &shellCommand('mkdir "' . $hdfs_dir . '"');
+  }
+  else
+  {
+    &hdfsCommand('mkdir', $hdfs_dir);
+  }
   # - search $src_dir for files
   opendir(DH, $src_dir) or die("Error! Cannot open directory for reading: " . $src_dir);
@@ -379,5 +488,12 @@
       {
         my $hdfs_path = $hdfs_dir . '/' . $file;
-        &hdfsCommand('put', $src_path, $hdfs_path);
+        if ($use_nfs)
+        {
+          &shellCommand('cp "' . $src_path . '" "' . $hdfs_path . '"');
+        }
+        else
+        {
+          &hdfsCommand('put', $src_path, $hdfs_path);
+        }
         $file_count++;
       }
@@ -386,5 +502,6 @@
   return $file_count;
 }
-# /** recursiveCopy() **/
+## recursiveCopy() ##
+
 
 # /** @function shellCommand
@@ -427,2 +544,15 @@
 }
 # /** dirIsEmpty() **/
+
+
+## @function recursiveDelete()
+#
+sub recursiveDelete
+{
+  my ($dir, $prefix) = @_;
+  if ($dir =~ /^$prefix/)
+  {
+    &shellCommand('rm -rf "' . $dir . '"');
+  }
+}
+## recursiveDelete() ##
