Changeset 27001

Timestamp:

2013-03-05T11:47:57+13:00 (11 years ago)

Author:

jmt12

Message:

Passing more environment variables (HADOOPPREFIX, HDFSHOST, HDFSPORT) through to hadoop (and thus on to compute nodes). Debug comments. Directory not empty test before attempting to copy log files preventing non-fatal errors. Whether on cluster determined by presence of 'rocks' executable rather than hardcoded hostname (one less thing for me to forget to change between computers)

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl (modified) (11 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl

@@ -8 +8 @@
   die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
   die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+  die "HDFS HOST not set (set in <gsdl>/ext/parallel_processing/setup.bash)\n" unless defined $ENV{'HDFSHOST'};
+  die "HDFS PORT not set (set in <gsdl>/ext/parallel_processing/setup.bash)\n" unless defined $ENV{'HDFSPORT'};
 }
 
@@ -16 +18 @@
 my $debug = 0;
 my $dry_run = 0;
-my $cluster_head = ''; # i.e. 'medusa.local';
-
 my $gsdl_home = $ENV{'GSDLHOME'};
-my $gsdl_hadoop_ext = $ENV{'GEXTHADOOP_INSTALLED'};
+my $gsdl_hadoop_ext = $ENV{'GEXTPARALLELBUILDING_INSTALLED'};
 my $hadoop_exe = 'hadoop'; # you may add path
-my $hdfs_fs_prefix = 'hdfs://localhost:54310';
+my $cluster_head = $ENV{'HDFSHOST'}; # may not be true on advanced configs
+my $hdfs_fs_prefix = 'hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'};
 my $refresh_import = 0;
 my $username = `whoami`;
 chomp($username);
 
+`rocks > /dev/null 2>&1`;
+my $is_rocks_cluster = ($? == 0);
+
 # 1. Read and validate parameters
 if (defined $ARGV[0])
@@ -47 +51 @@
   mkdir($gs_results_dir, 0755);
 }
+my $gs_archives_dir = $gs_collection_dir . '/archives';
 # - directories within HDFS
 my $hdfs_input_dir = &urlCat($hdfs_fs_prefix, 'user', $username, 'gsdl', 'collect', $collection, 'import');
@@ -73 +78 @@
 }
 # - clear out the archives regardless
+if (-e $gs_archives_dir)
+{
+  &shellCommand('rm -rf "' . $gs_archives_dir . '"');
+}
+mkdir($gs_archives_dir, 0755);
 if (&hdfsTest('d', 0, $hdfs_output_dir))
 {
@@ -79 +89 @@
   print "Done!\n";
 }
+# - clear out any old logs
+if (!&dirIsEmpty('/tmp/greenstone'))
+{
+  &shellCommand('rm /tmp/greenstone/*.*');
+}
+if ($is_rocks_cluster)
+{
+  &shellCommand('rocks run host "rm /tmp/greenstone/*.*"');
+}
 
 # 3. Special case for *Server type infodbs (namely TDBServer and GDBMServer)
 #    where we start the server now to ensure it lives on the head node
-my $server_host = 'localhost';
-my $server_port = '8191';
+my $server_host = '';
+my $server_port = '';
 my $configuration_path = $gs_collection_dir . '/etc/collect.cfg';
 my $infodbtype = `grep -P "^infodbtype" $configuration_path`;
@@ -106 +125 @@
     exit;
   }
-  # - use the client tool to add ourselve as a listener
+  # - use the client tool to add ourselves as a listener
   print " * Registering as listener... ";
   my $client_command = $server_prefix . "Client.pl " . $server_host . " " . $server_port . " \"#a:" . $$ . "\"";
@@ -126 +145 @@
 print " * Running import using Hadoop...";
 my $hadoop_log = $gs_results_dir . '/hadoop.log';
-my $hadoop_command = $hadoop_exe . " jar " . $gsdl_hadoop_ext . "/lib/hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest " . $gsdl_home . " " . $collection . " " .  $hdfs_input_dir . " " . $hdfs_output_dir . " > " . $hadoop_log . " 2>&1";
+my $hadoop_command = $hadoop_exe . ' jar ' . $gsdl_hadoop_ext . '/lib/hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest "' . $gsdl_home . '" "' . $ENV{'HDFSHOST'} . '" ' . $ENV{'HDFSPORT'} . ' "' . $ENV{'HADOOP_PREFIX'} . '" ' . $collection . " " .  $hdfs_input_dir . " " . $hdfs_output_dir . " > " . $hadoop_log . " 2>&1";
 &shellCommand($hadoop_command);
 print "Done!\n";
@@ -147 +166 @@
 print " * Gathering logs from compute nodes... ";
 # - local files
-&shellCommand('cp /tmp/greenstone/*.* ' . $gs_results_dir);
+if (!&dirIsEmpty('/tmp/greenstone'))
+{
+  &shellCommand('cp /tmp/greenstone/*.* ' . $gs_results_dir);
+}
 if (-d $gs_collection_dir . '/logs')
 {
@@ -153 +175 @@
 }
 # - remote files
-if ($cluster_head ne '')
-{
-  &shellCommand('rocks run host "scp /tmp/greenstone/*.* ' $cluster_head . ':' . $gs_results_dir . '"');
+if ($is_rocks_cluster)
+{
+  &shellCommand('rocks run host "scp /tmp/greenstone/*.* ' . $cluster_head . ':' . $gs_results_dir . '"');
 &shellCommand('rocks run host "scp /tmp/gsimport-*/logs/*.log ' . $cluster_head . ':' . $gs_results_dir . '"');
 }
@@ -163 +185 @@
 print " * Cleaning up temporary files... ";
 &shellCommand('rm -rf /tmp/greenstone');
-if ($cluster)
+if ($is_rocks_cluster)
 {
   &shellCommand('rocks run host "rm -rf /tmp/greenstone"');
@@ -276 +298 @@
 }
 # /** urlCat() **/
+
+# /**
+#  */
+sub dirIsEmpty
+{
+  my $dir = shift(@_);
+  my @files;
+  if (-e $dir)
+  {
+    opendir(DIR, $dir) or die $!;
+    @files = grep { !m/\A\.{1,2}\Z/} readdir(DIR);
+    closedir(DIR);
+  }
+  @files ? 0 : 1;
+}
+# /** dirIsEmpty() **/