Changeset 27414 for gs2-extensions/parallel-building

Timestamp:

2013-05-24T09:24:16+12:00 (11 years ago)

Author:

jmt12

Message:

Allowing more processing arguments to be configured at the call, and passing information such as the desired HDFS driver through to Hadoop processing

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl (modified) (7 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl

@@ -16 +16 @@
 # 0. Init
 my $collection = 'test';
+my $use_thrift = 1;
 my $debug = 0;
 my $dry_run = 0;
@@ -22 +23 @@
 my $hadoop_exe = 'hadoop'; # you may add path
 my $cluster_head = $ENV{'HDFSHOST'}; # may not be true on advanced configs
-my $hdfs_fs_prefix = 'hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'};
+my $hdfs_fs_prefix = 'HDThriftFS://';
 my $refresh_import = 0;
 my $username = `whoami`;
@@ -35 +36 @@
   $collection = $ARGV[0];
 }
+else
+{
+  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-dry_run] [-disable_thrift] [-refresh_import] \n\n";
+}
+my $offset = 1;
+while (defined $ARGV[$offset])
+{
+  if ($ARGV[$offset] eq '-debug')
+  {
+    $debug = 1;
+  }
+  if ($ARGV[$offset] eq '-disable_thrift')
+  {
+    $use_thrift = 0;
+  }
+  if ($ARGV[$offset] eq '-dry_run')
+  {
+    $dry_run = 1;
+  }
+  if ($ARGV[$offset] eq '-refresh_import')
+  {
+    $refresh_import = 1;
+  }
+  $offset++;
+}
+
+if (!$use_thrift)
+{
+  $hdfs_fs_prefix = 'HDFSShell://';
+}
+
 my $gs_collection_dir = $gsdl_home . '/collect/' . $collection;
 my $gs_import_dir = $gs_collection_dir . '/import';
@@ -53 +85 @@
 my $gs_archives_dir = $gs_collection_dir . '/archives';
 # - directories within HDFS
-my $hdfs_input_dir = &urlCat($hdfs_fs_prefix, 'user', $username, 'gsdl', 'collect', $collection, 'import');
-my $hdfs_output_dir = &urlCat($hdfs_fs_prefix, 'user', $username, 'gsdl', 'collect', $collection, 'archives');
+my $hdfs_input_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'import');
+my $hdfs_output_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'archives');
 
 # 2. Copy the import directory into HDFS
@@ -103 +135 @@
 # - flush DNS cache too, so we are playing on a level field
 &shellCommand('flush_caches.pl');
-&shellCommand('rocks run host "flush_caches.pl"');
+if ($is_rocks_cluster)
+{
+  &shellCommand('rocks run host "flush_caches.pl"');
+}
 
 # 3. Special case for *Server type infodbs (namely TDBServer and GDBMServer)
@@ -151 +186 @@
 print " * Running import using Hadoop...";
 my $hadoop_log = $gs_results_dir . '/hadoop.log';
-my $hadoop_command = $hadoop_exe . ' jar ' . $gsdl_hadoop_ext . '/lib/hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest "' . $gsdl_home . '" "' . $ENV{'HDFSHOST'} . '" ' . $ENV{'HDFSPORT'} . ' "' . $ENV{'HADOOP_PREFIX'} . '" ' . $collection . " " .  $hdfs_input_dir . " " . $hdfs_output_dir . " > " . $hadoop_log . " 2>&1";
+my $hadoop_command = $hadoop_exe . ' jar ' . $gsdl_hadoop_ext . '/lib/hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest "' . $gsdl_home . '" "' . $hdfs_fs_prefix . '" "' . $ENV{'HADOOP_PREFIX'} . '" ' . $collection . " " .  $hdfs_input_dir . " " . $hdfs_output_dir . " > " . $hadoop_log . " 2>&1";
 &shellCommand($hadoop_command);
 print "Done!\n";
@@ -223 +258 @@
   my $paths = '"' . join('" "', @_) . '"';
   my $hdfs_command = $hadoop_exe . ' fs -' . $command . ' ' . $paths . ' 2>&1';
+  if ($debug)
+  {
+    print STDERR "[DEBUG] $hdfs_command\n";
+  }
   &shellCommand($hdfs_command);
   return $?;