Changeset 30354 for gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl

Timestamp:

2015-12-16T16:15:39+13:00 (8 years ago)

Author:

jmt12

Message:

Extending manifest v2 support to allow for directories to be listed in manifest. Matched with changes in Directory plugin to allow paths into systems like HDFS to be listed in manifest.cd

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl (modified) (4 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl

@@ -33 +33 @@
 my $hdfs_fs_prefix = 'HDThriftFS://';
 my $refresh_import = 0;
+my $remove_old = 0;
 my $username = `whoami`;
 chomp($username);
@@ -49 +50 @@
 else
 {
-  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-disable_thrift] [-dry_run] [-start_thrift] [-refresh_import] [-flush_diskcache] [-use_nfs] [-stagger]\n";
+  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-enable_thrift] [-dry_run] [-start_thrift] [-refresh_import] [-flush_diskcache] [-use_nfs] [-stagger] [-removeold]\n";
   print STDERR "where: [debug] print more debug messages to STDERR\n";
   print STDERR "       [dry_run] don't actually perform an file actions\n";
@@ -61 +62 @@
     $debug = 1;
   }
-  if ($ARGV[$offset] eq '-disable_thrift')
-  {
-    $use_thrift = 0;
+  if ($ARGV[$offset] eq '-enable_thrift')
+  {
+    $use_thrift = 1;
   }
   if ($ARGV[$offset] eq '-dry_run')
@@ -88 +89 @@
   {
     $use_nfs = 1;
+  }
+  if ($ARGV[$offset] eq '-removeold')
+  {
+    $remove_old = 1;
   }
   if ($ARGV[$offset] eq '-logdir')