Changeset 27641 for gs2-extensions

Timestamp:

2013-06-18T10:12:53+12:00 (11 years ago)

Author:

jmt12

Message:

Altered order of arguments and allow archives dir to be passed as argument - both of which were needed to support NFS access to HDFS

File:

• gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java (modified) (7 diffs)

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java

@@ -165 +165 @@
       String gsdlhome = conf.get("gsdlhome");
       String hdfs_prefix = conf.get("hdfsprefix");
-      String hadoop_prefix = conf.get("hadoopprefix");
+      String hadoop_home = conf.get("hadoophome");
       String collection = conf.get("collection");
       String task_id = conf.get("mapred.task.id");
       task_id = task_id.substring(8); // remove "attempt_" prefix
+
       // Programatically rewrite the protocol as appropriate for the given
-      // archives directory
-      file_path = file_path.replace("hdfs://", hdfs_prefix);
+      // archives directory (not necessary if path is local or NFS)
+      if (hdfs_prefix.equals("/hdfs"))
+      {
+        file_path = file_path.replaceFirst("hdfs://[^/]*", hdfs_prefix);
+      }
+      else
+      {
+        file_path = file_path.replace("hdfs://", hdfs_prefix);
+      }
+
       // - create a temporary directory
       File greenstone_tmp_dir = new File("/tmp/greenstone");
@@ -178 +187 @@
         greenstone_tmp_dir.mkdir();
       }
+
       // - open a unique log file
       File import_process_log = new File("/tmp/greenstone/import-hadoop-" + task_id + ".log");
@@ -220 +230 @@
 
       // - call Greenstone passing in the path to the manifest
-      ProcessBuilder import_process_builder
-        = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-verbosity", "42", "-archivedir", hdfs_prefix + "/user/jmt12/gsdl/collect/" + collection + "/archives", collection);
+      ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
       fw1.write("[Command:" + import_process_builder.command() + "]\n");
       // - alter environment
@@ -229 +238 @@
       path = gsdlhome + "/ext/parallel-building/bin/script:" + path;
       path = gsdlhome + "/ext/parallel-building/linux/bin:" + path;
-      path = hadoop_prefix + "/bin:" + path;
+      path = hadoop_home + "/bin:" + path;
       path = gsdlhome + "/ext/tdb-edit/linux/bin:" + path;
       path = gsdlhome + "/ext/tdb-edit/bin/script:" + path;
@@ -251 +260 @@
       import_process_env.put("GEXTVIDEO_INSTALLED", gsdlhome + "/ext/video-and-audio/linux");
       // - Hadoop specific
-      import_process_env.put("HADOOP_PREFIX", hadoop_prefix);
-      fw1.write("[HADOOP_PREFIX: " + hadoop_prefix + "]\n");
+      import_process_env.put("HADOOP_PREFIX", hadoop_home);
+      fw1.write("[HADOOP_PREFIX: " + hadoop_home + "]\n");
 
       // - change working directory
@@ -310 +319 @@
     if (args.length < 6)
     {
-      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <hdfsprefix> <hadoop prefix> <collection> <hdfsin> <hdfsout>\n");
+      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <hadoop home> <collection> <archivesdir> <hdfsprefix> <hdfsin> <hdfsout>\n");
       System.exit(0);
     }
 
     Configuration conf = new Configuration();
-    conf.set("gsdlhome", args[0]);
-    conf.set("hdfsprefix", args[1]); // HDThriftFS or HDFSShell
-    conf.set("hadoopprefix", args[2]);
-    conf.set("collection", args[3]);
+    conf.set("gsdlhome",    args[0]);
+    conf.set("hadoophome",  args[1]);
+    conf.set("collection",  args[2]);
+    conf.set("archivesdir", args[3]);
+    conf.set("hdfsprefix",  args[4]); // "HDThriftFS", "HDFSShell", or ""
+    conf.set("hdfsin",      args[5]);
+    conf.set("hdfsout",     args[6]);
+
     // Set the number of retries to 1 - hopefully one of the following will work
     conf.setInt("mapred.map.max.attempts", 1); // Old Hadoop
@@ -348 +361 @@
     // - this input path should be to a file (in HDFS) that lists the paths to
     //   the manifest files
-    FileInputFormat.setInputPaths(job, new Path(args[4]));
+    FileInputFormat.setInputPaths(job, new Path(conf.get("hdfsin")));
     // - for now the output isn't that important, but in the future I may use
     //   this mechanism to produce a time based log.
-    FileOutputFormat.setOutputPath(job, new Path(args[5]));
+    FileOutputFormat.setOutputPath(job, new Path(conf.get("hdfsout")));
 
     // Recommended notation despite my hatiness of ?: syntax