Changeset 27413 for gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java

Timestamp:

2013-05-24T09:23:29+12:00 (11 years ago)

Author:

jmt12

Message:

Making the Hadoop processing aware that there are two different possible prefixes for HDFS based files (each using a different 'driver' under the new FileUtils system)

File:

• gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java (modified) (5 diffs)

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java

@@ -164 +164 @@
       Configuration conf = context.getConfiguration();
       String gsdlhome = conf.get("gsdlhome");
-      String hdfs_host = conf.get("hdfshost");
-      String hdfs_port = conf.get("hdfsport");
+      String hdfs_prefix = conf.get("hdfsprefix");
       String hadoop_prefix = conf.get("hadoopprefix");
       String collection = conf.get("collection");
@@ -199 +198 @@
       // - call Greenstone passing in the path to the manifest
       ProcessBuilder import_process_builder
-        = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-verbosity", "42", "-archivedir", "hdfs://" + hdfs_host + ":" + hdfs_port + "/user/jmt12/gsdl/collect/" + collection + "/archives", collection);
+        = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-verbosity", "42", "-archivedir", hdfs_prefix + "/user/jmt12/gsdl/collect/" + collection + "/archives", collection);
       fw1.write("[Command:" + import_process_builder.command() + "]\n");
       // - alter environment
@@ -284 +283 @@
     if (args.length < 6)
     {
-      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <hdfs host> <hdfs port> <hadoop prefix> <collection> <hdfsin> <hdfsout>\n");
+      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <hdfsprefix> <hadoop prefix> <collection> <hdfsin> <hdfsout>\n");
       System.exit(0);
     }
@@ -290 +289 @@
     Configuration conf = new Configuration();
     conf.set("gsdlhome", args[0]);
-    conf.set("hdfshost", args[1]);
-    conf.set("hdfsport", args[2]);
-    conf.set("hadoopprefix", args[3]);
-    conf.set("collection", args[4]);
+    conf.set("hdfsprefix", args[1]); // HDThriftFS or HDFSShell
+    conf.set("hadoopprefix", args[2]);
+    conf.set("collection", args[3]);
     // Set the number of retries to 1 - hopefully one of the following will work
     conf.setInt("mapred.map.max.attempts", 1); // Old Hadoop
@@ -323 +321 @@
     // - this input path should be to a file (in HDFS) that lists the paths to
     //   the manifest files
-    FileInputFormat.setInputPaths(job, new Path(args[5]));
+    FileInputFormat.setInputPaths(job, new Path(args[4]));
     // - for now the output isn't that important, but in the future I may use
     //   this mechanism to produce a time based log.
-    FileOutputFormat.setOutputPath(job, new Path(args[6]));
+    FileOutputFormat.setOutputPath(job, new Path(args[5]));
 
     // Recommended notation despite my hatiness of ?: syntax