Changeset 26980 for gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl

Timestamp:

2013-03-05T10:09:55+13:00 (11 years ago)

Author:

jmt12

Message:

setting svnignore

Location:

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl

Files:

• . (modified) (1 prop)
• HadoopGreenstoneIngest.java (modified) (8 diffs)

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java

@@ -161 +161 @@
       Configuration conf = context.getConfiguration();
       String gsdlhome = conf.get("gsdlhome");
+      String hdfs_host = conf.get("hdfshost");
+      String hdfs_port = conf.get("hdfsport");
+      String hadoop_prefix = conf.get("hadoopprefix");
       String collection = conf.get("collection");
       String task_id = conf.get("mapred.task.id");
@@ -193 +196 @@
       // - call Greenstone passing in the path to the manifest
       ProcessBuilder import_process_builder
-        = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-verbosity", "42", "-archivedir", "hdfs://localhost:54310/user/jmt12/gsdl/collect/" + collection + "/archives", collection);
+        = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-verbosity", "42", "-archivedir", "hdfs://" + hdfs_host + ":" + hdfs_port + "/user/jmt12/gsdl/collect/" + collection + "/archives", collection);
       fw1.write("[Command:" + import_process_builder.command() + "]\n");
       // - alter environment
@@ -199 +202 @@
       //   - path
       String path = import_process_env.get("PATH");
-      path = gsdlhome + "/ext/hadoop/bin/script:" + path;
-      path = gsdlhome + "/ext/hadoop/linux/bin:" + path;
-      path = gsdlhome + "/ext/hadoop/packages/hadoop/bin:" + path;
+      path = gsdlhome + "/ext/parallel-building/bin/script:" + path;
+      path = gsdlhome + "/ext/parallel-building/linux/bin:" + path;
+      path = hadoop_prefix + "/bin:" + path;
       path = gsdlhome + "/ext/tdb-edit/linux/bin:" + path;
       path = gsdlhome + "/ext/tdb-edit/bin/script:" + path;
@@ -209 +212 @@
       import_process_env.put("PATH", path);
       //   - ld_library_path
-      import_process_env.put("LD_LIBRARY_PATH", gsdlhome + "/ext/hadoop/linux/lib:" + gsdlhome + "/ext/video-and-audio/linux/lib:" + gsdlhome + "/ext/tdb-edit/linux/lib:" + gsdlhome + "/ext/parallel-building/linux/lib:" + gsdlhome + "/lib/linux");
+      import_process_env.put("LD_LIBRARY_PATH", gsdlhome + "/ext/parallel-building/linux/lib:" + gsdlhome + "/ext/hadoop/linux/lib:" + gsdlhome + "/ext/video-and-audio/linux/lib:" + gsdlhome + "/ext/tdb-edit/linux/lib");
       //   - dyld_library_path
       import_process_env.put("DYLD_LIBRARY_PATH", gsdlhome + "/ext/video-and-audio/linux/lib");
@@ -215 +218 @@
       import_process_env.put("GSDLHOME", gsdlhome);
       import_process_env.put("GSDLOS", "linux");
-      //import_process_env.put("GSDLEXTS", "hadoop:parallel-building:tdb-edit:video-and-audio");
-      import_process_env.put("GSDLEXTS", "video-and-audio:tdb-edit:hadoop");
+      import_process_env.put("GSDLEXTS", "parallel-building:tdb-edit:video-and-audio");
       //   - installed extension paths
-      import_process_env.put("GEXTHADOOP_INSTALLED", gsdlhome + "/ext/hadoop/linux");
+      import_process_env.put("GEXTPARALLELBUILDING_INSTALLED", gsdlhome + "/ext/parallel-building/linux");
       import_process_env.put("GEXTVIDEO_INSTALLED", gsdlhome + "/ext/video-and-audio/linux");
       import_process_env.put("GEXTTDBEDIT_INSTALLED", gsdlhome + "/ext/tdb-edit/linux");
       // - Hadoop specific
-      import_process_env.put("HADOOP_PREFIX", gsdlhome + "/ext/hadoop/packages/hadoop");
+      import_process_env.put("HADOOP_PREFIX", hadoop_prefix);
       // - change working directory
       import_process_builder.directory(new File(gsdlhome));
@@ -268 +270 @@
     throws Exception
   {
-    if (args.length < 4)
-    {
-      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <collection> <hdfsin> <hdfsout>\n");
+    if (args.length < 6)
+    {
+      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <hdfs host> <hdfs port> <hadoop prefix> <collection> <hdfsin> <hdfsout>\n");
       System.exit(0);
     }
@@ -276 +278 @@
     Configuration conf = new Configuration();
     conf.set("gsdlhome", args[0]);
-    conf.set("collection", args[1]);
+    conf.set("hdfshost", args[1]);
+    conf.set("hdfsport", args[2]);
+    conf.set("hadoopprefix", args[3]);
+    conf.set("collection", args[4]);
     // prevent timeouts
     long milli_seconds = 60*60*1000; // 1 hour
@@ -303 +308 @@
     // - this input path should be to a file (in HDFS) that lists the paths to
     //   the manifest files
-    FileInputFormat.setInputPaths(job, new Path(args[2]));
+    FileInputFormat.setInputPaths(job, new Path(args[5]));
     // - for now the output isn't that important, but in the future I may use
     //   this mechanism to produce a time based log.
-    FileOutputFormat.setOutputPath(job, new Path(args[3]));
+    FileOutputFormat.setOutputPath(job, new Path(args[6]));
 
     // Recommended notation despite my hatiness of ?: syntax