Changeset 30354 for gs2-extensions/parallel-building/trunk/src/src

Timestamp:

2015-12-16T16:15:39+13:00 (8 years ago)

Author:

jmt12

Message:

Extending manifest v2 support to allow for directories to be listed in manifest. Matched with changes in Directory plugin to allow paths into systems like HDFS to be listed in manifest.cd

Location:

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl

Files:

• HadoopGreenstoneIngest.java (modified) (2 diffs)
• HadoopGreenstoneIngest2.java (modified) (3 diffs)

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java

@@ -262 +262 @@
 
       // - call Greenstone passing in the path to the manifest
-      ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
+      //ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
+      String environment_script_filename = "setup.bash";
+      StringBuffer cmd_buffer = new StringBuffer();
+      cmd_buffer.append("source ./");
+      cmd_buffer.append(environment_script_filename);
+      cmd_buffer.append(" && time -p import.pl -keepold -manifest \"");
+      cmd_buffer.append(manifest_path.toString());
+      cmd_buffer.append("\" -archivedir \"");
+      cmd_buffer.append(conf.get("archivesdir"));
+      cmd_buffer.append("\" ");
+      cmd_buffer.append(collection);
+      ProcessBuilder import_process_builder = new ProcessBuilder("bash", "-c", cmd_buffer.toString());
       fw1.write("[Command:" + import_process_builder.command() + "]\n");
+      /*
       // - alter environment
       Map<String, String> import_process_env = import_process_builder.environment();
-      //   - path
+      // - build up the path
       String path = import_process_env.get("PATH");
       path = gsdlhome + "/ext/parallel-building/bin/script:" + path;
@@ -294 +306 @@
       import_process_env.put("HADOOP_PREFIX", hadoop_home);
       fw1.write("[HADOOP_PREFIX: " + hadoop_home + "]\n");
+      */
 
       // - change working directory

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest2.java

@@ -324 +324 @@
       manifest_writer.close();
 
+      /* Original process calling - sets up environment in Java
       // - call Greenstone passing in the path to the manifest
       ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
@@ -357 +358 @@
       import_process_env.put("HADOOP_PREFIX", hadoop_home);
       fw1.write("[HADOOP_PREFIX: " + hadoop_home + "]\n");
+      */
+
+      /* New process call - adds call to setup.bash first to prepare
+       * environment... hopefully */
+      // - call Greenstone passing in the path to the manifest
+      String environment_script_filename = "setup.bash";
+      StringBuffer cmd_buffer = new StringBuffer();
+      cmd_buffer.append("source ./");
+      cmd_buffer.append(environment_script_filename);
+      cmd_buffer.append(" && time -p import.pl -keepold -manifest \"");
+      cmd_buffer.append(manifest_path.toString());
+      cmd_buffer.append("\" -archivedir \"");
+      cmd_buffer.append(conf.get("archivesdir"));
+      cmd_buffer.append("\" ");
+      cmd_buffer.append(collection);
+      ProcessBuilder import_process_builder = new ProcessBuilder("bash", "-c", cmd_buffer.toString());
+      fw1.write("[Command:" + import_process_builder.command() + "]\n");
 
       // - change working directory
@@ -738 +756 @@
     job.setReducerClass(GSReducer.class);
 
-    // Sets the input and output handlers - may need to adjust input to provide me
-    // a series of filenames (TextInputFormat will instead read in a text file and
-    // return each line...)
+    // Sets the input and output handlers - may need to adjust input to provide
+    // a series of filenames (TextInputFormat will instead read in a text file
+    // and return each line...)
     job.setInputFormatClass(GSFileInputFormat.class);
     //job.setOutputFormatClass(NullOutputFormat.class);