Index: /gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java	(revision 26954)
+++ /gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java	(revision 26954)
@@ -0,0 +1,314 @@
+package org.nzdl.gsdl;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.ProcessBuilder;
+import java.lang.ProcessBuilder.*;
+import java.net.InetAddress;
+import java.util.Map;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.*;
+
+/** @class WordCount
+ */
+public class HadoopGreenstoneIngest
+{
+
+  /** @class GSFileRecordReader
+   */
+  public static class GSFileRecordReader
+    extends RecordReader<Text, IntWritable>
+  {
+    /** Uncompressed file name */
+    private Text current_key;
+
+    private IntWritable current_value = new IntWritable(1);
+
+    /** Used to indicate progress */
+    private boolean is_finished = false;
+
+    /**
+     */
+    @Override
+    public void initialize( InputSplit inputSplit, TaskAttemptContext taskAttemptContext )
+      throws IOException, InterruptedException
+    {
+      FileSplit split = (FileSplit) inputSplit;
+      current_key = new Text(split.getPath().toString());
+    }
+    /** initialize() **/
+
+    /**
+     *  We only ever have a single 
+     */
+    @Override
+    public boolean nextKeyValue()
+      throws IOException, InterruptedException
+    {
+      if (!is_finished)
+      {
+        is_finished = true;
+        return true;
+      }
+      return false;
+    }
+    /** nextKeyValue() **/
+
+    /** @function getProgress
+     *  Rather than calculating progress, we just keep it simple
+     */
+    @Override
+    public float getProgress()
+      throws IOException, InterruptedException
+    {
+      return is_finished ? 1 : 0;
+    }
+    /** getProgress() **/
+
+    /**
+     *  Returns the current key (name of the zipped file)
+     */
+    @Override
+    public Text getCurrentKey()
+      throws IOException, InterruptedException
+    {
+        return current_key;
+    }
+    /** getCurrentKey() **/
+
+    /**
+     * Returns the current value (contents of the zipped file)
+     */
+    @Override
+    public IntWritable getCurrentValue()
+      throws IOException, InterruptedException
+    {
+      return current_value;
+    }
+    /** getCurrentValue() **/
+
+    /**
+     * Close quietly, ignoring any exceptions
+     */
+    @Override
+    public void close()
+      throws IOException
+    {
+      // nothing to do
+    }
+    /** close() **/
+
+  }
+  /** GSFileRecordReader **/
+
+  /** @class GSFileInputFormat
+   */
+  public static class GSFileInputFormat
+    extends FileInputFormat<Text, IntWritable>
+  {
+    /**
+     *  Don't split the files
+     */
+    @Override
+    protected boolean isSplitable(JobContext context, Path filename)
+    {
+      return false;
+    }
+    /** isSplitable() **/
+
+    /**
+     */
+    @Override
+    public RecordReader<Text, IntWritable> createRecordReader(InputSplit split, TaskAttemptContext content)
+      throws IOException, InterruptedException
+    {
+      return new GSFileRecordReader();
+    }
+    /** createRecordReader() **/
+
+  }
+  /** class GSFileInputFormat **/
+
+ /** @class GSMap
+   */
+  public static class GSMap
+    extends Mapper<Text, IntWritable, Text, IntWritable>
+  {
+    /** @function map
+     *  The key is the full path (HDFS) of the file to be processed.
+     */
+    public void map(Text key, IntWritable value, Context context)
+      throws IOException, InterruptedException
+    {
+      String file_path = key.toString();
+      // - configuration for the task
+      Configuration conf = context.getConfiguration();
+      String gsdlhome = conf.get("gsdlhome");
+      String collection = conf.get("collection");
+      String task_id = conf.get("mapred.task.id");
+      task_id = task_id.substring(8); // remove "attempt_" prefix
+      // - create a temporary directory
+      File greenstone_tmp_dir = new File("/tmp/greenstone");
+      if (!greenstone_tmp_dir.isDirectory())
+      {
+        greenstone_tmp_dir.mkdir();
+      }
+      // - open a unique log file
+      File import_process_log = new File("/tmp/greenstone/import-hadoop-" + task_id + ".log");
+      // - start the log by writing the time and the manifest line
+      FileWriter fw1 = new FileWriter(import_process_log, true);
+      long start_time = System.currentTimeMillis()/1000;
+      fw1.write("[Started:" + start_time + "]\n");
+      fw1.write("[Host:" + InetAddress.getLocalHost().getHostName() + "]\n");
+      fw1.write("[Task:" + task_id + "]\n");
+      fw1.write("[Map:" + file_path + " => " + value + "]\n");
+
+      // - create a temporary manifest file to process this file. Overwrite any
+      //   existing file
+      File manifest_path = new File("/tmp/greenstone/manifest" + task_id + ".xml");
+      FileWriter manifest_writer = new FileWriter(manifest_path);
+      manifest_writer.write("<Manifest version=\"1.0\">\n");
+      manifest_writer.write("\t<Index>\n");
+      manifest_writer.write("\t\t<Filename>" + file_path + "</Filename>\n");
+      manifest_writer.write("\t</Index>\n");
+      manifest_writer.write("</Manifest>\n");
+      manifest_writer.close();
+
+      // - call Greenstone passing in the path to the manifest
+      ProcessBuilder import_process_builder
+        = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-verbosity", "42", "-archivedir", "hdfs://localhost:54310/user/jmt12/gsdl/collect/" + collection + "/archives", collection);
+      fw1.write("[Command:" + import_process_builder.command() + "]\n");
+      // - alter environment
+      Map<String, String> import_process_env = import_process_builder.environment();
+      //   - path
+      String path = import_process_env.get("PATH");
+      path = gsdlhome + "/ext/hadoop/bin/script:" + path;
+      path = gsdlhome + "/ext/hadoop/linux/bin:" + path;
+      path = gsdlhome + "/ext/hadoop/packages/hadoop/bin:" + path;
+      path = gsdlhome + "/ext/tdb-edit/linux/bin:" + path;
+      path = gsdlhome + "/ext/tdb-edit/bin/script:" + path;
+      path = gsdlhome + "/ext/video-and-audio/linux/bin:" + path;
+      path = gsdlhome + "/bin/script:" + path;
+      path = gsdlhome + "/bin/linux:" + path;
+      import_process_env.put("PATH", path);
+      //   - ld_library_path
+      import_process_env.put("LD_LIBRARY_PATH", gsdlhome + "/ext/hadoop/linux/lib:" + gsdlhome + "/ext/video-and-audio/linux/lib:" + gsdlhome + "/ext/tdb-edit/linux/lib:" + gsdlhome + "/ext/parallel-building/linux/lib:" + gsdlhome + "/lib/linux");
+      //   - dyld_library_path
+      import_process_env.put("DYLD_LIBRARY_PATH", gsdlhome + "/ext/video-and-audio/linux/lib");
+      //   - misc
+      import_process_env.put("GSDLHOME", gsdlhome);
+      import_process_env.put("GSDLOS", "linux");
+      //import_process_env.put("GSDLEXTS", "hadoop:parallel-building:tdb-edit:video-and-audio");
+      import_process_env.put("GSDLEXTS", "video-and-audio:tdb-edit:hadoop");
+      //   - installed extension paths
+      import_process_env.put("GEXTHADOOP_INSTALLED", gsdlhome + "/ext/hadoop/linux");
+      import_process_env.put("GEXTVIDEO_INSTALLED", gsdlhome + "/ext/video-and-audio/linux");
+      import_process_env.put("GEXTTDBEDIT_INSTALLED", gsdlhome + "/ext/tdb-edit/linux");
+      // - Hadoop specific
+      import_process_env.put("HADOOP_PREFIX", gsdlhome + "/ext/hadoop/packages/hadoop");
+      // - change working directory
+      import_process_builder.directory(new File(gsdlhome));
+      // - close our output to the log before opening in the process
+      fw1.close();
+
+      // - write output to log
+      import_process_builder.redirectErrorStream(true);
+      import_process_builder.redirectOutput(Redirect.appendTo(import_process_log));
+      // - run process
+      Process import_process = import_process_builder.start();
+      try
+      {
+        int import_status = import_process.waitFor();
+        if (import_status != 0)
+        {
+          throw new Exception("exit status: " + import_status);
+        }
+      }
+      catch (Exception e)
+      {
+        System.err.println("Error! Import command failed (" + e.toString() + ")");
+        System.exit(0);
+      }
+
+      // - write end time to log
+      FileWriter fw2 = new FileWriter(import_process_log, true);
+      long end_time = System.currentTimeMillis()/1000;
+      fw2.write("[Completed:" + end_time + "]\n");
+      fw2.close();
+
+      // - for now return a dummy output. In the future I may want to parse the
+      //   output from Greenstone as output and allow reducing to make me a
+      //   pretty timebased log
+      context.write(key, value);
+    }
+    /** map(LongWritable,Text,Context) **/
+
+  }
+  /** class GSMap **/
+
+  /** @function main
+   */
+  public static void main(String[] args)
+    throws Exception
+  {
+    if (args.length < 4)
+    {
+      System.out.println("Usage: bin/hadoop jar hadoop-greenstone.jar org.nzdl.gsdl.HadoopGreenstoneIngest <gsdlhome> <collection> <hdfsin> <hdfsout>\n");
+      System.exit(0);
+    }
+
+    Configuration conf = new Configuration();
+    conf.set("gsdlhome", args[0]);
+    conf.set("collection", args[1]);
+    // prevent timeouts
+    long milli_seconds = 60*60*1000; // 1 hour
+    conf.setLong("mapred.task.timeout", milli_seconds);
+
+    Job job = new Job(conf, "hadoopgreenstoneingest");
+    job.setJarByClass(HadoopGreenstoneIngest.class);
+
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(IntWritable.class);
+
+    // Register the map, combiner, and reducer classes
+    job.setMapperClass(GSMap.class);
+     // - in theory, uses the IdentityReducer by default, which simply returns
+    //   the input as the output (so no processing)
+    job.setNumReduceTasks(0);
+
+    // Sets the input and output handlers - may need to adjust input to provide me
+    // a series of filenames (TextInputFormat will instead read in a text file and
+    // return each line...)
+    job.setInputFormatClass(GSFileInputFormat.class);
+    job.setOutputFormatClass(NullOutputFormat.class);
+    //job.setOutputFormatClass(TextOutputFormat.class);
+
+    // Register the input and output paths
+    // - this input path should be to a file (in HDFS) that lists the paths to
+    //   the manifest files
+    FileInputFormat.setInputPaths(job, new Path(args[2]));
+    // - for now the output isn't that important, but in the future I may use
+    //   this mechanism to produce a time based log.
+    FileOutputFormat.setOutputPath(job, new Path(args[3]));
+
+    // Recommended notation despite my hatiness of ?: syntax
+    System.exit(job.waitForCompletion(true)?0:1);
+  }
+  /** main(String[]) **/
+}
Index: /gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/compile.sh
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/compile.sh	(revision 26954)
+++ /gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/compile.sh	(revision 26954)
@@ -0,0 +1,6 @@
+#!/bin/sh
+rm $GEXTHADOOP_INSTALLED/lib/hadoop-greenstone.jar
+rm -rf hgi_classes
+mkdir hgi_classes
+javac -cp $HADOOP_PREFIX/hadoop-core-$HADOOP_VERSION.jar -d hgi_classes HadoopGreenstoneIngest.java
+jar -cvf $GEXTHADOOP_INSTALLED/lib/hadoop-greenstone.jar -C hgi_classes/ .
