Index: gs2-extensions/video-and-audio/trunk/src/opt/DSpace/ItemUpdate.java
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/DSpace/ItemUpdate.java	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/DSpace/ItemUpdate.java	(revision 26186)
@@ -0,0 +1,624 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.itemupdate;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.dspace.content.Item;
+import org.dspace.core.ConfigurationManager;
+import org.dspace.core.Context;
+import org.dspace.eperson.EPerson;
+
+/**
+ *   
+ *   Provides some batch editing capabilities for items in DSpace:
+ *   	Metadata fields - Add, Delete
+ *      Bitstreams - Add, Delete
+ * 
+ *   The design has been for compatibility with ItemImporter
+ *      in the use of the DSpace archive format which is used to
+ *      specify changes on a per item basis.  The directory names
+ *      to correspond to each item are arbitrary and will only be
+ *      used for logging purposes.  The reference to the item is
+ *      from a required dc.identifier with the item handle to be 
+ *      included in the dublin_core.xml (or similar metadata) file.
+ *      
+ *   Any combination of these actions is permitted in a single run of this class 
+ *      The order of actions is important when used in combination.  
+ *      It is the responsibility of the calling class (here, ItemUpdate)
+ *      to register UpdateAction classes in the order to which they are
+ *      to be performed.
+ *      
+ * 
+ *   It is unfortunate that so much code needs to be borrowed 
+ *      from ItemImport as it is not reusable in private methods, etc.
+ *      Some of this has been placed into the MetadataUtilities class
+ *      for possible reuse elsewhere. 
+ *   
+ * 
+ * @author W. Hays based on a conceptual design by R. Rodgers
+ *
+ */
+public class ItemUpdate {
+	
+	public static final String SUPPRESS_UNDO_FILENAME = "suppress_undo";
+
+	public static final String CONTENTS_FILE = "contents";
+	public static final String DELETE_CONTENTS_FILE = "delete_contents";
+
+	public static String HANDLE_PREFIX = null;
+	public static final Map<String, String> filterAliases = new HashMap<String, String>();
+	
+	public static boolean verbose = false;
+	
+	static
+	{
+		filterAliases.put("ORIGINAL", "org.dspace.app.itemupdate.OriginalBitstreamFilter");
+		filterAliases.put("ORIGINAL_AND_DERIVATIVES", "org.dspace.app.itemupdate.OriginalWithDerivativesBitstreamFilter");
+		filterAliases.put("TEXT", "org.dspace.app.itemupdate.DerivativeTextBitstreamFilter");		
+		filterAliases.put("THUMBNAIL", "org.dspace.app.itemupdate.ThumbnailBitstreamFilter");		
+	}
+	          	                
+    // File listing filter to check for folders
+    static FilenameFilter directoryFilter = new FilenameFilter()
+    {
+        public boolean accept(File dir, String n)
+        {
+            File f = new File(dir.getAbsolutePath() + File.separatorChar + n);
+            return f.isDirectory();
+        }
+    };
+ 
+    // File listing filter to check for files (not directories)
+    static FilenameFilter fileFilter = new FilenameFilter()
+    {
+        public boolean accept(File dir, String n)
+        {
+            File f = new File(dir.getAbsolutePath() + File.separatorChar + n);
+            return (f.isFile());
+        }
+    };
+
+    // instance variables
+    private ActionManager actionMgr = new ActionManager(); 
+    private List<String> undoActionList = new ArrayList<String>();  
+    private String eperson;
+   
+    /**
+     * 
+     * @param argv
+     */
+    public static void main(String[] argv)
+    {
+        // create an options object and populate it
+        CommandLineParser parser = new PosixParser();
+
+        Options options = new Options();
+
+     //processing basis for determining items    
+        //item-specific changes with metadata in source directory with dublin_core.xml files  
+        options.addOption("s", "source", true, "root directory of source dspace archive ");
+      
+    //actions  on items     
+        options.addOption("a", "addmetadata", true, "add metadata specified for each item; multiples separated by semicolon ';'");
+        options.addOption("d", "deletemetadata", true, "delete metadata specified for each item");
+        
+        options.addOption("A", "addbitstreams", false, "add bitstreams as specified for each item");
+        
+        // extra work to get optional argument
+        Option delBitstreamOption = new Option("D", "deletebitstreams", true, "delete bitstreams as specified for each item");
+        delBitstreamOption.setOptionalArg(true);
+        delBitstreamOption.setArgName("BitstreamFilter");
+        options.addOption(delBitstreamOption);
+                               
+//other params        
+        options.addOption("e", "eperson", true, "email of eperson doing the update");
+        options.addOption("i", "itemfield", true, "optional metadata field that containing item identifier; default is dc.identifier.uri");
+        options.addOption("F", "filter-properties", true, "filter class name; only for deleting bitstream");  
+        options.addOption("v", "verbose", false, "verbose logging");
+      
+//special run states        
+        options.addOption("t", "test", false, "test run - do not actually import items");
+        options.addOption("P", "provenance", false, "suppress altering provenance field for bitstream changes");
+        options.addOption("h", "help", false, "help");
+        
+        // New option to supress undo functionality [jmt12]
+        options.addOption("n", "noundo", false, "do not produce undo script");
+
+        int status = 0;
+        boolean isTest = false;
+        boolean alterProvenance = true;
+        String itemField = null;
+        String metadataIndexName = null;
+        boolean noUndo = false; // [jmt12]
+        
+        Context context = null;
+        ItemUpdate iu = new ItemUpdate();	
+
+        try
+        {       
+	        CommandLine line = parser.parse(options, argv);
+	
+	        if (line.hasOption('h'))
+	        {
+	            HelpFormatter myhelp = new HelpFormatter();
+	            myhelp.printHelp("ItemUpdate", options);
+	            pr("");
+	            pr("Examples:");
+	            pr("  adding metadata:     ItemUpdate -e jsmith@mit.edu -s sourcedir -a dc.contributor -a dc.subject ");
+	            pr("  deleting metadata:   ItemUpdate -e jsmith@mit.edu -s sourcedir -d dc.description.other");
+	            pr("  adding bitstreams:   ItemUpdate -e jsmith@mit.edu -s sourcedir -A -i dc.identifier");
+	            pr("  deleting bitstreams: ItemUpdate -e jsmith@mit.edu -s sourcedir -D ORIGINAL ");
+	            pr("");
+	
+	            System.exit(0);
+	        }
+	        
+	        if (line.hasOption('v'))
+	        {
+	        	verbose = true;
+	        }
+
+ 
+	        if (line.hasOption('P'))
+	        {
+	        	alterProvenance = false;
+	        	pr("Suppressing changes to Provenance field option");
+	        }
+
+	        iu.eperson = line.getOptionValue('e'); // db ID or email
+	        
+	        if (!line.hasOption('s')) // item specific changes from archive dir
+	        {
+	        	pr("Missing source archive option");
+	        	System.exit(1);
+	        }
+	        String sourcedir = line.getOptionValue('s');
+	        
+        	if (line.hasOption('t'))  //test
+        	{
+        		isTest = true;
+	            pr("**Test Run** - not actually updating items.");
+       		       	
+	        }
+        	
+                // [jmt12]
+                if (line.hasOption('n'))
+                {
+                  noUndo = true;
+                  pr("Supressing undo functionality");
+                }
+
+        	if (line.hasOption('i'))
+        	{
+        		itemField = line.getOptionValue('i');      		
+        	}
+
+	        if (line.hasOption('d'))
+	        {
+	        	String[] targetFields = line.getOptionValues('d'); 
+	        	
+	        	DeleteMetadataAction delMetadataAction = (DeleteMetadataAction) iu.actionMgr.getUpdateAction(DeleteMetadataAction.class);
+	        	delMetadataAction.addTargetFields(targetFields);
+	        	
+	        	//undo is an add
+	        	for (String field : targetFields)
+	        	{
+	        		iu.undoActionList.add(" -a " + field + " ");
+	        	}
+	        	
+	        	pr("Delete metadata for fields: ");
+	        	for (String s : targetFields)
+	        	{
+	        		pr("    " + s);
+	        	}
+	        }
+	        
+    		if (line.hasOption('a'))
+    		{
+    			String[] targetFields = line.getOptionValues('a'); 
+        	
+    			AddMetadataAction addMetadataAction = (AddMetadataAction) iu.actionMgr.getUpdateAction(AddMetadataAction.class); 
+    			addMetadataAction.addTargetFields(targetFields);
+        		
+        		//undo is a delete followed by an add of a replace record for target fields
+	        	for (String field : targetFields)
+	        	{
+	        		iu.undoActionList.add(" -d " + field + " ");
+	        	}
+    			
+	        	for (String field : targetFields)
+	        	{
+	        		iu.undoActionList.add(" -a " + field + " ");
+	        	}
+    			
+	        	pr("Add metadata for fields: ");
+	        	for (String s : targetFields)
+	        	{
+	        		pr("    " + s);
+	        	}
+        	}
+    		
+	        if (line.hasOption('D'))  // undo not supported 
+	        {
+	        	pr("Delete bitstreams ");
+	        	
+	        	String[] filterNames = line.getOptionValues('D');
+	        	if ((filterNames != null) && (filterNames.length > 1))
+	        	{
+	        		pr("Error: Only one filter can be a used at a time.");
+	        		System.exit(1);
+	        	}
+	        	
+	        	String filterName = line.getOptionValue('D');
+	        	pr("Filter argument: " + filterName);
+	        	
+	        	if (filterName == null)  // indicates using delete_contents files
+	        	{
+	        		DeleteBitstreamsAction delAction = (DeleteBitstreamsAction) iu.actionMgr.getUpdateAction(DeleteBitstreamsAction.class);
+		        	delAction.setAlterProvenance(alterProvenance);
+	        	}
+	        	else
+	        	{
+	        	    // check if param is on ALIAS list
+	        		String filterClassname = filterAliases.get(filterName);
+	        		
+	        		if (filterClassname == null)
+	        		{
+	        			filterClassname = filterName;
+	        		}
+	        			        		
+	        	    BitstreamFilter filter = null; 
+
+	        	    try
+	        	    {
+		    			Class<?> cfilter = Class.forName(filterClassname);
+		    			pr("BitstreamFilter class to instantiate: " + cfilter.toString());
+		    			
+		    			filter =  (BitstreamFilter) cfilter.newInstance();  //unfortunate cast, an erasure consequence
+	        	    }
+	        	    catch(Exception e)
+	        	    {
+	        	    	pr("Error:  Failure instantiating bitstream filter class: " + filterClassname);
+	        	    	System.exit(1);
+	        	    }
+	        	    
+	        	    String filterPropertiesName = line.getOptionValue('F');
+	        	    if (filterPropertiesName != null)  //not always required
+	        	    {
+	        	    	try
+	        	    	{
+	        	    		// TODO try multiple relative locations, e.g. source dir
+	        	    		if (!filterPropertiesName.startsWith("/"))
+	        	    		{
+	        	    			filterPropertiesName = sourcedir + File.separator + filterPropertiesName;
+	        	    		}
+	        	    		
+	        	    		filter.initProperties(filterPropertiesName);
+	        	    	}
+	        	    	catch(Exception e)
+	        	    	{
+		        	    	pr("Error:  Failure finding properties file for bitstream filter class: " + filterPropertiesName);
+		        	    	System.exit(1);
+	        	    	}
+	        	    }
+	        	    
+	        		DeleteBitstreamsByFilterAction delAction = 
+	        			(DeleteBitstreamsByFilterAction) iu.actionMgr.getUpdateAction(DeleteBitstreamsByFilterAction.class);
+		        	delAction.setAlterProvenance(alterProvenance);
+		        	delAction.setBitstreamFilter(filter);
+	        	    //undo not supported
+	        	}        	
+	        }
+
+	        if (line.hasOption('A'))
+	        {
+	        	pr("Add bitstreams ");    	        	
+        		AddBitstreamsAction addAction = (AddBitstreamsAction) iu.actionMgr.getUpdateAction(AddBitstreamsAction.class);
+	        	addAction.setAlterProvenance(alterProvenance);
+	        	
+                        iu.undoActionList.add(" -D ");  // delete_contents file will be written, no arg required
+	        }    	        
+	        	        
+	        if (!iu.actionMgr.hasActions())
+	        {
+                pr("Error - an action must be specified");
+                System.exit(1);
+	        }
+	        else  
+	        {
+	        	pr("Actions to be performed: ");
+	        	
+	        	for (UpdateAction ua : iu.actionMgr)
+	        	{
+	        		pr("    " + ua.getClass().getName());
+	        	}
+	        }
+	        
+	        pr("ItemUpdate - initializing run on " + (new Date()).toString());
+	               	
+	        context = new Context();  
+	        iu.setEPerson(context, iu.eperson);	
+	        //context.setIgnoreAuthorization(true); // Deprecated
+                context.turnOffAuthorisationSystem();
+
+	    	HANDLE_PREFIX = ConfigurationManager.getProperty("handle.canonical.prefix");
+	    	if (HANDLE_PREFIX == null || HANDLE_PREFIX.length() == 0)
+	    	{
+	    		HANDLE_PREFIX = "http://hdl.handle.net/";
+	    	}
+	    		        
+        	iu.processArchive(context, sourcedir, itemField, metadataIndexName, alterProvenance, isTest, noUndo); // [jmt12]
+
+	        context.complete();  // complete all transactions
+	        //context.setIgnoreAuthorization(false); // Deprecated
+                context.restoreAuthSystemState();
+        }
+        catch (Exception e)
+        {
+            if (context != null && context.isValid())
+            {
+                context.abort();
+            	//context.setIgnoreAuthorization(false); // Deprecated
+                context.restoreAuthSystemState();
+            }
+            e.printStackTrace();
+            pr(e.toString());
+            status = 1;
+        }
+
+        if (isTest)
+        {
+            pr("***End of Test Run***");
+        }
+        else
+        {
+        	pr("End.");
+        	
+        }
+        System.exit(status);
+    }
+    
+  // noUndo added [jmt12]
+    private void processArchive(Context context, String sourceDirPath, String itemField,
+                                String metadataIndexName, boolean alterProvenance, boolean isTest, boolean noUndo)
+    throws Exception
+    {
+        // open and process the source directory
+        File sourceDir = new File(sourceDirPath);
+
+        if ((sourceDir == null) || !sourceDir.exists() || !sourceDir.isDirectory())
+        {
+            pr("Error, cannot open archive source directory " + sourceDirPath);
+            throw new Exception("error with archive source directory " + sourceDirPath);
+        }
+
+        String[] dircontents = sourceDir.list(directoryFilter);  //just the names, not the path
+        Arrays.sort(dircontents);
+       
+        //Undo is suppressed to prevent undo of undo
+        boolean suppressUndo = noUndo; // [jmt12]
+        File fSuppressUndo = new File(sourceDir, SUPPRESS_UNDO_FILENAME);
+        if (fSuppressUndo.exists())
+        {
+        	suppressUndo = true;
+        }
+
+        File undoDir = null;  //sibling directory of source archive
+        
+    	if (!suppressUndo && !isTest) 
+    	{
+    		undoDir = initUndoArchive(sourceDir);	
+    	}
+        
+    	int itemCount = 0;
+    	int successItemCount = 0;
+    	
+        for (String dirname : dircontents)
+        {
+        	itemCount++;
+    		pr("");
+    		pr("processing item " + dirname);
+    		
+    		try
+    		{
+	    		ItemArchive itarch = ItemArchive.create(context, new File(sourceDir, dirname), itemField);
+	    		
+	    		for (UpdateAction action : actionMgr)
+	    		{
+	    			pr("action: " + action.getClass().getName());
+	    			action.execute(context, itarch, isTest, suppressUndo);
+	    			if (!isTest && !suppressUndo)
+                    {
+                        itarch.writeUndo(undoDir);
+                    }
+	    		}
+	    		if (!isTest)
+	    		{
+    				Item item = itarch.getItem();
+    				item.update();  //need to update before commit
+    				context.commit();  
+	    			item.decache();
+	    		}
+    			ItemUpdate.pr("Item " + dirname + " completed");
+    			successItemCount++;
+    		}
+    		catch(Exception e)
+    		{
+    			pr("Exception processing item " + dirname + ": " + e.toString());
+    		}
+        }  
+        
+        if (!suppressUndo && !isTest)
+        {        	     	
+        	StringBuilder sb = new StringBuilder("dsrun org.dspace.app.itemupdate.ItemUpdate ");
+        	sb.append(" -e ").append(this.eperson);
+        	sb.append(" -s ").append(undoDir);
+        	
+        	if (itemField != null)
+        	{
+        		sb.append(" -i ").append(itemField);
+        	}
+        	
+         	if (!alterProvenance)
+        	{
+        		sb.append(" -P ");
+        	}
+        	if (isTest)
+        	{
+        		sb.append(" -t ");
+        	}
+        	
+        	for (String actionOption : undoActionList)
+        	{
+        		sb.append(actionOption);
+        	}   
+        	
+        	PrintWriter pw = null;
+        	try
+        	{
+               	File cmdFile = new File (undoDir.getParent(), undoDir.getName() + "_command.sh");
+	        	pw = new PrintWriter(new BufferedWriter(new FileWriter(cmdFile)));
+	        	pw.println(sb.toString());
+        	}
+        	finally
+        	{
+        		pw.close();
+        	}
+        }
+        
+        pr("");
+        pr("Done processing.  Successful items: " + successItemCount + " of " + itemCount + " items in source archive");
+        pr("");
+    }
+           
+    
+    /**
+     * 
+     * to avoid overwriting the undo source tree on repeated processing
+     * sequence numbers are added and checked 
+     * 
+     * @param sourceDir - the original source directory
+     * @return the directory of the undo archive
+     * @throws FileNotFoundException
+     * @throws IOException
+     */
+	private File initUndoArchive(File sourceDir)
+	throws FileNotFoundException, IOException
+	{				
+		File parentDir = sourceDir.getAbsoluteFile().getParentFile(); 
+		if (parentDir == null)
+		{
+			throw new FileNotFoundException("Parent directory of archive directory not found; unable to write UndoArchive; no processing performed");    			
+		}
+		
+		String sourceDirName = sourceDir.getName();
+		int seqNo = 1;
+		
+		File undoDir = new File(parentDir, "undo_" + sourceDirName + "_" + seqNo);
+		while (undoDir.exists())
+		{
+    		undoDir = new File(parentDir, "undo_" + sourceDirName+ "_" + ++seqNo); //increment
+		}
+		
+		// create root directory
+		if (!undoDir.mkdir())
+		{
+        	pr("ERROR creating  Undo Archive directory ");
+        	throw new IOException("ERROR creating  Undo Archive directory ");
+		}
+		
+        //Undo is suppressed to prevent undo of undo
+        File fSuppressUndo = new File(undoDir, ItemUpdate.SUPPRESS_UNDO_FILENAME);
+        try
+        {
+        	fSuppressUndo.createNewFile();
+        }
+        catch(IOException e)
+        {
+        	pr("ERROR creating Suppress Undo File " + e.toString());
+        	throw e;
+        }
+        return undoDir;		
+	}
+	
+	//private void write
+
+    private void setEPerson(Context context, String eperson)
+    throws Exception
+    {
+        if (eperson == null)
+        {
+            pr("Error - an eperson to do the importing must be specified");
+            pr(" (run with -h flag for details)");
+            throw new Exception("EPerson not specified.");        }
+
+        EPerson myEPerson = null;
+
+        if (eperson.indexOf('@') != -1)
+        {
+            // @ sign, must be an email
+            myEPerson = EPerson.findByEmail(context, eperson);
+        }
+        else
+        {
+            myEPerson = EPerson.find(context, Integer.parseInt(eperson));
+        }
+
+        if (myEPerson == null)
+        {
+            pr("Error, eperson cannot be found: " + eperson);
+            throw new Exception("Invalid EPerson");
+        }
+
+        context.setCurrentUser(myEPerson);
+    }
+    
+    /**
+     *   poor man's logging
+	 *   As with ItemImport, API logging goes through log4j to the DSpace.log files
+	 *   whereas the batch logging goes to the console to be captured there.
+     * @param s
+     */
+    static void pr(String s)
+    {
+    	System.out.println(s);
+    }
+    
+    /**
+     *  print if verbose flag is set
+     * @param s
+     */
+    static void prv(String s)
+    {
+    	if (verbose) 
+    	{
+        	System.out.println(s);
+    	}
+    }
+    
+} //end of class
+
Index: gs2-extensions/video-and-audio/trunk/src/opt/DSpace/README.txt
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/DSpace/README.txt	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/DSpace/README.txt	(revision 26186)
@@ -0,0 +1,49 @@
+===== Video Processing in DSpace =====
+
+==== Installation Instructions ====
+
+1. Copy the VideoFilter into place:
+
+cd <greenstone>/ext/video-and-audio/opt/DSpace
+cp VideoFilter.java <dspace>/dspace-source/dspace-api/src/main/java/org/dspace/app/mediafilter/VideoFilter.java
+
+2. Replace default ItemUpdate application with one that supports the 'noundo' flag (otherwise you're working directory will fill up with undo files for every video file processed):
+
+cd <greenstone>/ext/video-and-audio/opt/DSpace
+cp ItemUpdate.java <dspace>/dspace-source/dspace-api/src/main/java/org/dspace/app/itemupdate/ItemUpdate.java
+
+3. Configure the new plugin:
+
+cd <dspace>/dspace-source/dspace/config/
+emacs dspace.cfg
+...
+#Names of the enabled MediaFilter or FormatFilter plugins
+filter.plugins = PDF Text Extractor, HTML Text Extractor, \
+                 ...
+		 Video Converter
+#Assign 'human-understandable' names to each filter
+plugin.named.org.dspace.app.mediafilter.FormatFilter = \
+             ...
+             org.dspace.app.mediafilter.VideoFilter = Video Converter
+#Configure each filter's input format(s)
+...
+filter.org.dspace.app.mediafilter.VideoFilter.inputFormats = MPEG2 Transport Stream
+[ctrl][s] [ctrl][x]
+
+4. Recompile DSpace and update installation
+
+cd <dspace>/dspace-source
+mvn package
+cd <dspace>/dspace-source/target/dspace-1.8.2-build/
+ant update
+
+5. Open the administrator's wed interface and add a new file format "MPEG2 Transport Stream" with the file extension "*.ts"
+
+6. Still in the web interface, add your video files as raw TS (as exported from ReplayMe!) *or* use batch import to put in an entire directory tree as exported from Greenstone
+
+7. Run the media filter from the command line to ingest video files properly (the -f forces rebuild - useful as previous media filter runs will mark documents as filtered and otherwise won't process them again):
+
+cd <dspace>/bin/
+./dspace filter-media -f
+
+(Note, you can add a "-i" argument with an internal identifier to reprocess a single video file - that's basically how parallel processing works, by calling this command, once per document, on multiple machines).
Index: gs2-extensions/video-and-audio/trunk/src/opt/DSpace/VideoFilter.java
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/DSpace/VideoFilter.java	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/DSpace/VideoFilter.java	(revision 26186)
@@ -0,0 +1,497 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.mediafilter;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import java.lang.Thread;
+
+import java.nio.file.Files;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import java.util.Arrays;
+import java.util.HashMap;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.dspace.content.Bitstream;
+import org.dspace.content.DCValue;
+import org.dspace.content.Item;
+import org.dspace.core.ConfigurationManager;
+import org.dspace.core.Context;
+
+/** A media filter to provide basic video processing to DSpace. This class will
+ *  attempt to convert the input media into a web-optimized format (mp4) and at
+ *  the same time will generate a series of keyframe images and attach them to
+ *  the items bundle.
+ */
+public class VideoFilter
+extends MediaFilter
+{
+
+  // Path to dspace
+  String dspace_home;
+
+  // We need the handle uri when doing an itemupdate
+  String item_handle;
+  // We need the internal id when performing a metadata import
+  int item_id;
+  // We can extract an aired date from the dc.Title of some media files
+  String item_date;
+
+  Path temp_dir_path;
+
+  String streaming_hq_size;
+  String streaming_hq_video_bitrate;
+  String streaming_hq_audio_bitrate;
+  boolean video_deinterlacing_filter;
+
+  /** Constructor - read in configuration options
+   */
+  public VideoFilter()
+  {
+    // Look up configuration
+    dspace_home = ConfigurationManager.getProperty("dspace.dir");
+    //String streaming_format =  ConfigurationManager.getProperty(MediaFilterManager.FILTER_PREFIX + ".VideoFilter.outputFormat");
+    streaming_hq_size = "720";
+    streaming_hq_video_bitrate = "496";
+    streaming_hq_audio_bitrate = "80";
+    video_deinterlacing_filter = false;
+  }
+  /** VideoFilter() **/
+
+  /**
+   *  @return name of the bundle this filter will stick its generated
+   *          Bitstreams
+   */
+  public String getBundleName()
+  {
+    // For now we will just add the converted bitstream to the original
+    // bundle in order to make it easy to see. In theory it should probably
+    // have its own bundle and black magic associated ala THUMBNAIL...
+    // however, I note there is a lot of discussion on the DSpace forums of
+    // exactly this point - that it's black magic and not scalable/portable/
+    // modular/safe.
+    return "ORIGINAL";
+  }
+  /** getBundleName() **/
+
+  /**
+   *  @return string to describe the newly-generated Bitstream's - how it was
+   *          produced is a good idea
+   */
+  public String getDescription()
+  {
+    // @todo make this description dependent on the configuration options
+    return "Converted into streaming format using Handbrake and generated keyframes (if required) using Hive2";
+  }
+  /** getDescription() **/
+
+  /**
+   *  @param source
+   *            source input stream
+   *
+   *  @return InputStream the resulting input stream
+   */
+  public InputStream getDestinationStream(InputStream source)
+    throws Exception
+  {
+    System.out.println("VideoFilter::getDestinationStream()");
+    // 0. Create temporary directory and make a local file copy of the source
+    //    video as video conversion tools require a file for random access.
+    temp_dir_path = Files.createTempDirectory("dspace-video-filter");
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println("[temporary directory: " + temp_dir_path.toString() + "]");
+    }
+    Path item_path = temp_dir_path.resolve("item_000");
+    item_path.toFile().mkdirs(); // bet there is a nice nio way to do this
+    Path raw_video_path = temp_dir_path.resolve("gsv.ts");
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println(" * saving copy of media to temporary directory");
+    }
+    Files.copy(source, raw_video_path);
+
+    // 1. Use MediaFilter to extract metadata about this multimedia file
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println(" * extracting metadata using MediaInfo");
+    }
+    String metadata_command[] = {
+      "mediainfo",
+      "--Output=XML",
+      raw_video_path.toString()
+    };
+    Process metadata_process = Runtime.getRuntime().exec(metadata_command);
+    StreamGobbler metadata_process_error_gobbler = new StreamGobbler(metadata_process.getErrorStream());
+    metadata_process_error_gobbler.start();
+    BufferedReader metadata_br = new BufferedReader(new InputStreamReader(metadata_process.getInputStream()));
+    String line;
+    String type = "Unknown";
+    HashMap metadata = new HashMap();
+    Pattern type_pattern = Pattern.compile("<track type=\"([a-zA-Z]+)\">");
+    Pattern metadata_pattern = Pattern.compile("<([a-zA-Z_]+)>(.*)</\\1>");
+    while ((line = metadata_br.readLine()) != null)
+    {
+      Matcher type_matcher = type_pattern.matcher(line);
+      if (type_matcher.matches())
+      {
+        type = type_matcher.group(1);
+      }
+      else
+      {
+        Matcher metadata_matcher = metadata_pattern.matcher(line);
+        if (metadata_matcher.matches())
+        {
+          String field = metadata_matcher.group(1);
+          String value = metadata_matcher.group(2);
+          metadata.put(type + ":" + field, value);
+        }
+      }
+    }
+    int metadata_status = metadata_process.waitFor();
+    // * write a comma separated file containing the new metadata
+    Path metadata_path = temp_dir_path.resolve("metadata.csv");
+    BufferedWriter metadata_bw = new BufferedWriter(new FileWriter(metadata_path.toString()));
+    // - first row contains the (dublin core) headers
+    metadata_bw.write("id,dc.type,dc.date,dc.format,dc.format.extent");
+    metadata_bw.newLine();
+    // - second row contains multimedia-centric metadata ala
+    //   Greenstone:SimpleVideoPlugin
+    metadata_bw.write(item_id + ",");
+    metadata_bw.write("\"multimedia (" + metadata.get("General:Format") + ")\",");
+    metadata_bw.write("\"" + item_date + "\",");
+    // - dc.format
+    metadata_bw.write("\"" + metadata.get("General:Format") + "||video:" + metadata.get("Video:Format_Info") + " (" + metadata.get("Video:Format") + ")||audio:" + metadata.get("Audio:Format_Info") + " (" + metadata.get("Audio:Format") + ")\",");
+    // - dc.format.extent
+    metadata_bw.write("\"" + metadata.get("General:File_size") + "||" + metadata.get("General:Duration") + "||" + metadata.get("Video:Width") + " x " + metadata.get("Video:Height") + "\"");
+    metadata_bw.flush();
+    metadata_bw.close();
+    // * Call CLI script to attach this metadata
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println(" * adding multimedia metadata using DSpace's metadata-import");
+      ///ystem.out.println("[DEBUG] command: " + Arrays.toString(metadata_import_command)); 
+    }
+    String metadata_import_command[] = {
+      dspace_home + "/bin/dspace",
+      "metadata-import",
+      "-s",
+      "-f", metadata_path.toString()
+    };
+    Process metadata_import_process = Runtime.getRuntime().exec(metadata_import_command);
+    StreamGobbler metadata_import_process_err_gobbler = new StreamGobbler(metadata_import_process.getErrorStream());
+    metadata_import_process_err_gobbler.start();
+    StreamGobbler metadata_import_process_out_gobbler = new StreamGobbler(metadata_import_process.getInputStream());
+    metadata_import_process_out_gobbler.start();
+
+    int metadata_import_status = metadata_import_process.waitFor();
+    if (metadata_import_status != 0)
+    {
+      throw new Exception("Metadata import command failed (exit status: " + metadata_import_status + ")");
+    }
+
+    // 2. Call Handbrake to convert the video into streaming format
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println(" * performing conversion using HandBrake");
+    }
+    Path converted_video_path = temp_dir_path.resolve("gsv.mp4");
+    String convert_command[] = {
+      "HandBrakeCLI",
+      "-i", raw_video_path.toString(),
+      "-t", "1",
+      "-c", "1",
+      "-o", converted_video_path.toString(),
+      "-f", "mp4",
+      "-O",
+      "-w", streaming_hq_size,
+      "--loose-anamorphic",
+      "-e", "x264",
+      "-b", streaming_hq_video_bitrate,
+      "-a", "1",
+      "-E", "faac",
+      "-6", "dpl2",
+      "-R", "Auto",
+      "-B", streaming_hq_audio_bitrate,
+      "-D", "0.0",
+      "-x", "ref=2:bframes=2:subq=6:mixed-refs=0:weightb=0:8x8dct=0:trellis=0:threads=1"
+    };
+    // @todo determine the best way to account for configuration options
+    Process convert_process = Runtime.getRuntime().exec(convert_command);
+    StreamGobbler convert_process_error_gobbler = new StreamGobbler(convert_process.getErrorStream());
+    convert_process_error_gobbler.start();
+    StreamGobbler convert_process_out_gobbler = new StreamGobbler(convert_process.getInputStream());
+    convert_process_out_gobbler.start();
+    int convert_status = convert_process.waitFor();
+    if (convert_status != 0 || !converted_video_path.toFile().exists())
+    {
+      throw new Exception("Convert command failed (exit status: " + convert_status + ")");
+    }
+
+    // 3. Generate keyframes from converted video
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println(" * generating keyframes using Hive2");
+    }
+    Path shots_path = temp_dir_path.resolve("shots.xml");
+    String keyframe_command[] = {
+      "hive2_ffmpegsvn",
+      "-o", shots_path.toString(),
+      "-k", item_path.toString(),
+      "-m", "0.5",
+      "-l", "0.05",
+      converted_video_path.toString()
+    };
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println("[DEBUG " + Arrays.toString(keyframe_command) + "]");
+    }
+    Process keyframe_process = Runtime.getRuntime().exec(keyframe_command);
+    //Path keyframe_err_file = temp_dir_path.resolve("hive2-err.txt");
+    StreamGobbler keyframe_error_gobbler = new StreamGobbler(keyframe_process.getErrorStream());//, keyframe_err_file.toString());
+    keyframe_error_gobbler.start();
+    //Path keyframe_out_file = temp_dir_path.resolve("hive2-out.txt");
+    StreamGobbler keyframe_out_gobbler = new StreamGobbler(keyframe_process.getInputStream()); //, keyframe_out_file.toString());
+    keyframe_out_gobbler.start();
+    int keyframe_status = keyframe_process.waitFor();
+    if (keyframe_status != 0 || !shots_path.toFile().exists())
+    {
+      throw new Exception("Keyframe command failed (exit status: " + keyframe_status + ")");
+    }
+
+    // 4. Create the necessary files for the itemupdate script in the item
+    //    directory
+    // * a metadata file (dublin_core.xml) containing the handle (uri) for the
+    //   item to update
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println(" * adding keyframes to item using DSpace's itemupdate");
+    }
+    Path dcmeta_path = item_path.resolve("dublin_core.xml");
+    BufferedWriter dcmeta_bw = new BufferedWriter(new FileWriter(dcmeta_path.toString()));
+    dcmeta_bw.write("<dublin_core>");
+    dcmeta_bw.newLine();
+    dcmeta_bw.write("\t<dcvalue element=\"identifier\" qualifier=\"uri\">http://hdl.handle.net/" + item_handle + "</dcvalue>");
+    dcmeta_bw.newLine();
+    dcmeta_bw.write("</dublin_core>");
+    dcmeta_bw.newLine();
+    dcmeta_bw.flush();
+    dcmeta_bw.close();
+    // * a file (contents) referring to all of the keyframe images to add (and
+    //   the bundle to add them to)
+    File item_dir = item_path.toFile();
+    File files[] = item_dir.listFiles();
+    Arrays.sort(files);
+    Path contents_path = item_path.resolve("contents");
+    BufferedWriter contents_bw = new BufferedWriter(new FileWriter(contents_path.toString()));
+    Pattern image_filename_pattern = Pattern.compile(".*\\.jpg");
+    for (int i = 0; i < files.length; i++)
+    {
+      File image_file = files[i];
+      Path image_path = image_file.toPath();
+      Path image_filename_path = image_path.getFileName();
+      String image_filename = image_filename_path.toString();
+      System.out.println(" - considering file: " + image_filename);
+      Matcher image_filename_matcher = image_filename_pattern.matcher(image_filename);
+      if (image_filename_matcher.matches())
+      {
+        contents_bw.write(image_filename + "\tbundle:ORIGINAL\tdescription:Hive2 Keyframe");
+        contents_bw.newLine();
+      }
+    }
+    contents_bw.flush();
+    contents_bw.close();
+    // * Call itemupdate
+    String itemupdate_command[] = {
+      dspace_home + "/bin/dspace",
+      "itemupdate",
+      "--noundo", // Supress undo
+      "-A", // Add bitstreams
+      "-e", "videofilter@dspace", // Special eperson
+      "-s", temp_dir_path.toString() // Within which lies 'item_000'
+    };
+    Process itemupdate_process = Runtime.getRuntime().exec(itemupdate_command);
+    StreamGobbler itemupdate_process_error_gobbler = new StreamGobbler(itemupdate_process.getErrorStream());
+    itemupdate_process_error_gobbler.start();
+    StreamGobbler itemupdate_process_out_gobbler = new StreamGobbler(itemupdate_process.getInputStream());
+    itemupdate_process_out_gobbler.start();
+    int itemupdate_status = itemupdate_process.waitFor();
+    if (itemupdate_status != 0)
+    {
+      throw new Exception("ItemUpdate command failed (exit status: " + itemupdate_status + ")");
+    }
+
+    if (MediaFilterManager.isVerbose)
+    {
+      System.out.println("VideoFilter::getDestinationSteam() - Complete!");
+    }
+
+    // Read back in the converted video and return as stream
+    InputStream result = Files.newInputStream(converted_video_path);
+    return result;
+  }
+  /** getDestinationStream(InputStream source) **/
+
+  /** Get a filename for a newly created filtered bitstream
+   *
+   *  @param sourceName
+   *            name of source bitstream
+   *  @return filename generated by the filter - for example, document.pdf
+   *          becomes document.pdf.txt
+   */
+  public String getFilteredName(String sourceName)
+  {
+    // @todo make this dependent on the configuration options (flv|mp4)
+    return sourceName + ".mp4";
+  }
+  /** getFilteredName(String sourceName) **/
+
+  /**
+   *  @return name of the bitstream format (say "HTML" or "Microsoft Word")
+   *         returned by this filter look in the bitstream format registry or
+   *         mediafilter.cfg for valid format strings.
+   */
+  public String getFormatString()
+  {
+    return "video/mp4";
+  }
+  /** getFormatString() **/
+
+  /** Perform any pre-processing of the source bitstream *before* the actual
+   *  filtering takes place in MediaFilterManager.processBitstream().
+   *
+   *  Return true if pre-processing is successful (or no pre-processing
+   *  is necessary).  Return false if bitstream should be skipped
+   *  for any reason.
+   *
+   *  @param c
+   *            context
+   *  @param item
+   *            item containing bitstream to process
+   *  @param source
+   *            source bitstream to be processed
+   * 
+   *  @return true  if bitstream processing should continue,
+   *          false  if this bitstream should be skipped
+   */
+  public boolean preProcessBitstream(Context c, Item item, Bitstream source)
+    throws Exception
+  {
+    item_handle = item.getHandle();
+    item_id = item.getID();
+    // We can extract the aired date from the title (hopefully)
+    item_date = "Unknown";
+    DCValue titles[] = item.getMetadata("dc", "title", Item.ANY, "en_US");
+    if (titles.length > 0)
+    {
+      String title = titles[0].value;
+      Pattern date_pattern = Pattern.compile(".*(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d).*");
+      Matcher date_matcher = date_pattern.matcher(title);
+      if (date_matcher.matches())
+      {
+        item_date = date_matcher.group(1) + date_matcher.group(2) + date_matcher.group(3);
+      }
+    }
+    return true;
+  }
+  /** preProcessBitstream(Context c, Item item, Bitstream source) **/
+
+  /** Perform any post-processing of the generated bitstream *after* this
+   *  filter has already been run.
+   *
+   *  Return true if pre-processing is successful (or no pre-processing
+   *  is necessary).  Return false if bitstream should be skipped
+   *  for some reason.
+   *
+   *  @param c
+   *            context
+   *  @param item
+   *            item containing bitstream to process
+   *  @param generatedBitstream
+   *            the bitstream which was generated by
+   *            this filter.
+   *
+   *  @return true  if bitstream processing should continue,
+   *          false  if this bitstream should be skipped
+   */
+  public void postProcessBitstream(Context c, Item item, Bitstream generatedBitstream)
+    throws Exception
+  {
+    // Clean up all temporary files
+    // @todo@ some sort of recursive delete... although it looks like there is
+    //        still nothing built in
+  }
+  /** postProcessBitstream(Context c, Item item, Bitstream generatedBitstream) **/
+
+}
+
+class StreamGobbler
+extends Thread
+{
+  InputStream is;
+  String file_path;
+  boolean output_to_file;
+
+  StreamGobbler(InputStream is)
+  {
+    this.is = is;
+    this.output_to_file = false;
+  }
+
+  StreamGobbler(InputStream is, String file_path)
+  {
+    this.is = is;
+    this.file_path = file_path;
+    this.output_to_file = true;
+  }
+
+  public void run()
+  {
+    try
+    {
+      InputStreamReader isr = new InputStreamReader(is);
+      BufferedReader br = new BufferedReader(isr);
+      String line = null;
+      if (output_to_file)
+      {
+        PrintWriter pw = new PrintWriter(new BufferedOutputStream(new FileOutputStream(file_path)));
+        while ( (line = br.readLine()) != null)
+        {
+          pw.println(line);
+        }
+        pw.flush();
+        pw.close();
+      }
+      else
+      {
+        while ( (line  = br.readLine()) != null)
+        {
+          // Do nothing - equivalent to > /dev/null
+        }
+      }
+    }
+    catch (IOException ioe)
+    {
+      ioe.printStackTrace();
+    }
+  }
+}
Index: gs2-extensions/video-and-audio/trunk/src/opt/README.txt
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/README.txt	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/README.txt	(revision 26186)
@@ -0,0 +1,11 @@
+===== Optional Plugins =====
+
+A couple of smaller projects to replicate Greenstone's basic video processing in other digital libraries.  Produced as part of parallel building work, these all require Greenstone's video-and-audio component to be compiled and available via GSDLHOME.
+
+==== DSpace ====
+
+A basic video media filter.
+
+==== Terrier ====
+
+Both an image plugin and a video plugin.
Index: gs2-extensions/video-and-audio/trunk/src/opt/Terrier/ImageDocument.java
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/Terrier/ImageDocument.java	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/Terrier/ImageDocument.java	(revision 26186)
@@ -0,0 +1,312 @@
+/**
+ *  Adding support for Images in Terrier
+ *  @author: John Thompson, jmt12, #9826509
+ *
+ *  The contents of this file are subject to the Mozilla Public License
+ *  Version 1.1 (the "License"); you may not use this file except in
+ *  compliance with the License. You may obtain a copy of the License at
+ *  http://www.mozilla.org/MPL/
+ *
+ *  Software distributed under the License is distributed on an "AS IS"
+ *  basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ *  the License for the specific language governing rights and limitations
+ *  under the License.
+ *
+ *  Copyright (c) 2011 The University of Waikato. All Rights Reserved.
+ */
+package org.terrier.indexing;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.io.Reader;
+import java.lang.Thread;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.terrier.indexing.tokenisation.TokenStream;
+import org.terrier.indexing.tokenisation.Tokeniser;
+import org.terrier.utility.ApplicationSetup;
+
+public class ImageDocument
+  implements Document
+{
+  /** A reference to the logger for messaging */
+  protected static final Logger logger = Logger.getLogger(FileDocument.class);
+  /** The map of properties (fields) for this document. */
+  protected Map<String,String> properties;
+  /** A reader built from a dummy text string. */
+  protected Reader reader;
+  /** A token stream produced by the configured tokeniser when feed the dummy
+   *  reader.
+   */
+  protected TokenStream tokenizer;
+
+  /** The preview filetype. **/
+  protected final String preview_format = ApplicationSetup.getProperty("ImageDocument.preview_format", "jpg");
+  /** The preview size (width). **/
+  protected final String preview_width = ApplicationSetup.getProperty("ImageDocument.preview_width", "200");
+
+  /** Default constructor. **/
+  protected ImageDocument() {}
+
+  /** Constructs an instance of the ImageDocument from the given input stream.
+   *  @param docStream the input stream that reads the file.
+   *  @param docProperties the initial properties (docno, filename)
+   *  @param tok the tokeniser defined for this collection
+   */
+  public ImageDocument(InputStream istream, Map<String,String> default_properties, Tokeniser tok)
+  {
+    logger.info("ImageDocument::ImageDocument()");
+    // Initialization from arguments
+    this.properties = default_properties;
+
+    // Set properties
+    logger.info("ImageDocument - extracting properties");
+    // A. Hardcoded properties
+    this.properties.put("parser", "ImageDocument");
+    this.properties.put("abstract", "This is an image so here is some dummy text to prevent indexer failing.");
+    // B. Properties derived from filename
+    String title = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(System.getProperty("file.separator")) + 1);
+    this.properties.put("title", title);
+    String ext = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(".") + 1);
+    String target_filename = "doc." + ext;
+    this.properties.put("source","doc." + ext);
+    String assoc_filename = "D" + properties.get("docno");
+    this.properties.put("assocfile", assoc_filename);
+
+    // Copy (symlink) the file into place in the shared directory
+    Path source_path = Paths.get(properties.get("filename"));
+    Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename);
+    assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this
+    Path target_path = assoc_path.resolve(target_filename);
+    if (target_path.toFile().exists())
+    {
+      logger.info("ImageDocument - removing existing (old) associated image");
+      try
+      {
+        Files.delete(target_path);
+      }
+      catch (Exception e)
+      {
+        logger.error("Exception while deleting old image: ", e);
+      }
+    }
+    logger.info("ImageDocument - symlinking image into assoc directory");
+    try
+    {
+      Files.createSymbolicLink(target_path, source_path);
+    }
+    // not supported? We'll try copying below
+    catch (UnsupportedOperationException ex)
+    {
+    }
+    // All other exceptions can be fatal
+    catch (Exception e)
+    {
+      logger.error("Exception while symlinking image: ", e);
+    }
+    // - copy if the file doesn't exist yet
+    if (!target_path.toFile().exists())
+    {
+      logger.info("ImageDocument - symlink filaed, copying instead");
+      try
+      {
+        Files.copy(source_path, target_path);
+      }
+      // Fatality!
+      catch (Exception e)
+      {
+        logger.error("Exception while copying image: ", e);
+      }
+    }
+
+    // Generate preview image
+    logger.info("ImageDocument - generate preview image");
+    try
+    {
+      String preview_filename = this.generatePreview(source_path, assoc_path);
+      this.properties.put("preview",preview_filename);
+    }
+    catch (Exception e)
+    {
+      logger.error("Exception while generating preview image: ", e);
+    }
+
+    // Create a dummy reader around some dummy text and then tokenize it
+    logger.info("ImageDocument - feed dummy text as token stream to indexer");
+    try
+    {
+      this.reader = new StringReader(this.properties.get("abstract"));
+      this.tokenizer = tok.tokenise(this.reader);
+    }
+    catch (Exception e)
+    {
+      logger.error("Exception while creating dummy text stream: ", e);
+    }
+    logger.info("ImageDocument - Complete!");
+  }
+  /** ImageDocument(InputStream istream, Map<String,String> default_properties, Tokeniser tok) **/
+
+  /** Returns true when the end of the document has been reached, and there
+   *  are no other terms to be retrieved from it.
+   *  @return boolean true if there are no more terms in the document, otherwise
+   *          it returns false.
+   */
+  public boolean endOfDocument()
+  {
+    return !this.tokenizer.hasNext();
+  }
+  /** endOfDocument() **/
+
+  /** Use ImageMagick to generate a preview image.
+   *  @pre assumes you have ImageMagick installed and available on Path
+   *  @pre uses member variables preview_format and preview_width
+   *  @return the filename of the preview image (within the assoc directory)
+   */
+  private String generatePreview(Path source_path, Path assoc_path)
+    throws Exception
+  {
+    String preview_filename = "preview." + this.preview_format;
+    Path preview_path = assoc_path.resolve(preview_filename);
+    String convert_command[] = {
+      "convert",
+      source_path.toString(),
+      "-resize",
+      this.preview_width + "x",
+      preview_path.toString()
+    };
+    logger.info("ImageDocument - convert command: " + Arrays.toString(convert_command));
+    Process convert_process = Runtime.getRuntime().exec(convert_command);
+    // Gobble up the streams to prevent them hanging the process when buffers
+    // are full
+    StreamGobbler convert_process_error_gobbler = new StreamGobbler(convert_process.getErrorStream());
+    convert_process_error_gobbler.start();
+    StreamGobbler convert_process_input_gobbler = new StreamGobbler(convert_process.getInputStream());
+    convert_process_input_gobbler.start();
+    // Let the conversion finish
+    int convert_status = convert_process.waitFor();
+    if (convert_status != 0 || !preview_path.toFile().exists())
+    {
+      throw new Exception("Convert command failed (exit status: " + convert_status + ")");
+    }
+    return preview_filename;
+  }
+  /** generatePreview(Path, Path) **/
+
+  /** Returns the underlying map of all the properties defined by this Document.
+   *  @since 1.1.0
+   */
+  public Map<String,String> getAllProperties()
+  {
+    return this.properties;
+  }
+  /** getAllProperties() **/
+
+  /** Returns a list of the fields the current term appears in.
+   * @return HashSet a set of the terms that the current term appears in.
+   */
+  public Set<String> getFields()
+  {
+    // Returns null because there is no support for fields with file documents.
+    return Collections.emptySet();
+  }
+  /** getFields() **/
+
+  /** Gets the next term of the document.
+   *  <B>NB:</B>Null string returned from getNextTerm() should
+   *  be ignored. They do not signify the lack of any more terms.
+   *  endOfDocument() should be used to check that.
+   *  @return String the next term of the document. Null returns should be
+   *          ignored.
+   */
+  public String getNextTerm()
+  {
+    return this.tokenizer.next();
+  }
+  /** getNextTerm() **/
+
+  /** Allows access to a named property of the Document. Examples might be URL,
+   *  filename etc.
+   *  @param name Name of the property. It is suggested, but not required that
+   *         this name should not be case insensitive.
+   *  @since 1.1.0
+   */
+  public String getProperty(String name)
+  {
+    return this.properties.get(name.toLowerCase());
+  }
+  /** getProperty(String name) **/
+
+  /** Returns a Reader object so client code can tokenise the document
+   * or deal with the document itself. Examples might be extracting URLs,
+   * language detection. */
+  public Reader getReader()
+  {
+    return this.reader;
+  }
+  /** getReader() **/
+}
+
+class StreamGobbler
+extends Thread
+{
+  InputStream is;
+  String file_path;
+  boolean output_to_file;
+
+  StreamGobbler(InputStream is)
+  {
+    this.is = is;
+    this.output_to_file = false;
+  }
+
+  StreamGobbler(InputStream is, String file_path)
+  {
+    this.is = is;
+    this.file_path = file_path;
+    this.output_to_file = true;
+  }
+
+  public void run()
+  {
+    try
+    {
+      InputStreamReader isr = new InputStreamReader(is);
+      BufferedReader br = new BufferedReader(isr);
+      String line = null;
+      if (output_to_file)
+      {
+        PrintWriter pw = new PrintWriter(new BufferedOutputStream(new FileOutputStream(file_path)));
+        while ( (line = br.readLine()) != null)
+        {
+          pw.println(line);
+        }
+        pw.flush();
+        pw.close();
+      }
+      else
+      {
+        while ( (line  = br.readLine()) != null)
+        {
+          // Do nothing - equivalent to > /dev/null
+        }
+      }
+    }
+    catch (IOException ioe)
+    {
+      ioe.printStackTrace();
+    }
+  }
+}
Index: gs2-extensions/video-and-audio/trunk/src/opt/Terrier/README.txt
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/Terrier/README.txt	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/Terrier/README.txt	(revision 26186)
@@ -0,0 +1,28 @@
+===== Image and Video support for Terrier =====
+
+==== Installation Instructions ====
+
+1. Copy the two document plugins into Terrier:
+
+cd <greenstone>/ext/video-and-audio/opt/Terrier/
+cp *Document.java <terrier>/src/core/org/terrier/indexing/
+
+2. Recompile:
+
+cd <terrier>
+ant
+
+3. Edit the configuration to include the new document plugins. Your usage may vary depending on what type of Terrier installation you are setting up, but I'll show the example of the configuration for a SimpleFileIndexer installation - available as part of the parallel-building extension:
+
+cd <terrier>/etc/
+emacs terrier.properties
+###################################################################
+#SimpleFileCollection specific properties
+###################################################################
+trec.collection.class=SimpleFileCollection
+##use this to define parsers for know file extensions
+indexing.simplefilecollection.extensionsparsers=txt:FileDocument, pdf:PDFDocument, html:TaggedDocument,htm:TaggedDocument,xhtml:TaggedDocument,xml:TaggedDocument, jpg:ImageDocument, ts:VideoDocument
+
+4. Build your index including some JPG and TS files.
+
+5. Test the index in your preferred tool, and marvel at the beautiful image and video metadata (or thumbnails, if you are in a web interface)
Index: gs2-extensions/video-and-audio/trunk/src/opt/Terrier/VideoDocument.java
===================================================================
--- gs2-extensions/video-and-audio/trunk/src/opt/Terrier/VideoDocument.java	(revision 26186)
+++ gs2-extensions/video-and-audio/trunk/src/opt/Terrier/VideoDocument.java	(revision 26186)
@@ -0,0 +1,446 @@
+/**
+ *  Adding support for Videos in Terrier
+ *  @author: John Thompson, jmt12, #9826509
+ *
+ *  The contents of this file are subject to the Mozilla Public License
+ *  Version 1.1 (the "License"); you may not use this file except in
+ *  compliance with the License. You may obtain a copy of the License at
+ *  http://www.mozilla.org/MPL/
+ *
+ *  Software distributed under the License is distributed on an "AS IS"
+ *  basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ *  the License for the specific language governing rights and limitations
+ *  under the License.
+ *
+ *  Copyright (c) 2011 The University of Waikato. All Rights Reserved.
+ */
+package org.terrier.indexing;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.io.Reader;
+import java.lang.Thread;
+import java.nio.file.Files;
+import java.nio.file.FileVisitResult;
+import static java.nio.file.FileVisitResult.*;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Collections;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+import org.terrier.indexing.tokenisation.TokenStream;
+import org.terrier.indexing.tokenisation.Tokeniser;
+import org.terrier.utility.ApplicationSetup;
+
+public class VideoDocument
+  implements Document
+{
+  /** A reference to the logger for messaging */
+  protected static final Logger logger = Logger.getLogger(FileDocument.class);
+  /** The map of properties (fields) for this document. */
+  protected Map<String,String> properties;
+  /** A reader built from a dummy text string. */
+  protected Reader reader;
+  /** A token stream produced by the configured tokeniser when feed the dummy
+   *  reader.
+   */
+  protected TokenStream tokenizer;
+
+  // Handbrake Configuration
+  protected String streaming_hq_size = ApplicationSetup.getProperty("VideoDocument.streaming_hq_size", "720");
+  protected String streaming_hq_video_bitrate = ApplicationSetup.getProperty("VideoDocument.streaming_hq_video_bitrate", "496");
+  protected String streaming_hq_audio_bitrate = ApplicationSetup.getProperty("VideoDocument.streaming_hq_audio_bitrate", "80");
+
+  /** The preview size (width). **/
+  protected final String preview_width = ApplicationSetup.getProperty("ImageDocument.preview_width", "200");
+
+
+  /** Default constructor. **/
+  protected VideoDocument() {}
+
+  /** Constructs an instance of the ImageDocument from the given input stream.
+   *  @param docStream the input stream that reads the file.
+   *  @param docProperties the initial properties (docno, filename)
+   *  @param tok the tokeniser defined for this collection
+   */
+  public VideoDocument(InputStream istream, Map<String,String> default_properties, Tokeniser tok)
+  {
+    logger.info("VideoDocument::VideoDocument()");
+    // 0. Initialization from arguments
+    this.properties = default_properties;
+
+    // Set properties
+    logger.info("VideoDocument - extracting properties");
+    // A. Hardcoded properties
+    this.properties.put("parser", "VideoDocument");
+    this.properties.put("abstract", "This is a video so here is some dummy text to prevent indexer failing.");
+    // B. Properties derived from filename
+    String title = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(System.getProperty("file.separator")) + 1);
+    this.properties.put("title", title);
+    String ext = this.properties.get("filename").substring(properties.get("filename").lastIndexOf(".") + 1);
+    String target_filename = "doc." + ext;
+    this.properties.put("source","doc." + ext);
+    String assoc_filename = "D" + properties.get("docno");
+    this.properties.put("assocfile", assoc_filename);
+
+    // Copy (symlink) the file into place in the shared directory
+    Path raw_video_path = Paths.get(properties.get("filename"));
+    Path assoc_path = Paths.get(ApplicationSetup.TERRIER_SHARE, "images", "assoc", assoc_filename);
+    // - if the assoc path already exists, we need to recursively delete it and
+    //   its contents
+    if (Files.exists(assoc_path))
+    {
+      logger.info("VideoDocument - removing existing (old) associated files");
+      try
+      {
+        Files.walkFileTree(assoc_path, new SimpleFileVisitor<Path>()
+        {
+          @Override
+          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+            throws IOException
+          {
+            ///ystem.out.println("Deleting file: " + file);
+            Files.delete(file);
+            return CONTINUE;
+          }
+          @Override
+          public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+            throws IOException
+          {
+            ///ystem.out.println("Deleting dir: " + dir);
+            if (exc == null)
+            {
+              Files.delete(dir);
+              return CONTINUE;
+            }
+            else
+            {
+              throw exc;
+            }
+          }
+        });
+      }
+      catch (Exception e)
+      {
+        logger.error("Exception while recursively deleting assoc folder:", e);
+      }
+    }
+    assoc_path.toFile().mkdirs(); // bet there is a nice nio way to do this
+    Path target_path = assoc_path.resolve(target_filename);
+    logger.info("VideoDocument - symlinking original video into assoc directory");
+    try
+    {
+      Files.createSymbolicLink(target_path, raw_video_path);
+    }
+    // not supported? We'll try copying below
+    catch (UnsupportedOperationException ex)
+    {
+    }
+    // All other exceptions can be fatal
+    catch (Exception e)
+    {
+      logger.error("Exception while symlinking video: ", e);
+    }
+    // - copy if the file doesn't exist yet
+    if (!target_path.toFile().exists())
+    {
+      logger.info("VideoDocument - symlink failed, copying instead");
+      try
+      {
+        Files.copy(raw_video_path, target_path);
+      }
+      // Fatality!
+      catch (Exception e)
+      {
+        logger.error("Exception while copying video: ", e);
+      }
+    }
+
+    // 1. Extract Metadata using MediaInfo and store as properties
+    logger.info("VideoDocument - extracting video metadata");
+    try
+    {
+      String metadata_command[] = {
+        "mediainfo",
+        "--Output=XML",
+        raw_video_path.toString()
+      };
+      logger.info("VideoDocument - metadata command: " + Arrays.toString(metadata_command));
+      Process metadata_process = Runtime.getRuntime().exec(metadata_command);
+      StreamGobbler metadata_process_error_gobbler = new StreamGobbler(metadata_process.getErrorStream());
+      metadata_process_error_gobbler.start();
+      BufferedReader metadata_br = new BufferedReader(new InputStreamReader(metadata_process.getInputStream()));
+      String line;
+      String type = "Unknown";
+      Pattern type_pattern = Pattern.compile("<track type=\"([a-zA-Z]+)\">");
+      Pattern metadata_pattern = Pattern.compile("<([a-zA-Z_]+)>(.*)</\\1>");
+      while ((line = metadata_br.readLine()) != null)
+      {
+        Matcher type_matcher = type_pattern.matcher(line);
+        if (type_matcher.matches())
+        {
+          type = type_matcher.group(1);
+        }
+        else
+        {
+          Matcher metadata_matcher = metadata_pattern.matcher(line);
+          if (metadata_matcher.matches())
+          {
+            String field = type.toLowerCase() + ":" + metadata_matcher.group(1).toLowerCase();
+            String value = metadata_matcher.group(2);
+            logger.info("VideoDocument - adding metadata: " + field + " => " + value);
+            this.properties.put(field, value);
+          }
+        }
+      }
+      int metadata_status = metadata_process.waitFor();
+    }
+    catch (Exception e)
+    {
+      logger.error("Exception while extracting video metadata:", e);
+    }
+
+    // 2. Convert Video to streamable format using HandbrakeCLI
+    logger.info("VideoDocument - convert video to streamable format");
+    Path converted_video_path = assoc_path.resolve("tsv.mp4");
+    try
+    {
+      String convert_command[] = {
+        "HandBrakeCLI",
+        "-i", raw_video_path.toString(),
+        "-t", "1",
+        "-c", "1",
+        "-o", converted_video_path.toString(),
+        "-f", "mp4",
+        "-O",
+        "-w", this.streaming_hq_size,
+        "--loose-anamorphic",
+        "-e", "x264",
+        "-b", this.streaming_hq_video_bitrate,
+        "-a", "1",
+        "-E", "faac",
+        "-6", "dpl2",
+        "-R", "Auto",
+        "-B", this.streaming_hq_audio_bitrate,
+        "-D", "0.0",
+        "-x", "ref=2:bframes=2:subq=6:mixed-refs=0:weightb=0:8x8dct=0:trellis=0:threads=1"
+      };
+      logger.info("VideoDocument - convert command: " + Arrays.toString(convert_command));
+      // @todo determine the best way to account for configuration options
+      Process convert_process = Runtime.getRuntime().exec(convert_command);
+      StreamGobbler convert_process_error_gobbler = new StreamGobbler(convert_process.getErrorStream());
+      convert_process_error_gobbler.start();
+      StreamGobbler convert_process_out_gobbler = new StreamGobbler(convert_process.getInputStream());
+      convert_process_out_gobbler.start();
+      int convert_status = convert_process.waitFor();
+      if (convert_status != 0 || !Files.exists(converted_video_path))
+      {
+        throw new Exception("Convert command failed (exit status: " + convert_status + ")");
+      }
+    }
+    catch (Exception e)
+    {
+      logger.error("Exception while converting video to streamable format: ", e);
+    }
+
+    // 3. Generate keyframes from streamable video and attach the shot names
+    //    as a property
+    logger.info("VideoDocument - extracting keyframes from video");
+    try
+    {
+      Path shots_path = assoc_path.resolve("shots.xml");
+      String keyframe_command[] = {
+        "hive2_ffmpegsvn",
+        "-o", shots_path.toString(),
+        "-k", assoc_path.toString(),
+        "-m", "0.5",
+        "-l", "0.05",
+        converted_video_path.toString()
+      };
+      logger.info("VideoDocument - keyframe command: " + Arrays.toString(keyframe_command));
+      Process keyframe_process = Runtime.getRuntime().exec(keyframe_command);
+      //Path keyframe_err_file = temp_dir_path.resolve("hive2-err.txt");
+      StreamGobbler keyframe_error_gobbler = new StreamGobbler(keyframe_process.getErrorStream());//, keyframe_err_file.toString());
+      keyframe_error_gobbler.start();
+      //Path keyframe_out_file = temp_dir_path.resolve("hive2-out.txt");
+      StreamGobbler keyframe_out_gobbler = new StreamGobbler(keyframe_process.getInputStream()); //, keyframe_out_file.toString());
+      keyframe_out_gobbler.start();
+      int keyframe_status = keyframe_process.waitFor();
+      if (keyframe_status != 0 || !Files.exists(shots_path))
+      {
+        throw new Exception("Keyframe command failed (exit status: " + keyframe_status + ")");
+      }
+      File files[] = assoc_path.toFile().listFiles();
+      Arrays.sort(files);
+      Pattern image_filename_pattern = Pattern.compile("tsv.*\\.jpg");
+      String keyframe_images = "";
+      for (int i = 0; i < files.length; i++)
+      {
+        String image_filename = files[i].toPath().getFileName().toString();
+        logger.info("VideoDocument - considering keyframe image: " + image_filename);
+        Matcher image_filename_matcher = image_filename_pattern.matcher(image_filename);
+        if (image_filename_matcher.matches())
+        {
+          if (keyframe_images.equals(""))
+          {
+            keyframe_images = image_filename;
+          }
+          else
+          {
+            keyframe_images += "," + image_filename;
+          }
+        }
+      }
+      this.properties.put("preview", keyframe_images);
+    }
+    catch (Exception e)
+    {
+      logger.error("Exception while extracting keyframes from video: ", e);
+    }
+
+    // 4. Create a dummy reader around some dummy text and then tokenize it
+    logger.info("VideoDocument - feed dummy text as token stream to indexer");
+    try
+    {
+      this.reader = new StringReader(this.properties.get("abstract"));
+      this.tokenizer = tok.tokenise(this.reader);
+    }
+    catch (Exception e)
+    {
+      logger.error("Exception while creating dummy text stream: ", e);
+    }
+    logger.info("VideoDocument - Complete!");
+  }
+  /** VideoDocument(InputStream istream, Map<String,String> default_properties, Tokeniser tok) **/
+
+  /** Returns true when the end of the document has been reached, and there
+   *  are no other terms to be retrieved from it.
+   *  @return boolean true if there are no more terms in the document, otherwise
+   *          it returns false.
+   */
+  public boolean endOfDocument()
+  {
+    return !this.tokenizer.hasNext();
+  }
+  /** endOfDocument() **/
+
+  /** Returns the underlying map of all the properties defined by this Document.
+   *  @since 1.1.0
+   */
+  public Map<String,String> getAllProperties()
+  {
+    return this.properties;
+  }
+  /** getAllProperties() **/
+
+  /** Returns a list of the fields the current term appears in.
+   * @return HashSet a set of the terms that the current term appears in.
+   */
+  public Set<String> getFields()
+  {
+    // Returns null because there is no support for fields with file documents.
+    return Collections.emptySet();
+  }
+  /** getFields() **/
+
+  /** Gets the next term of the document.
+   *  <B>NB:</B>Null string returned from getNextTerm() should
+   *  be ignored. They do not signify the lack of any more terms.
+   *  endOfDocument() should be used to check that.
+   *  @return String the next term of the document. Null returns should be
+   *          ignored.
+   */
+  public String getNextTerm()
+  {
+    return this.tokenizer.next();
+  }
+  /** getNextTerm() **/
+
+  /** Allows access to a named property of the Document. Examples might be URL,
+   *  filename etc.
+   *  @param name Name of the property. It is suggested, but not required that
+   *         this name should not be case insensitive.
+   *  @since 1.1.0
+   */
+  public String getProperty(String name)
+  {
+    return this.properties.get(name.toLowerCase());
+  }
+  /** getProperty(String name) **/
+
+  /** Returns a Reader object so client code can tokenise the document
+   * or deal with the document itself. Examples might be extracting URLs,
+   * language detection. */
+  public Reader getReader()
+  {
+    return this.reader;
+  }
+  /** getReader() **/
+}
+
+class StreamGobbler
+extends Thread
+{
+  InputStream is;
+  String file_path;
+  boolean output_to_file;
+
+  StreamGobbler(InputStream is)
+  {
+    this.is = is;
+    this.output_to_file = false;
+  }
+
+  StreamGobbler(InputStream is, String file_path)
+  {
+    this.is = is;
+    this.file_path = file_path;
+    this.output_to_file = true;
+  }
+
+  public void run()
+  {
+    try
+    {
+      InputStreamReader isr = new InputStreamReader(is);
+      BufferedReader br = new BufferedReader(isr);
+      String line = null;
+      if (output_to_file)
+      {
+        PrintWriter pw = new PrintWriter(new BufferedOutputStream(new FileOutputStream(file_path)));
+        while ( (line = br.readLine()) != null)
+        {
+          pw.println(line);
+        }
+        pw.flush();
+        pw.close();
+      }
+      else
+      {
+        while ( (line  = br.readLine()) != null)
+        {
+          // Do nothing - equivalent to > /dev/null
+        }
+      }
+    }
+    catch (IOException ioe)
+    {
+      ioe.printStackTrace();
+    }
+  }
+}
+
