Index: /gs2-extensions/video-and-audio/trunk/src/opt/Terrier/SimpleFileCollection.java
===================================================================
--- /gs2-extensions/video-and-audio/trunk/src/opt/Terrier/SimpleFileCollection.java	(revision 26957)
+++ /gs2-extensions/video-and-audio/trunk/src/opt/Terrier/SimpleFileCollection.java	(revision 26957)
@@ -0,0 +1,434 @@
+/*
+ * Terrier - Terabyte Retriever
+ * Webpage: http://terrier.org
+ * Contact: terrier{a.}dcs.gla.ac.uk
+ * University of Glasgow - School of Computing Science
+ * http://www.gla.ac.uk/
+ *
+ * The contents of this file are subject to the Mozilla Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is SimpleFileCollection.java.
+ *
+ * The Original Code is Copyright (C) 2004-2011 the University of Glasgow.
+ * All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Craig Macdonald <craigm{a.}dcs.gla.ac.uk> (original author)
+ *   Vassilis Plachouras <vassilis{a.}dcs.gla.ac.uk>
+ */
+package org.terrier.indexing;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.terrier.indexing.tokenisation.Tokeniser;
+import org.terrier.utility.ApplicationSetup;
+import org.terrier.utility.Files;
+/** 
+ * Implements a collection that can read arbitrary files on disk. It will 
+ * use the file list given to it in the constructor, or it will read the
+ * file specified by the property <tt>collection.spec</tt>.
+ * <b>Properties:</b>
+ * <ul>
+ * <li><tt>indexing.simplefilecollection.extensionsparsers</tt> - a comma delimited lists of tuples, in the form "extension:DocumentClass". 
+ * For instance, one tuple could be "txt:FileDocument". The default <tt>txt:FileDocument,text:FileDocument,tex:FileDocument,bib:FileDocument,pdf:PDFDocument,html:TaggedDocument,htm:TaggedDocument,xhtml:TaggedDocument,xml:TaggedDocument,doc:MSWordDocument,ppt:MSPowerpointDocument,xls:MSExcelDocument</tt>.
+ * </li>
+ * <li><tt>indexing.simplefilecollection.defaultparser</tt> - the default parser for any unknown extensions.
+ * If this property is empty, then such documents will not be opened.</li>
+ * <li><tt>indexing.simplefilecollection.recurse</tt> - whether directories should be opened looking for
+ * files.</li>
+ * </ul>
+ * @author Craig Macdonald &amp; Vassilis Plachouras
+ */
+public class SimpleFileCollection implements Collection/*, DocumentExtractor*/
+{
+	protected static final Logger logger = Logger.getLogger(SimpleFileCollection.class);
+	/** The default namespace for all parsers to be loaded from. Only used if
+	  * the class name specified does not contain any periods ('.') */
+	public final static String NAMESPACE_DOCUMENTS = "org.terrier.indexing.";
+
+	/** The list of files to index.*/
+	protected LinkedList<String> FileList = new LinkedList<String>();
+
+	/** Contains the list of files first handed to the SimpleFileCollection, allowing
+	  * the SimpleFileCollection instance to be simply reset. */
+	protected List<String> firstList;
+
+	/** This is filled during traversal, so document IDs can be matched with filenames */
+	protected List<String> indexedFiles = new ArrayList<String>();
+
+	/** The identifier of a document in the collection.*/
+	protected int Docid = 0;
+
+	/** Whether directories should be recursed into by this class */
+	protected boolean Recurse = Boolean.parseBoolean(ApplicationSetup.getProperty("indexing.simplefilecollection.recurse", "true"));
+
+	/** Maps filename extensions to Document classes. 
+	  * The entry |DEFAULT| maps to the default document parser, specified 
+	  * by <tt>indexing.simplefilecollection.defaultparser</tt> */
+	protected Map<String,Class<? extends Document>> extension_DocumentClass = new HashMap<String,Class<? extends Document>>();
+
+	/** The filename of the current file we are processing. */
+	protected String thisFilename;
+
+	/** The InputStream of the most recently opened document. This
+	  * is used to ensure that files are closed once they have been
+	  * finished reading. */
+	protected InputStream currentStream = null;
+	
+	protected Tokeniser tokeniser = Tokeniser.getTokeniser();
+
+	/**
+	 * Constructs an instance of the class with the given list of files.
+	 * @param filelist ArrayList the files to be processed by this collection.
+	 */
+	public SimpleFileCollection(List<String> filelist, boolean recurse) {
+		FileList = new LinkedList<String>(filelist);
+		//keep a backup copy for reset()
+		firstList = new LinkedList<String>(filelist);
+		createExtensionDocumentMapping();
+	}
+
+	/**
+	 * A default constructor that uses the files to be processed
+	 * by this collection, as specified by the property
+	 * <tt>collection.spec</tt>
+	 */
+	public SimpleFileCollection()
+	{
+		this(ApplicationSetup.COLLECTION_SPEC);
+	}
+
+
+	/**
+	 * Creates an instance of the class. The files to be processed are
+	 * specified in the file with the given name. 
+	 * @param addressCollectionFilename String the name of the file that 
+	 *        contains the list of files to be processed by this collecion.
+	 */
+	public SimpleFileCollection(String addressCollectionFilename)
+	{
+		ArrayList<String> generatedFileList = new ArrayList<String>();
+		try{
+			//opening the address_collection file
+			BufferedReader br = Files.openFileReader(addressCollectionFilename);
+			//iterate through each entry of the address_collection file
+			String filename = br.readLine();
+			while (filename != null) {
+				//if the line starts with #, then assume it is 
+				//a comment and proceed to the next one
+				if (filename.startsWith("#")) {
+					filename = br.readLine();
+					continue;
+				}
+				if(logger.isDebugEnabled()){
+				logger.debug("Added "+filename+" to filelist for SimpleFileCollection");
+				}
+				generatedFileList.add(filename);
+				filename = br.readLine();
+			}
+		
+		}catch(IOException ioe) {
+			logger.error("problem opening address list of files in SimpleFileCollectio: ",ioe);	
+		}
+		FileList = new LinkedList<String>(generatedFileList);
+		firstList = new LinkedList<String>(generatedFileList);
+		createExtensionDocumentMapping();
+	}
+
+
+	/** Parses the properties <tt>indexing.simplefilecollection.extensionsparsers</tt>
+	  * and <tt>indexing.simplefilecollection.defaultparser</tt> and attempts to load
+	  * all the mentioned classes, in a hashtable mapping filename extension to their
+	  * respective parsers. If <tt>indexing.simplefilecollection.defaultparser</tt>
+	  * is set, then that class will be used to attempt to parse documents that no
+	  * explicit parser is set. */
+	protected void createExtensionDocumentMapping()
+	{
+		String staticMappings = ApplicationSetup.getProperty("indexing.simplefilecollection.extensionsparsers",
+			"txt:FileDocument,text:FileDocument,tex:FileDocument,bib:FileDocument," +
+			"pdf:PDFDocument,html:TaggedDocument,htm:TaggedDocument,xhtml:TaggedDocument,xml:TaggedDocument,"+
+			"doc:MSWordDocument,ppt:MSPowerpointDocument,xls:MSExcelDocument");
+		String defaultMapping = ApplicationSetup.getProperty("indexing.simplefilecollection.defaultparser","");
+		if (staticMappings.length() > 0)
+		{
+			String[] mappings = staticMappings.split("\\s*,\\s*");
+			for(int i=0;i<mappings.length;i++)
+			{
+				if (mappings[i].indexOf(":") < 1)
+					continue;
+				String[] mapping = mappings[i].split(":");
+				if (mapping.length == 2 && mapping[0].length() > 0
+					&& mapping[1].length() > 0)
+				{
+					if (mapping[1].indexOf(".") == -1)
+						mapping[1] = NAMESPACE_DOCUMENTS + mapping[1];
+					else if (mapping[1].startsWith("uk.ac.gla.terrier"))
+						mapping[1] = mapping[1].replaceAll("uk.ac.gla.terrier", "org.terrier");
+					try{
+						Class<? extends Document> d = Class.forName(mapping[1], false, this.getClass().getClassLoader()).asSubclass(Document.class);
+						extension_DocumentClass.put(mapping[0].toLowerCase(), d);
+					}catch (Exception e){
+						/*warning, just ignore */
+						logger.warn("Missing class " + mapping[1] + " for " +
+							mapping[0].toLowerCase() + " files.",e);
+					}
+				}
+			}
+		}
+		//set the mapping for the default parser
+		if (!defaultMapping.equals("")) {
+  			if (defaultMapping.indexOf(".") == -1)
+  				defaultMapping = NAMESPACE_DOCUMENTS + defaultMapping;
+  			else if (defaultMapping.startsWith("uk.ac.gla.terrier"))
+  				defaultMapping = defaultMapping.replaceAll("uk.ac.gla.terrier", "org.terrier");
+			
+  			try{
+  				Class<? extends Document> d =  Class.forName(defaultMapping, false, this.getClass().getClassLoader()).asSubclass(Document.class);
+  				extension_DocumentClass.put("|DEFAULT|", d);
+  			}catch (Exception e){
+  				logger.warn("Missing default class " + defaultMapping, e);
+  			}  			
+  		}
+	}
+
+	/**
+	 * Check whether there is a next document in the collection to be processed
+	 * @return has next
+	 */
+	public boolean hasNext() {
+		return ! endOfCollection();
+	}
+	
+	/**
+	 * Move onto the next document in the collection to be processed.
+	 * @return next document
+	 */
+	public Document next()
+	{
+		nextDocument();
+		return getDocument();
+	}
+	
+	/**
+	 * This is unsupported by this Collection implementation, and
+	 * any calls will throw UnsupportedOperationException
+	 * Throws UnsupportedOperationException on all invocations */
+	public void remove()
+	{
+		throw new UnsupportedOperationException("Iterator.remove() not supported");
+	}
+
+	/** 
+	 * Move onto the next document in the collection to be processed.
+	 * @return boolean true if there are more documents in the collection, 
+	 *         otherwise return false.*/
+	public boolean nextDocument()
+	{
+		if (FileList.size() == 0)
+			return false;
+		boolean rtr = false;
+		thisFilename = null;
+		while(FileList.size() > 0 && ! rtr)
+		{
+			thisFilename = FileList.removeFirst();
+			logger.info("NEXT: "+thisFilename);
+				
+			if (! Files.exists(thisFilename) || ! Files.canRead(thisFilename) )
+			{
+				if (! Files.exists(thisFilename))
+					logger.warn("File doesn't exist: "+thisFilename);
+				else if (! Files.canRead(thisFilename) )
+					logger.warn("File cannot be read: "+thisFilename);
+				rtr = nextDocument();
+			}
+			else if (Files.isDirectory(thisFilename))
+			{
+				//we're allowed to recurse into directories
+				if(Recurse)
+					addDirectoryListing();
+			}
+			else
+			{	//this file is fine - use it!
+				//this block ensures that DocId is only increased once per file
+				Docid++;
+				rtr = true;
+			}
+		}//loop ends
+		return rtr;
+	}
+	/**
+	 * Return the current document in the collection.
+	 * @return Document the next document object from the collection.
+	 */
+	public Document getDocument()
+	{
+		InputStream in = null;
+		if (currentStream != null)
+		{
+			try{
+				currentStream.close();
+				currentStream = null;
+			}catch (IOException ioe) {
+				logger.warn("IOException while closing file being read", ioe);
+			}
+		}
+		if (thisFilename == null)
+		{
+			return null;
+		}
+		String filename = null;
+		try{
+			in = Files.openFileStream(thisFilename);
+			filename = thisFilename.replaceAll("\\.gz$","");
+		}catch(IOException ioe){
+			logger.warn("Problem reading "+thisFilename+" in "+
+			"SimpleFileCollection.getDocuent() : ",ioe);
+		}
+		currentStream = in;
+		return makeDocument(filename, in);
+
+	}
+
+
+	/** Given the opened document in, of Filename and File f, work out which
+	  * parser to try, and instantiate it. If you wish to use a different
+	  * constructor for opening documents, then you need to subclass this method.
+	  * @param Filename the filename of the currently open document
+	  * @param in The stream of the currently open document 
+	  * @return Document object to parse the document, or null if no suitable parser
+      * exists.*/
+	protected Document makeDocument(String Filename, InputStream in)
+	{
+		if (Filename == null || in == null)
+			return null;
+		String[] splitStr = Filename.split("\\.");
+		String ext = splitStr[splitStr.length-1].toLowerCase();
+		Class<? extends Document> reader = extension_DocumentClass.get(ext);
+		Document rtr = null;
+		
+		/*If a document doesn't have an associated parser,
+		  check the default one */
+		if (reader == null) {
+			reader = extension_DocumentClass.get("|DEFAULT|");
+		} 
+		/*if there is no default parser, then tough luck for that file,
+		  but it's ignored */
+		if (reader == null) {
+			logger.warn("No available parser for file " + Filename + ", file is ignored.");
+			return null;
+		}
+		logger.debug("Using "+reader.getName() + " to read "+ Filename);
+			
+		/* now attempt to instantiate the class */	
+		try{
+			Map<String,String> docProperties = new HashMap<String,String>(5);
+			docProperties.put("filename", Filename);
+			//and instantiate
+			rtr = reader.getConstructor(InputStream.class, Map.class, Tokeniser.class).newInstance(in, docProperties, tokeniser);
+			indexedFiles.add(thisFilename);
+		}catch (OutOfMemoryError e){
+			logger.warn("Problem instantiating a document class; Out of memory error occured: ",e);			
+			System.gc();
+		}catch (StackOverflowError e){
+			logger.warn("Problem instantiating a document class; Stack Overflow error occured: ",e);			
+		}catch (Exception e){
+			logger.warn("Problem instantiating a document class: ",e);			
+		}
+		rtr.getAllProperties().put("docno", this.getDocid());
+		return rtr;
+	}
+
+	/** 
+	 * Checks whether there are more documents in the colection.
+	 * @return boolean true if there are no more documents in the collection,
+	 *         otherwise it returns false.
+	 */	
+	public boolean endOfCollection()
+	{
+		return (FileList.size() == 0);
+	}
+
+	/** 
+	 * Starts again from the beginning of the collection.
+	 */
+	public void reset()
+	{
+		Docid = 0;
+		FileList = new LinkedList<String>(firstList);
+		indexedFiles = new ArrayList<String>();
+	}
+
+	/** 
+	 * Returns the current document's identifier string.
+	 * @return String the identifier of the current document.
+	 */	
+	public String getDocid()
+	{
+		return Docid+"";
+	}
+
+	@Override
+	public void close() 
+	{
+		if (currentStream != null)
+		{
+			try{
+				currentStream.close();
+			} catch (IOException ioe) {
+					logger.error("Exception occured while trying to close an IO stream",ioe);
+			}
+		}
+	}
+
+	/** Returns the ist of indexed files in the order they were indexed in. */
+	public List<String> getFileList()
+	{
+		return indexedFiles;
+	}
+
+	/** Called when <tt>thisFile</tt> is identified as a directory, this adds the entire
+	  * contents of the directory onto the list to be processed. */
+	protected void addDirectoryListing()
+	{
+		//File[] contents = thisFile.listFiles();
+		String[] dirContents = Files.list( thisFilename );
+		if (dirContents == null)
+			return;
+		for(String e : dirContents)
+		{
+			if (e.equals(".") || e.equals(".."))
+				continue;
+			FileList.add(thisFilename + ApplicationSetup.FILE_SEPARATOR + e);
+		}
+		/*for(int i=0;i<contents.length;i++)
+		{
+			FileList.add(contents[i].getAbsolutePath());
+		}*/
+	}
+
+	/**
+	 * Simple test case. Pass the filename of a file that lists files
+	 * to be processed to this test case.
+	 */
+	public static void main(String[] args) {
+		Indexer in = new BasicIndexer(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX);
+		in.createDirectIndex(new Collection[] {new SimpleFileCollection(args[0])});
+		in.createInvertedIndex();
+	}
+
+}
