Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33577)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33578)
@@ -2,12 +2,49 @@
 
 import java.io.*;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.ArrayList;
+//import java.util.HashMap;
+//import java.util.Map;
 import java.lang.ArrayIndexOutOfBoundsException;
-    
+
+import org.apache.log4j.Logger;
+
+/**
+ * Class to process the dump text files produced for each site (e.g. site "00001") that
+ * Nutch has finished crawling and whose text has been dumped out to a file called dump.txt.
+ * This reads in the dump.txt file contained in each site folder within the input folder.
+ * (e.g. input folder "crawled" could contain folders 00001 to 01465. Each contains a dump.txt)
+ * Each dump.txt could contain the text contents for an entire site, or for individual pages.
+ * This class then uses class TextDumpPage to parse each webpage within a dump.txt,
+ * which parses out the actual text body content of each webpage's section within a dump.txt.
+ * Finally, MaoriTextDetector is run over that to determine whether the full body text is
+ * likely to be in Maori or not.
+ *
+ * Potential issues: since a web page's text is dumped out by nutch with neither paragraph
+ * nor even newline separator, it's hard to be sure that the entire page is in language.
+ * If it's in multiple languages, there's no way to be sure there aren't promising Maori language
+ * paragraphs contained in a page, if the majority/the remainder happen to be in English.
+ * 
+ * So if we're looking for any paragraphs in Maori to store in a DB, perhaps it's better to run
+ * the MaoriTextDetector.isTextInMaori(BufferedReader reader) over two "lines" at a time,
+ * instead of running it over the entire html body's text.
+ *
+ * TO COMPILE OR RUN, FIRST DO:
+ *    cd maori-lang-detection/apache-opennlp-1.9.1
+ *    export OPENNLP_HOME=`pwd`
+ *    cd maori-lang-detection/src
+ *
+ * TO COMPILE:
+ *    maori-lang-detection/src$
+ *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/*" org/greenstone/atea/NutchTextDumpProcessor.java
+ *
+ * TO RUN:
+ *    maori-lang-detection/src$
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor
+ *
+*/
 public class NutchTextDumpProcessor {
     private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
 
-    private static MaoriTextDetector maoriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
+    private final MaoriTextDetector maoriTxtDetector;
     
     public final String siteID; // is this necessary?
@@ -17,9 +54,9 @@
     
     
-    public NutchTextDumpProcessor(String siteID, File txtDumpFile) {
+    public NutchTextDumpProcessor(MaoriTextDetector maoriTxtDetector, String siteID, File txtDumpFile) {
 	// siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
 	this.siteID = siteID;
-
-
+	this.maoriTxtDetector = maoriTxtDetector;
+    
 	pages = new ArrayList<TextDumpPage>();
 
@@ -37,5 +74,5 @@
 		    pageDump.append("\n");
 		} else {
-		    TextDumpPage page = new TextDumpPage(pageDump.toString());
+		    TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
 		    // parses the fields and body text of a webpage in nutch's txt dump of entire site
 		    //page.parseFields();
@@ -57,4 +94,6 @@
 
 	String text = getTextForPage(pageID);
+	// QTODO: what to do when page body text is empty?
+	if(text.equals("")) return false;
 	return maoriTxtDetector.isTextInMaori(text);
     }
@@ -120,8 +159,9 @@
 
 	try {
+	    MaoriTextDetector mriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
 	    File[] sites = sitesDir.listFiles();
 	    for(File siteDir : sites) { // e.g. 00001
 		// look for dump.txt
-		File txtDumpFile = new File(siteDir, dump.txt);
+		File txtDumpFile = new File(siteDir, "dump.txt");
 		if(!txtDumpFile.exists()) {
 		    error("Text dump file " + txtDumpFile + " did not exist");
@@ -131,5 +171,5 @@
 		else {
 		    String siteID = siteDir.getName();
-		    NutchTextDumpProcessor nutchTxtDump = NutchTextDumpProcessor(siteID, txtDumpFile);
+		    NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);
 		    
 		}
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33577)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33578)
@@ -5,13 +5,17 @@
 import java.util.Map;
 
+import org.apache.log4j.Logger;
+
+
 public class TextDumpPage {
-
+    private static Logger logger = Logger.getLogger(org.greenstone.atea.TextDumpPage.class.getName());
+    
     public static final String TEXT_START_MARKER="text:start:";
     public static final String TEXT_END_MARKER="text:end:";
     
-    private Map tuples;
+    private Map<String, String> tuples;
     
     public TextDumpPage(String siteID, String unparsedPageDump) {
-	tuples = new HashMap();
+	tuples = new HashMap<String, String>();
 	
 	try (
@@ -25,8 +29,8 @@
 	    String pageURL = line.substring(endIndex);
 	    
-	    tuples.add("pageURL", pageURL.trim());
+	    tuples.put("pageURL", pageURL.trim());
 
 	    String key = line.substring(endIndex);
-	    tuples.add("key", key.trim());
+	    tuples.put("key", key.trim());
 
 	    boolean readingPageText = false;
@@ -42,9 +46,9 @@
 			String k = line.substring(0, endIndex);
 			String v = line.substring(endIndex+1);
-			tuples.add(k.trim(), v.trim());
+			tuples.put(k.trim(), v.trim());
 		    }
 		    
 		    else if(line.equals(TEXT_START_MARKER)) { // dealing with the page body text
-			StringBuilder pageText = new StringBuilder();
+			pageText = new StringBuilder();
 			readingPageText = true;
 		    }				    
@@ -56,5 +60,5 @@
 			// finished with a page body		    
 			// remove any FINAL artificial newline we introduced
-			tuples.add("pageText", pageText.toString().trim());
+			tuples.put("pageText", pageText.toString().trim());
 			readingPageText = false;
 			pageText = null;
@@ -71,5 +75,5 @@
 	    // If the page had no pageText, add a "pageText" -> "" mapping
 	    if(!tuples.containsKey("pageText")) {
-		tuples.add("pageText", "");
+		tuples.put("pageText", "");
 	    }
 	    
@@ -91,3 +95,27 @@
 	return tuples.get(key);
     }
+
+    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
+    public static void info(String msg) {
+	System.err.println(msg);
+	logger.info(msg);
+    }
+    public static void debug(String msg) {
+	System.err.println(msg);
+	logger.debug(msg);
+    }
+    public static void warn(String msg) {
+	System.err.println(msg);
+	logger.warn(msg);
+    }
+    public static void error(String msg) {
+	System.err.println(msg);
+	logger.error(msg);
+    }
+    public static void error(String msg, Exception e) {
+	logger.error(msg, e);
+	System.err.println("\n"+msg);
+	e.printStackTrace();
+    }
+    
 }
