Index: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33581)
+++ gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33582)
@@ -240,5 +240,5 @@
      * This retains any www. or subdomain prefix.
      */
-    private String getDomainForURL(String url, boolean withProtocol) {
+    public static String getDomainForURL(String url, boolean withProtocol) {
 	int startIndex = startIndex = url.indexOf("//"); // for http:// or https:// prefix
 	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
Index: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MRIWebPageStats.java
===================================================================
--- gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MRIWebPageStats.java	(revision 33582)
+++ gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MRIWebPageStats.java	(revision 33582)
@@ -0,0 +1,19 @@
+package org.greenstone.atea;
+
+
+//import org.apache.log4j.Logger;
+
+
+public class MRIWebPageStats {
+    //private static Logger logger = Logger.getLogger(org.greenstone.atea.MRIWebPageStats.class.getName());
+
+    public final String siteID; // crawled site's folder name e.g. 00510
+    public final String URL; // URL of webpage
+    public final int pageID; // index into NutchTextDumpProcessor::pages ArrayList
+    
+    public MRIWebPageStats(String siteID, String url, int pageID) {
+	this.siteID = siteID;
+	this.URL = url;
+	this.pageID = pageID;
+    }
+}
Index: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java
===================================================================
--- gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33581)
+++ gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33582)
@@ -42,14 +42,22 @@
  *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled
  *
+ * or:
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled > ../crawled/bla.txt 2>&1
+ *
 */
 public class NutchTextDumpProcessor {
     private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
 
+    static boolean DEBUG_MODE = true;
+    
     private final MaoriTextDetector maoriTxtDetector;
     
     public final String siteID; // is this necessary?
-
+    private String domainOfSite;
+    
     /** keep a list to store the text of each page */
     private ArrayList<TextDumpPage> pages;
+
+    private ArrayList<MRIWebPageStats> pagesInMRI;
 
     private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
@@ -62,12 +70,13 @@
 
     public void debugPageDump(StringBuilder pageDump) {
-			
-	// START DEBUG
-	debug("__________________________________________");
-	debug("@@@ Found page entry: ");
-	debug("__________________________________________");
-	debug(pageDump.toString());
-	debug("------------------------------------------");
-	// END DEBUG
+	if(DEBUG_MODE) {
+	    // START DEBUG
+	    debug("__________________________________________");
+	    debug("@@@ Found page entry: ");
+	    debug("__________________________________________");
+	    debug(pageDump.toString());
+	    debug("------------------------------------------");
+	    // END DEBUG
+	}
     }
     
@@ -137,5 +146,17 @@
 	    error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
 	}
-	
+
+	// Just do this once: get domain of site.
+	// Passing true to get domain with protocol prefix
+	if(pages.size() > 0) {
+	    TextDumpPage firstPage = pages.get(0);	    
+	    String url = firstPage.getPageURL();	    
+	    this.domainOfSite = CCWETProcessor.getDomainForURL(url, true);	    
+	}
+	else {
+	    this.domainOfSite = "UNKNOWN";
+	}
+	
+	prepareSiteStats();
     }
     
@@ -144,4 +165,5 @@
 
 	String text = getTextForPage(pageID);
+	
 	// QTODO: what to do when page body text is empty?
 	if(text.equals("")) return false;
@@ -165,6 +187,63 @@
 	TextDumpPage page = getPage(pageID);
 	return page.getPageURL();
-    }
-    
+    }   
+       
+    public int totalNumPages() {
+	return pages.size();
+    } 
+
+    private void prepareSiteStats() {
+	pagesInMRI = new ArrayList<MRIWebPageStats>();
+
+
+	TextDumpPage page = null;
+	for(int i = 0; i < pages.size(); i++) {
+
+	    
+	    page = pages.get(i);
+
+	    /*
+	    // just do this once: get domain. Passing true to get domain with protocol prefix
+	    if(this.domainOfSite == null) {		
+		this.domainOfSite = CCWETProcessor.getDomainForURL(url, true);
+	    }
+	    */
+	    
+	    String text = page.getPageText();
+	    if(text.equals("")) {
+		page.addMRILanguageStatus(false);
+		continue;
+	    }
+	    else {
+		boolean isMRI = maoriTxtDetector.isTextInMaori(text);
+		page.addMRILanguageStatus(isMRI);
+
+		if(isMRI) { // add page to list of meaningful pages.		    
+		    String url = page.getPageURL();
+		    MRIWebPageStats MRIpageStats = new MRIWebPageStats(this.siteID, url, i);
+		    pagesInMRI.add(MRIpageStats);
+		}
+		
+	    }
+	}
+    }
+    
+    public void printSiteStats() {
+	
+	
+	info("------------- " + this.siteID + " SITE STATS -----------");
+
+	info("SITE DOMAIN: " + this.domainOfSite);
+	info("Total number of web pages in site: " + pages.size());
+	info("Of these, the number of pages in MÄori (mri) were: " + this.pagesInMRI.size());
+
+	info("The following were the pages detected by OpenNLP as being in MÄori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
+	for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
+	    info("URL: " + mriWebPageInfo.URL);
+	    info("siteID: " + mriWebPageInfo.siteID);
+	}
+	info("                      -----------                   ");
+    }
+
     
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
@@ -208,6 +287,8 @@
 	}
 
+	NutchTextDumpProcessor.DEBUG_MODE = false;
+	
 	try {
-	    MaoriTextDetector mriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
+	    MaoriTextDetector mriTxtDetector = new MaoriTextDetector(true); // true: run silent
 	    File[] sites = sitesDir.listFiles();
 	    for(File siteDir : sites) { // e.g. 00001
@@ -223,5 +304,11 @@
 			String siteID = siteDir.getName();
 			debug("Found siteID: " + siteID);		    
-			NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);		    
+			NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);
+			// now it's parsed all the web pages in the site's text dump
+
+			// Let's print stats on each web page's detected language being MRI or not
+			// and how many pages there were in the site in total.
+			
+			nutchTxtDump.printSiteStats();
 		    }
 		}
Index: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java
===================================================================
--- gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33581)
+++ gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33582)
@@ -77,4 +77,7 @@
 		    }
 		    else { // dealing with the rest of the page dump's metadata
+			// QTODO: nutch's text dump output is problematic
+			// strange characters are in the stream and end up here
+			// and can't detect end of metadata or even end of line.
 			endIndex = line.indexOf(":");
 			if(endIndex != -1) { 
@@ -83,5 +86,8 @@
 			    tuples.put(k.trim(), v.trim());
 			} else {
-			    error("No meta key for meta: " + line);
+			    if(NutchTextDumpProcessor.DEBUG_MODE) {
+				error("No meta key for meta: " + line);
+				error(unparsedPageDump);
+			    }
 			}
 		    }
@@ -102,18 +108,24 @@
 	
 	// START DEBUG
-	debug("__________________________________________");
-	for(Map.Entry<String, String> entry : tuples.entrySet()) {
-	    String key = entry.getKey();
-	    String value = entry.getValue();	    
-	    debug(key + " - " + value);
-	}
-	debug("__________________________________________");
+	debugTuples();
 	// END DEBUG
 	
     }
 
+    public void debugTuples() {
+	if(NutchTextDumpProcessor.DEBUG_MODE) {
+	    debug("__________________________________________");
+	    for(Map.Entry<String, String> entry : tuples.entrySet()) {
+		String key = entry.getKey();
+		String value = entry.getValue();	    
+		debug(key + " - " + value);
+	    }
+	    debug("__________________________________________");
+	}
+    }
+    
     
     public String getPageURL() {
-	return tuples.get("url");
+	return tuples.get("pageURL");
     }
 
@@ -126,4 +138,30 @@
     }
 
+    public void add(String key, String value) {
+	tuples.put(key, value);
+    }
+
+    public void addMRILanguageStatus(boolean status) {
+	if(status) {
+	    tuples.put("isMRI", "true");
+	} else {
+	    tuples.put("isMRI", "false");
+	}
+    }
+
+    public boolean getMRILanguageStatus() {
+	String value = tuples.get("isMRI");
+	if(value == null) {
+	    return false;
+	}
+	if(value.equals("true")) {
+	    return true;
+	}
+	else {
+	    return false;
+	}
+
+    }
+    
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
     public static void info(String msg) {
