Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33633)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33634)
@@ -4,6 +4,10 @@
 import com.mongodb.client.MongoCollection; 
 import com.mongodb.client.MongoDatabase; 
+//import com.mongodb.client.MongoIterable;
+import com.mongodb.BasicDBObject;
 import com.mongodb.MongoClient; 
-import com.mongodb.MongoCredential;  
+import com.mongodb.MongoCredential;
+import com.mongodb.ServerAddress;
+import com.mongodb.MongoClientOptions;
 
 import org.bson.Document;
@@ -12,5 +16,8 @@
 import java.io.File;
 import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Properties;
+
 
 import org.apache.log4j.Logger;
@@ -43,18 +50,19 @@
  *
  */
-public class MongoDBAccess {
+public class MongoDBAccess implements AutoCloseable {
 
     private static Logger logger = Logger.getLogger(org.greenstone.atea.MongoDBAccess.class.getName());
     
     static final String PROPS_FILENAME = "config.properties";
-    public static final String DB_NAME = "anupama"; //"ateacrawldata";
     public static final String WEBPAGES_COLLECTION = "webpages";
     public static final String WEBSITES_COLLECTION = "websites";    
-    
+
+    // configuration details, some with fallback values
     private String HOST = "localhost";
     private int PORT = 27017; // mongodb port
     private String USERNAME;
     private String PASSWORD;    
-
+    private String DB_NAME ="ateacrawldata";
+    
     private MongoClient mongo = null;
     private MongoDatabase database = null;
@@ -108,4 +116,5 @@
      */
     public void connectToDB() throws Exception {
+	
 	// Creating a Mongo client 
 	mongo = new MongoClient( HOST, PORT ); 
@@ -117,34 +126,58 @@
 	
 	// Accessing the database 
-	database = mongo.getDatabase(DB_NAME); 
+	this.database = mongo.getDatabase(DB_NAME); 
 	logger.info("Credentials: "+ credential);
-    }
-
-    
-    public void insertWebSiteInfo(int SITE_COUNTER, int siteID, String domainOfSite,
-		   int numPages, int numPagesInMRI, int numPagesContainingMRI,
-				  /* TODO: String geoLocationCountryCode, boolean miURL */
-		   String siteCrawledTimestamp, String siteCrawlUnfinished, boolean redoCrawl)
+
+	/*
+	MongoCredential credential; 
+	credential = MongoCredential.createCredential(USERNAME, DB_NAME, PASSWORD.toCharArray());
+	logger.info("Credentials: "+ credential);
+	
+	// Create our Mongo client
+	mongo = new MongoClient( new ServerAddress(HOST, PORT), credential, new MongoClientOptions.Builder().build());
+	System.out.println("Connected to the database successfully");
+
+	this.database = mongo.getDatabase(DB_NAME); 
+	*/
+	
+    }
+
+    // TODO: which fields should be indexed?
+
+    public void showCollections() {
+	//MongoIterable<String> colls = this.database.listCollectionNames();
+	for(String coll : this.database.listCollectionNames()) {
+	    System.err.println("coll: " + coll);
+	}
+    }
+    
+    
+    public void insertWebsiteInfo(WebsiteInfo website)
     {
 	MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
-	Document document = new Document("id", SITE_COUNTER)
-	    .append("siteFolderName", siteID) 
-	    .append("domain", domainOfSite) 
-	    .append("totalPages", numPages) 
-	    .append("numPagesInMRI", numPagesInMRI)
-	    .append("numPagesContainingMRI", numPagesContainingMRI)
-	    .append("siteCrawledTimestamp", siteCrawledTimestamp)
-	    .append("siteCrawlUnfinished", siteCrawlUnfinished)
-	    .append("redoCrawl", redoCrawl);
+	Document document = new Document("_id", website.id)
+	    .append("siteFolderName", website.siteFolderName) 
+	    .append("domain", website.domain) 
+	    .append("totalPages", website.totalPages)
+	    .append("numPagesWithBodyText", website.countOfWebPagesWithBodyText)
+	    .append("numPagesInMRI", website.numPagesInMRI)
+	    .append("siteCrawledTimestamp", website.siteCrawledTimestamp)
+	    .append("siteCrawlUnfinished", website.siteCrawlUnfinished)
+	    .append("redoCrawl", website.redoCrawl);
+
+	document.put("urlContainsLangCodeInpath", website.urlContainsLangCodeInpath);
+	if(website.geoLocationCountryCode != null && !website.geoLocationCountryCode.equals("")) {
+	    document.put("countryCode", website.geoLocationCountryCode);
+	}
+
 	collection.insertOne(document); 
-	System.out.println("website info inserted successfully into " + WEBSITES_COLLECTION);  
-    }
-
-    
-    public void insertWebPage(int WEBPAGE_COUNTER, int site_id, /* id of websites_collection*/
-			      String url, String charEncoding, String modTime, String fetchTime,
-			      boolean isMRI, int totalSentences, int numSentencesInMRI,
-			      ArrayList<SentenceInfo> singleSentences,
-			      ArrayList<SentenceInfo> overlappingSentences)
+	logger.debug("Website info for " + website.id + "(" + website.siteFolderName + ")"
+			   + " inserted successfully into " + WEBSITES_COLLECTION);  
+    }
+
+    /* TODO:
+    https://stackoverflow.com/questions/39433775/mongodb-java-inserting-throws-org-bson-codecs-configuration-codecconfigurationex
+    */
+    public void insertWebpageInfo(WebpageInfo webpage)
     {
 	// load the webpages db 'table'
@@ -152,20 +185,44 @@
 	MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);
 
-	Document document = new Document("id", WEBPAGE_COUNTER)
-	    .append("siteid", site_id) 
-	    .append("url", url) 
-	    .append("charEncoding", charEncoding) 
-	    .append("modTime", modTime)
-	    .append("fetchTime", fetchTime)
-	    .append("isMRI", isMRI)
-	    .append("totalSentences", totalSentences)
-	    .append("numSentencesInMRI", numSentencesInMRI);
-
-	document.put("singleSentences", singleSentences);
-	document.put("overlappingSentences", overlappingSentences);
+	Document document = new Document("_id", webpage.webpageID)
+	    .append("siteid", webpage.websiteID)
+	    .append("url", webpage.URL)
+	    .append("isMRI", webpage.isMRI)
+	    .append("totalSentences", webpage.totalSentences)
+	    .append("charEncoding", webpage.charEncoding) 
+	    .append("modTime", webpage.modifiedTime)
+	    .append("fetchTime", webpage.fetchTime);
+
+	// DOESN'T WORK, AS EXPECTED, BUT DIDN'T KNOW HOW TO DO IT:
+	//document.put("singleSentences", webpage.singleSentences);
+	//document.put("overlappingSentences", webpage.overlappingSentences);
+
+	// INSTEAD, ARRAY OF OBJECTS TO BE INSERTED AS PER:
+	// https://stackoverflow.com/questions/15371839/how-to-add-an-array-to-a-mongodb-document-using-java
+	List<BasicDBObject> sentencesList = new ArrayList<>();
+	for(SentenceInfo sentence : webpage.singleSentences) {
+	    sentencesList.add(new BasicDBObject("langCode", sentence.langCode));
+	    sentencesList.add(new BasicDBObject("confidence", sentence.confidenceLevel));
+	    sentencesList.add(new BasicDBObject("sentence", sentence));
+	}	
+	document.put("singleSentences", sentencesList);
+	
+	List<BasicDBObject> overlappingSentencesList = new ArrayList<>();
+	for(SentenceInfo sentence : webpage.overlappingSentences) {
+	    sentencesList.add(new BasicDBObject("langCode", sentence.langCode));
+	    sentencesList.add(new BasicDBObject("confidence", sentence.confidenceLevel));
+	    sentencesList.add(new BasicDBObject("sentence", sentence));
+	}	
+	document.put("singleSentences", overlappingSentencesList);
+
+	// also put the full text in there
+	document.put("text", webpage.text);
 	
 	collection.insertOne(document);
-	System.out.println("website info inserted successfully into " + WEBPAGES_COLLECTION);  
-    }
+	logger.debug("\nwebpage info for " + webpage.webpageID + " inserted successfully into " + WEBPAGES_COLLECTION);  
+    }
+
+    /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */
+    public void close() {}
     
 
@@ -184,6 +241,7 @@
 	    MongoDBAccess mongodbCon = new MongoDBAccess();
 	    mongodbCon.connectToDB();
-	    //mongodbCon.insertDocument();
-	}catch(Exception e) {
+	    mongodbCon.showCollections();
+
+	} catch(Exception e) {
 	    e.printStackTrace();
 	}
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToCSV.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToCSV.java	(revision 33633)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToCSV.java	(revision 33634)
@@ -244,5 +244,5 @@
 	    
 	    if(text.equals("")) {
-		page.addMRILanguageStatus(false);
+		//page.addMRILanguageStatus(false);
 		continue;
 	    }
@@ -250,5 +250,5 @@
 		boolean isMRI = maoriTxtDetector.isTextInMaori(text);
 		
-		page.addMRILanguageStatus(isMRI);
+		//page.addMRILanguageStatus(isMRI);
 		
     
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java	(revision 33634)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java	(revision 33634)
@@ -0,0 +1,385 @@
+package org.greenstone.atea;
+
+import java.io.*;
+import java.lang.ArrayIndexOutOfBoundsException;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.commons.csv.*;
+import org.apache.log4j.Logger;
+
+
+/**
+ * Class to process the dump text files produced FOR EACH SITE (e.g. site "00001") that
+ * Nutch has finished crawling and whose text has been dumped out to a file called dump.txt.
+ * This reads in the dump.txt file contained in each site folder within the input folder.
+ * (e.g. input folder "crawled" could contain folders 00001 to 01465. Each contains a dump.txt)
+ * Each dump.txt could contain the text contents for an entire site, or for individual pages.
+ * This class then uses class TextDumpPage to parse each webpage within a dump.txt,
+ * which parses out the actual text body content of each webpage's section within a dump.txt.
+ * Finally, MaoriTextDetector is run over that to determine whether the full body text is
+ * likely to be in Maori or not.
+ *
+ * Potential issues: since a web page's text is dumped out by nutch with neither paragraph
+ * nor even newline separator, it's hard to be sure that the entire page is in language.
+ * If it's in multiple languages, there's no way to be sure there aren't promising Maori language
+ * paragraphs contained in a page, if the majority/the remainder happen to be in English.
+ * 
+ * So if we're looking for any paragraphs in Maori to store in a DB, perhaps it's better to run
+ * the MaoriTextDetector.isTextInMaori(BufferedReader reader) over two "lines" at a time,
+ * instead of running it over the entire html body's text.
+ *
+ * TO COMPILE OR RUN, FIRST DO:
+ *    cd maori-lang-detection/apache-opennlp-1.9.1
+ *    export OPENNLP_HOME=`pwd`
+ *    cd maori-lang-detection/src
+ *
+ * TO COMPILE:
+ *    maori-lang-detection/src$
+ *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpToMongoDB.java
+ *
+ * TO RUN:
+ *    maori-lang-detection/src$
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpToMongoDB ../crawled-small
+ *
+ * or:
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpToMongoDB ../crawled-small > ../crawled-small/bla.txt 2>&1
+ *
+*/
+public class NutchTextDumpToMongoDB {
+    static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpToMongoDB.class.getName());
+
+    static boolean DEBUG_MODE = true;
+
+    /** Counter for number of sites.
+     * Should be equal to number of times NutchTextDumpToMongoDB constructor
+     * is called: once per site.
+     */
+    static private int SITE_COUNTER = 0;
+    static private long WEBPAGE_COUNTER = 0;
+    
+    private final MaoriTextDetector maoriTxtDetector;
+    private final MongoDBAccess mongodbAccess;
+    
+    public final String siteID;
+    public final boolean siteCrawlUnfinished;
+    public final long siteCrawledTimestamp; /** When the crawl of the site terminated */
+    
+    private int countOfWebPagesWithBodyText = 0;
+    
+    private String geoLocationCountryCode = null; /** 2 letter country code */
+    private boolean urlContainsLangCodeInPath = false; /** If any URL on this site contains a /mi(/) in its URL */
+    
+    private String domainOfSite;
+    private int numPagesInMRI = 0;
+    
+    /** keep a list to store the text of each page */
+    private ArrayList<TextDumpPage> pages;
+    
+    private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
+	// The start of a new web page's record in nutch's text dump of an entire site
+	// is denoted by a newline followed by a URL (protocol)
+	// or the very start of the file with a URL (protocol)
+	return ((prevLine == null || prevLine.equals(""))
+		&& (line.startsWith("http://") || line.startsWith("https://")));
+    }
+
+    public void debugPageDump(StringBuilder pageDump) {
+	if(DEBUG_MODE) {
+	    // START DEBUG
+	    logger.debug("__________________________________________");
+	    logger.debug("@@@ Found page entry: ");
+	    logger.debug("__________________________________________");
+	    logger.debug(pageDump.toString());
+	    logger.debug("------------------------------------------");
+	    // END DEBUG
+	}
+    }
+
+    /** A NutchTextDumpToMongoDB processes the dump.txt for one site */
+    public NutchTextDumpToMongoDB(MongoDBAccess mongodbAccess,
+				  MaoriTextDetector maoriTxtDetector, String siteID,
+				  File txtDumpFile, long lastModified, boolean siteCrawlUnfinished)
+	throws IOException
+    {	
+	// increment static counter of sites processed by a NutchTextDumpToMongoDB instance
+	SITE_COUNTER++;
+	
+	// siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
+	this.siteID = siteID;
+	this.siteCrawlUnfinished = siteCrawlUnfinished;
+	this.siteCrawledTimestamp = lastModified;
+	
+	this.maoriTxtDetector = maoriTxtDetector;
+	this.mongodbAccess = mongodbAccess;
+	
+	pages = new ArrayList<TextDumpPage>();
+
+	String line = null;
+	StringBuilder pageDump = null;
+	try (
+	     BufferedReader reader = new BufferedReader(new FileReader(txtDumpFile));
+	     ) {
+
+	    boolean readingText = false;
+	    String prevLine = null;
+	    
+	    while((line = reader.readLine()) != null) { // readLine removes newline separator
+		line = line.trim();
+		// iff outside of a page's body text, then an empty line marks the end of a page
+		// in nutch's text dump of a site.
+		// But note, there can be an empty line (or more?) between the start and end
+		// markers of a page's text, though.
+
+		if(isStartOfNewWebPageRecord(prevLine, line)) {
+
+		    if(pageDump != null) { // should also be the case then: if(prevLine != null)
+			// finish old pageDump and begin new one			
+
+			//debugPageDump(pageDump);
+			    
+			TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
+			// parses the fields and body text of a webpage in nutch's txt dump of entire site
+			//page.parseFields();
+			//page.getText();
+			pages.add(page);
+			pageDump = null;
+
+		    }		    
+
+		    // begin new webpage dump
+		    pageDump = new StringBuilder();
+		    pageDump.append(line);
+		    pageDump.append("\n");
+		    
+		}
+		else if(!line.equals("")) {
+		    pageDump.append(line);
+		    pageDump.append("\n");
+
+		}
+		// can throw away any newlines between text start and end markers.
+		
+		prevLine = line;		
+	    }
+
+	    // process final webpage record:
+	    //debugPageDump(pageDump);
+	    
+	    TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
+	    pages.add(page);
+	    pageDump = null;
+
+	    // for every site, we just need to work out if any of its pages
+	    // contains /mi(/) in its URL
+	    String url = page.getPageURL();
+	    if(!this.urlContainsLangCodeInPath && (url.contains("/mi/") || url.endsWith("/mi"))) {
+		this.urlContainsLangCodeInPath = true;
+	    }
+		
+	} catch (IOException ioe) {
+	    logger.error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
+	}
+
+	// Just do this once: get and store domain of site.
+	// Passing true to get domain with protocol prefix
+	if(pages.size() > 0) {
+	    TextDumpPage firstPage = pages.get(0);
+	    String url = firstPage.getPageURL();
+	    this.domainOfSite = Utility.getDomainForURL(url, true);
+	}
+	else {
+	    this.domainOfSite = "UNKNOWN";
+	}
+	
+	
+	prepareSiteStats(mongodbAccess);
+    }
+
+    
+    private void prepareSiteStats(MongoDBAccess mongodbAccess) throws IOException {
+    
+	TextDumpPage page = null;
+	for(int i = 0; i < pages.size(); i++) {
+	    
+	    page = pages.get(i);
+	    
+	    String text = page.getPageText();
+	    
+	    if(text.equals("")) {
+		// don't care about empty pages
+		continue;
+	    }
+	    else {
+		WEBPAGE_COUNTER++; // count of cumulative total of webpages for all sites
+		countOfWebPagesWithBodyText++; // of this site alone				
+		
+		boolean isMRI = maoriTxtDetector.isTextInMaori(text);
+		if(isMRI) {
+		    numPagesInMRI++;
+		}
+		
+		String[] sentences = maoriTxtDetector.getAllSentences(text);
+		int totalSentences = sentences.length;
+		ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences);
+		ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences);
+		
+		WebpageInfo webpage = page.convertStoredDataToWebpageInfo(WEBPAGE_COUNTER,
+								     SITE_COUNTER,
+								     isMRI,
+								     totalSentences,
+								     singleSentences,
+								     overlappingSentences);
+
+		
+		mongodbAccess.insertWebpageInfo(webpage);
+	    }
+	}
+    }
+
+    /*
+    public void printSiteStats() {
+	
+	
+	logger.info("------------- " + this.siteID + " SITE STATS -----------");
+
+	logger.info("SITE DOMAIN: " + this.domainOfSite);
+	logger.info("Total number of web pages in site: " + pages.size());
+	logger.info("Of these, the number of pages in MÄori (mri) were: " + this.pagesInMRI.size());
+	
+	if(pagesInMRI.size() > 0) {
+	    logger.info("The following were the pages detected by OpenNLP as being in MÄori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
+	    for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
+		logger.info(mriWebPageInfo.toString());
+	    }
+	}
+
+	logger.info("                      -----------                   ");
+	if(pagesContainingMRI.size() > 0) {	    
+	    logger.info("The following pages weren't detected as primarily being in MÄori");
+	    logger.info("But still contained sentences detected as MÄori");
+	    for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
+		logger.info(mriWebPageInfo.toString());
+	    }
+	    
+	} else {
+	    logger.info("No further pages detected as containing any sentences in MRI");	   
+	}
+	logger.info("                      -----------                   ");
+    }
+    */
+
+    
+
+    public void websiteDataToDB() {
+	
+	
+	// https://stackoverflow.com/questions/35183146/how-can-i-create-a-java-8-localdate-from-a-long-epoch-time-in-milliseconds
+	// LocalDateTime date =
+	//     LocalDateTime.ofInstant(Instant.ofEpochMilli(this.siteCrawledTimestamp), ZoneId.systemDefault());
+	// String crawlTimestamp =
+	//     date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + " " + date.format(DateTimeFormatter.ofPattern("HH:mm:ss"));
+
+	boolean redoCrawl = false;
+
+	if(this.siteCrawlUnfinished) {
+	    // arbitrary decision, but need some indication that the MRI content was not close to one-off in the website
+	    if(this.numPagesInMRI > 2) { 
+		redoCrawl = true;
+	    }
+	}
+	
+	//File geoLiteCityDatFile = new File(this.getClass().getResource("GeoLiteCity.dat").getFile());
+	//this.geoLocationCountryCode = getCountryCodeOfDomain(this.domainOfSite, geoLiteCityDatFile);
+
+	int totalPages = pages.size();	
+
+	WebsiteInfo website = new WebsiteInfo(SITE_COUNTER, this.siteID, this.domainOfSite,
+	      totalPages, this.countOfWebPagesWithBodyText, this.numPagesInMRI, 
+	      this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl,
+	      this.geoLocationCountryCode, this.urlContainsLangCodeInPath);
+
+	mongodbAccess.insertWebsiteInfo(website);
+	
+    }
+
+    
+    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
+   
+    public static void printUsage() {
+	System.err.println("Run this program as:");
+	System.err.println("\tNutchTextDumpToMongoDB <path to 'crawled' folder>");
+    }
+    
+    public static void main(String[] args) {
+	if(args.length != 1) {
+	    printUsage();
+	    return;
+	}
+
+	File sitesDir = new File(args[0]);
+	if(!sitesDir.exists() || !sitesDir.isDirectory()) {
+	    logger.error("Error: " + args[0] + " does not exist or is not a directory");
+	    return;
+	}
+
+	NutchTextDumpToMongoDB.DEBUG_MODE = false;
+
+	
+	try (
+	     MongoDBAccess mongodb = new MongoDBAccess();
+	     ) {
+	    
+	    mongodb.connectToDB();
+	    //mongodb.showCollections();
+	    
+	    // print out the column headers for the websites csv file
+	    // https://commons.apache.org/proper/commons-csv/apidocs/org/apache/commons/csv/CSVPrinter.html
+	    // OPTIONAL TODO: creating collections can be done here if dropping and recreating
+	    
+	    MaoriTextDetector mriTxtDetector = new MaoriTextDetector(true); // true: run silent
+	    File[] sites = sitesDir.listFiles();
+
+	    // sort site folders in alphabetical order
+	    // https://stackoverflow.com/questions/7199911/how-to-file-listfiles-in-alphabetical-order
+	    Arrays.sort(sites);
+	    
+	    for(File siteDir : sites) { // e.g. 00001
+		if(siteDir.isDirectory()) {
+		    // look for dump.txt
+		    File txtDumpFile = new File(siteDir, "dump.txt");
+		    if(!txtDumpFile.exists()) {
+			logger.error("Text dump file " + txtDumpFile + " did not exist");
+			continue;
+		    }
+		    
+		    else {
+			File UNFINISHED_FILE = new File(siteDir, "UNFINISHED");			
+			
+			String siteID = siteDir.getName();
+			long lastModified = siteDir.lastModified();
+			logger.debug("Found siteID: " + siteID);		    
+			NutchTextDumpToMongoDB nutchTxtDump = new NutchTextDumpToMongoDB(
+				 mongodb, mriTxtDetector,
+				 siteID, txtDumpFile, lastModified, UNFINISHED_FILE.exists());
+			// now it's parsed all the web pages in the site's text dump
+
+			// Let's print stats on each web page's detected language being MRI or not
+			// and how many pages there were in the site in total.
+			
+			//nutchTxtDump.printSiteStats();
+			
+			nutchTxtDump.websiteDataToDB();
+		    }
+		}
+		
+	    }
+	    
+	} catch(Exception e) {
+	    // can get an exception when instantiating NutchTextDumpToMongoDB instance
+	    // or with CSV file
+	    logger.error(e.getMessage(), e);
+	}
+    }
+}
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/SentenceInfo.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/SentenceInfo.java	(revision 33634)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/SentenceInfo.java	(revision 33634)
@@ -0,0 +1,14 @@
+package org.greenstone.atea;
+
+public class SentenceInfo {
+    public final double confidenceLevel;
+    /** 3 letter lang code */
+    public final String langCode;
+    public final String sentence;
+    
+    public SentenceInfo(double confidence, String langCode, String sentence) {
+	this.confidenceLevel = confidence;
+	this.langCode = langCode;
+	this.sentence = sentence;
+    }
+}
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33633)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33634)
@@ -2,4 +2,5 @@
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
@@ -15,4 +16,8 @@
     
     private Map<String, String> tuples;
+
+    private boolean isMRI = false;
+
+    boolean DEBUG_MODE = false;
     
     public TextDumpPage(String siteID, String unparsedPageDump) {
@@ -90,5 +95,5 @@
 			    tuples.put(k.trim(), v.trim());
 			} else {
-			    if(NutchTextDumpProcessor.DEBUG_MODE) {
+			    if(DEBUG_MODE) {
 				logger.error("No meta key for meta: " + line);
 				logger.error(unparsedPageDump);
@@ -118,5 +123,5 @@
 
     public void debugTuples() {
-	if(NutchTextDumpProcessor.DEBUG_MODE) {
+	if(DEBUG_MODE) {
 	    logger.debug("__________________________________________");
 	    for(Map.Entry<String, String> entry : tuples.entrySet()) {
@@ -168,4 +173,32 @@
     }
 
+    /**
+     * IMPORTANT: This method deletes the data stored in this TextDumpPage object
+     * after converting relevant fields and parameters to a WebpageInfo object
+     */
+    public WebpageInfo convertStoredDataToWebpageInfo(
+      long webpageID, int websiteID, boolean isMRI, int totalSentences,
+      ArrayList<SentenceInfo> singleSentences, ArrayList<SentenceInfo> overlappingSentences)
+    {
+	// clear the map, after storing the important (meta)data
+	String pageText = getPageText();
+	String pageURL = getPageURL();
+	String charEncoding = getOriginalCharEncoding();
+	String modifiedTime = getModifiedTime();
+	String fetchTime = getFetchTime();
+
+	WebpageInfo webpage = new WebpageInfo(webpageID, websiteID,
+					      pageText, pageURL, isMRI, totalSentences,
+					      charEncoding, modifiedTime, fetchTime,
+					      singleSentences, overlappingSentences);
+	
+	tuples.clear();
+
+	return webpage;
+    }
+    
+
+
+    /*
     public void addMRILanguageStatus(boolean status) {
 	if(status) {
@@ -189,4 +222,4 @@
 
     }
-    
+    */
 }
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java	(revision 33633)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java	(revision 33634)
@@ -142,18 +142,4 @@
     }
 
-    /** inner class */
-    public class SentenceInfo {
-	public final double confidenceLevel;
-	/** 3 letter lang code */
-	public final String langCode;
-	public final String sentence;
-
-	public SentenceInfo(double confidence, String langCode, String sentence) {
-	    confidenceLevel = confidence;
-	    this.langCode = langCode;
-	    this.sentence = sentence;
-	}
-    }
-
     /** TODO: Is it sensible to use the Maori Language Sentence Model to split the text
      * into sentences? What if the text in any other language or a mix of languages?
@@ -183,5 +169,5 @@
 	    double confidence = bestLanguage.getConfidence();
 
-	    sentencesList.add(new SentenceInfo(confidence, bestLanguage, sentence));
+	    sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), sentence));
 	}
 
@@ -207,5 +193,5 @@
 		separator = " ";
 	    }
-	    sentence = sentence + separator + sentence[i]; 
+	    sentence = sentence + separator + sentences[i]; 
 	    
 	    //System.err.println(sentence);
@@ -214,5 +200,5 @@
 	    double confidence = bestLanguage.getConfidence();
 
-	    sentencesList.add(new SentenceInfo(confidence, bestLanguage, sentence));
+	    sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), sentence));
 	}
 
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WebpageInfo.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WebpageInfo.java	(revision 33634)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WebpageInfo.java	(revision 33634)
@@ -0,0 +1,47 @@
+package org.greenstone.atea;
+
+import java.util.ArrayList;
+
+public class WebpageInfo {
+
+    /** db table ids */
+    public final long webpageID;
+    public final int websiteID;
+
+    public final int totalSentences;
+    
+    public final String text;
+    public final String URL;
+    public final boolean isMRI;
+    
+    public final String charEncoding;
+    public final String modifiedTime;
+    public final String fetchTime;
+    public final ArrayList<SentenceInfo> singleSentences;
+    public final ArrayList<SentenceInfo> overlappingSentences;
+	
+    public WebpageInfo (long webpageID, int websiteID,
+			String pageText, String pageURL, boolean isMRI, int totalSentences,
+			String charEncoding, String modifiedTime, String fetchTime,
+			ArrayList<SentenceInfo> singleSentences,
+			ArrayList<SentenceInfo> overlappingSentences)
+    {
+
+	this.webpageID = webpageID;
+	this.websiteID = websiteID;
+
+	this.totalSentences = totalSentences;
+	
+	this.text = pageText;
+	this.URL = pageURL;
+	this.isMRI = isMRI;
+	
+	this.charEncoding = charEncoding;
+	this.modifiedTime = modifiedTime;
+	this.fetchTime = fetchTime;
+
+	this.singleSentences = singleSentences;
+	this.overlappingSentences = overlappingSentences;
+	
+    }
+}
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WebsiteInfo.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WebsiteInfo.java	(revision 33634)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WebsiteInfo.java	(revision 33634)
@@ -0,0 +1,40 @@
+package org.greenstone.atea;
+
+public class WebsiteInfo {
+
+    public final int id;
+    public final String siteFolderName;
+    public final String domain;
+    
+    public final int totalPages;
+    public final int countOfWebPagesWithBodyText;
+    public final int numPagesInMRI;
+    
+    public final long siteCrawledTimestamp;
+    public final boolean siteCrawlUnfinished;
+    public final boolean redoCrawl;
+    
+    public final String geoLocationCountryCode;
+    public final boolean urlContainsLangCodeInpath;
+	
+    public WebsiteInfo(int siteCount, String siteFolderName, String domainOfSite,
+		       int totalPages, int countOfWebPagesWithBodyText, int numPagesInMRI,
+		       long siteCrawledTimestamp, boolean siteCrawlUnfinished, boolean redoCrawl,
+		       String geoLocationCountryCode, boolean urlContainsLangCodeInpath)
+    {
+	this.id = siteCount;
+	this.siteFolderName = siteFolderName;
+	this.domain = domainOfSite;
+	
+	this.totalPages = totalPages;
+	this.countOfWebPagesWithBodyText = countOfWebPagesWithBodyText;
+	this.numPagesInMRI = numPagesInMRI;
+	
+	this.siteCrawledTimestamp = siteCrawledTimestamp;
+	this.siteCrawlUnfinished = siteCrawlUnfinished;
+	this.redoCrawl = redoCrawl;
+	
+	this.geoLocationCountryCode = geoLocationCountryCode;
+	this.urlContainsLangCodeInpath = urlContainsLangCodeInpath;
+    }
+}
