Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33910)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33911)
@@ -10,35 +10,7 @@
 import com.mongodb.MongoClientOptions;
 
-import com.mongodb.Block;
-
-import org.bson.BsonArray;
-import org.bson.BsonString;
-import org.bson.BsonValue;
 import org.bson.Document;
-import org.bson.conversions.Bson;
-import org.bson.json.JsonMode;
-import org.bson.json.JsonWriterSettings;
-    
-import com.mongodb.util.JSON;
-//import com.mongodb.DBObject;
-
-
-import com.google.gson.*; // for pretty printing
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.io.Writer;
-
-import java.util.Arrays;
-import java.util.ArrayList;
-import java.util.List;
+
 import java.util.Properties;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
 
 import org.apache.log4j.Logger;
@@ -47,5 +19,4 @@
 import dev.morphia.*;
 
-import org.apache.commons.csv.*;
 
 /** 
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/SummaryTool.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/SummaryTool.java	(revision 33911)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/SummaryTool.java	(revision 33911)
@@ -0,0 +1,392 @@
+package org.greenstone.atea;
+
+import java.util.*;
+import java.io.*;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Runs some of the important mongoDB queries I ran.
+ *
+ * TO COMPILE OR RUN, FIRST DO:
+ *    cd maori-lang-detection/apache-opennlp-1.9.1
+ *    export OPENNLP_HOME=`pwd`
+ *    cd maori-lang-detection/src
+ *
+ * TO COMPILE:
+ *    maori-lang-detection/src$
+ *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/SummaryTool.java
+ *
+ * TO RUN:
+ *    maori-lang-detection/src$
+ *       java -cp ".:../conf:../lib/*" org/greenstone/atea/SummaryTool
+ * OR:
+ *       java -cp ".:../conf:../lib/*" org/greenstone/atea/SummaryTool ../mongodb-data/domainsNZ_IsMRI.txt
+ *
+*/
+public class SummaryTool {
+    static Logger logger = Logger.getLogger(org.greenstone.atea.SummaryTool.class.getName());
+    static private final long FIXED_SEED = 1000;
+    
+    private final MongoDBQueryer mongodbQueryer;
+    private File outFolder;
+
+
+    
+    public static class Tuple {
+	public final String url;
+	public final String countryCode;
+	
+	public Tuple(String url, String countryCode) {
+	    this.url = url;
+	    this.countryCode = countryCode;
+	}
+
+	public String toString() {
+	    return this.url + "," + countryCode;
+	}
+    }
+    
+    
+    public SummaryTool(MongoDBQueryer mongodbQueryer, File outFolder)
+    {
+	this.mongodbQueryer = mongodbQueryer;
+	this.outFolder = outFolder;
+    }
+
+    
+    public void produceURLsForPagesInMRI(File domainsFile) {
+	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
+	File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
+	writeURLsToFile(urlsList, outFile, urlsList.size());
+	
+	System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
+			   + Utility.getFilePath(outFile));
+    }
+    
+    public void produceURLsForPagesContainingMRI(File domainsFile) {
+	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.CONTAINS_MRI, domainsFile);	
+	File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
+	writeURLsToFile(urlsList, outFile, urlsList.size());
+
+	System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
+			   + Utility.getFilePath(outFile));
+    }
+    
+    private ArrayList<Tuple> getURLsForAllWebPagesInSiteListing(int filterType, File domainsFile) {
+	ArrayList<Tuple> urlsList = new ArrayList<Tuple>();
+	
+	// 1. read each url from the domainsFile
+	// 1a. do the query
+	// 1b. add the arraylist result to urls
+	
+	try (
+	     BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
+	     ) {	    
+	    
+	    String domain;
+	    
+	    while((domain = reader.readLine()) != null) {
+		domain = domain.trim();
+		if(!domain.equals("")) {
+		    
+		    String countryCode = "";
+		    int index = domain.lastIndexOf(",");
+		    if(index != -1) {
+			countryCode = domain.substring(index+1).trim();
+			domain = domain.substring(0, index);
+		    }
+		    ArrayList<String> moreURLs = mongodbQueryer.queryAllMatchingURLsFilteredBy(domain, filterType);
+
+		    // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
+		    if(moreURLs.size() == 0 && filterType == MongoDBQueryer.IS_MRI) {
+			System.out.println("   " + countryCode + " domain " + domain + " had no isMRI webpages - only containsMRI.");
+		    }
+
+		    //urlsList.addAll(moreURLs);
+		    for(int i = 0; i < moreURLs.size(); i++) {
+			urlsList.add(new Tuple(moreURLs.get(i), countryCode));
+		    }
+		    
+		}
+	    }
+	    System.err.println("");
+	} catch(Exception e) {
+	    logger.error("Unable to read URLs from file " + Utility.getFilePath(domainsFile));
+	    logger.error(e.getMessage(), e);
+	}
+
+	return urlsList;
+    }
+    
+    /** Given a hand curated list of all sites with positive numPagesContainingMRI
+     * determined by manual inspection, get a listing of all their web pages that
+     * are IN_MRI (or CONTAINS_MRI?). 
+     * Total all these pages that are inMRI (N), then work out the correct sample size (n)
+     * at 90% confidence with 5% margin of error. Then generate a random listing
+     * of n of these pages in MRI of these trusted sites and output to a file
+     * for manual inspection of the sample webpage URLs at page-level. */
+    /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
+     * of all their web pages IN_MRI (or CONTAINS_MRI).
+     * Plus a listing of all the NZ pages IN_MRI. */
+    //public void webPagesOfAllNZSitesAndDomainListing(File domainsFile) {
+    public void mriWebPageListingForDomainListing(File domainsFile) {
+
+	int filterType = MongoDBQueryer.IS_MRI;
+	
+	// for overseas websites, 
+	//produceURLsForPagesContainingMRI(handCuratedOverseasDomainsFile);
+
+	// 0. get a list of all the web pages in the given domain listing where isMRI = true
+	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
+	    // produceURLsForPagesInMRI(domainsFile);
+	
+	// 1. calculate the population size, N, the number of all webpages in the given domain
+	// site listing where isMRI = true.
+	int N_totalNumPages = urlsList.size();
+
+	// 2. write all the URLs in urlsList to a file
+	//File outFolder = domainsFile.getParentFile();
+	String fileName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI_" : "containsMRI_";
+	File outFile = new File(outFolder, fileName+domainsFile.getName());
+
+	writeURLsToFile(urlsList, outFile, N_totalNumPages);
+	System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
+			   + "\ninto file: " + Utility.getFilePath(outFile));
+		
+	// 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
+	int n_numSampleURLs = calcSampleSize(N_totalNumPages); 
+
+	System.err.println("*** N, total number of web pages that matched: " + N_totalNumPages);
+	System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);
+    
+	// 4. Shuffle all the URLs and write the first n (sample size) URLs to a file
+	// Using a constant seed for reproducibility
+	// https://stackoverflow.com/questions/6284589/setting-a-seed-to-shuffle-arraylist-in-java-deterministically
+	Collections.shuffle(urlsList, new Random(FIXED_SEED)); 
+	
+	outFile = new File(outFolder, "random"+n_numSampleURLs+"_"+domainsFile.getName());
+	writeURLsToFile(urlsList, outFile, n_numSampleURLs);
+	System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
+			   + "for the sites in input domainsFile\ninto file: " + Utility.getFilePath(outFile));
+    }
+
+    /**
+     * Calculates sample size n for binary outcomes at 90% confidence and 5% margine of error
+     * for given population size N.
+     * @return n, the sample size.
+     */
+    public int calcSampleSize(int N) {
+	
+	// calculate sample size n for population size N if using 90% confidence and 5% margin of error
+	// https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+	// https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-sample-size/#CI1
+	// https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
+	
+	double m = 0.05; // margin of error = 5%
+	// for 90% confidence, alpha is the remainder = 10% and alpha/2 = 5%.
+	// For 90% confidence, use the table of known z_alpha/2 values from step 1 of
+	// https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
+	double z_alpha_over_2 = 1.6449;
+
+	// Formula: n = (zalpha2 ^ 2 * N) / ((z-alpha-2 ^ 2) + 4(N-1)*m^2)
+	// see https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+	double n = (Math.pow(z_alpha_over_2, 2.0) * N) / (Math.pow(z_alpha_over_2, 2.0) + (4 * (N-1) * Math.pow(m,2.0)));
+
+	// Round up to get a whole number:
+	return (int)Math.ceil(n);
+    }
+    
+    /**
+     * Writes out the first n URLs in urlsList into outFile.
+     */
+    private void writeURLsToFile(ArrayList<Tuple> urlsList, File outFile, final int n) {
+	try (
+	     Writer writer = new BufferedWriter(new FileWriter(outFile));
+	     ) {
+
+	    for (int i=0; i < n; i++) {
+		Tuple urlInfo = urlsList.get(i);
+		
+		//System.out.println(list.get(i));
+		writer.write(urlInfo + "\n"); // calls toString() on tuple of url -> countryCode
+	    }
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + Utility.getFilePath(outFile));
+	    logger.error(e.getMessage(), e);
+	}
+    }
+
+    /* ---------------------------------------- */
+    /** 
+     * Create the file 5counts_containsMRISites_allNZGrouped.json
+     * that contains the count and domains for NZ sites (NZ origin or nz TLD) with pages
+     * that CONTAIN_MRI, followed by counts and domains listing for overseas sites
+     * that CONTAIN_MRI.
+     * @return full path of file generated
+     */
+    public String writeContainsMRISites_nzSitesAndTLDsGrouped() {
+	
+	File outFile = new File(outFolder, "5counts_containsMRISites_allNZGrouped.json");
+
+	String filename = Utility.getFilePath(outFile);
+	
+	try (
+	     Writer writer = new BufferedWriter(new FileWriter(outFile));
+	     ) {
+	    // first write out NZ sites and .nz TLD count and domains
+	    mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
+	    // next write out all overseas sites (not NZ origin or .nz TLD)
+	    // that have no "mi" in the URL path as mi.* or */mi
+	    boolean isMiInURLPath = false;
+	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI);
+	    
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + filename);
+	    logger.error(e.getMessage(), e);
+	}
+	
+	System.err.println("*** Wrote file: " + filename);
+
+	return filename;
+    }
+    
+    /** 
+     * Create the file 5a_counts_tentativeNonAutotranslatedSites.json
+     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
+     * followed by counts and domain listing for overseas sites that are either from Australia
+     * or don't contain mi in their URL path.
+     * @return full path of file generated
+     */
+    public String writeTentativeNonAutotranslatedSites() {
+	
+	File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
+
+	String filename = Utility.getFilePath(outFile);
+	
+	try (
+	     Writer writer = new BufferedWriter(new FileWriter(outFile));
+	     ) {
+	    // first write out NZ sites and .nz TLD count and domains
+	    mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
+	    // next write out all overseas sites (not NZ origin or .nz TLD)
+	    // that have no "mi" in the URL path as mi.* or */mi
+	    boolean isMiInURLPath = false;
+	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
+	    
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + filename);
+	    logger.error(e.getMessage(), e);
+	}
+	
+	System.err.println("*** Wrote file: " + filename);
+
+	return filename;
+    }
+
+    /**
+     * Create the file 5b_counts_overseasSitesWithMiInPath.json
+     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
+     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
+     * This listing is separate to allow easier weeding out of product sites/autotranslated
+     * sites when eyeballing the listing output.
+     */
+    public String writeOverseasSitesWithMiInURLPath() {
+	File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
+
+	String filename = Utility.getFilePath(outFile);
+	try (
+	     Writer writer = new BufferedWriter(new FileWriter(outFile));
+	     ) {
+	    boolean isMiInURLPath = true;
+	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
+
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + filename);
+	    logger.error(e.getMessage(), e);
+	}
+
+	System.err.println("*** Wrote file: " + filename);
+	return filename;
+    }
+	
+    public static void printUsage() {
+	System.err.println("Usage: SummaryTool [domains.txt]");
+    }
+
+    /**
+     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
+     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
+     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
+     * translated and really contain at least one webpage containing at least one sentence in MRI.
+     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
+     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
+     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
+     * 90% confidence with 5% margin of error for testing binary outcomes, see
+     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+     */
+    public static void main(String args[]) {
+	if(args.length >= 2) {
+	    printUsage();
+	    System.exit(-1);
+	}
+
+	try (
+	     MongoDBQueryer mongodb = new MongoDBQueryer();
+	     ) {
+
+	    mongodb.connectToDB();
+
+	    // output files will be stored in mongodb-data-auto
+	    File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
+	    SummaryTool listing = new SummaryTool(mongodb, outFolder);
+
+	    System.out.println("*************************************");
+	    
+	    
+	    if(args.length >= 1) { // web page urls listing mode
+		File domainsFile = new File(args[0]);
+		if(!domainsFile.exists()) {
+		    System.err.println("File " + domainsFile + " does not exist");
+		    System.exit(-1);
+		}
+
+		//String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
+		//String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);
+
+
+		// TODO: for NZ, do IS_MRI. For overseas still CONTAINS_MRI
+		// then also do the shuffle to gen X num of random web page URLs.
+		//String filename = listing.webPagesOfAllNZSitesAndDomainListing(domainsFile);
+		listing.mriWebPageListingForDomainListing(domainsFile);
+
+		// TODO: generate the special table (6)
+		
+	    } else {		
+
+		// calculating sample size works:
+		//System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
+		//System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
+
+		// get all sites where >0 pages have containsMRI=true
+		// grouping NZ sites and .nz TLDs together and remainder under overseas
+		// geolocations.
+		String filename = listing.writeContainsMRISites_nzSitesAndTLDsGrouped();
+
+		// separately:
+		// - all NZ containsMRI + overseas tentative non-product sites with containMRI
+		// - overseas tentative product sites with containMRI
+		filename = listing.writeTentativeNonAutotranslatedSites();
+		filename = listing.writeOverseasSitesWithMiInURLPath();
+
+		// TODO: generate the tables
+
+		mongodb.writeTables(outFolder);
+	    }
+
+	    System.out.println("*************************************");
+	} catch(Exception e) {
+	    logger.error(e.getMessage(), e);
+	}
+    }
+}
Index: her-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33910)
+++ 	(revision )
@@ -1,392 +1,0 @@
-package org.greenstone.atea;
-
-import java.util.*;
-import java.io.*;
-
-import org.apache.log4j.Logger;
-
-/**
- * Runs some of the important mongoDB queries I ran.
- *
- * TO COMPILE OR RUN, FIRST DO:
- *    cd maori-lang-detection/apache-opennlp-1.9.1
- *    export OPENNLP_HOME=`pwd`
- *    cd maori-lang-detection/src
- *
- * TO COMPILE:
- *    maori-lang-detection/src$
- *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing.java
- *
- * TO RUN:
- *    maori-lang-detection/src$
- *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing
- * OR:
- *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt
- *
-*/
-public class WebPageURLsListing {
-    static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
-    static private final long FIXED_SEED = 1000;
-    
-    private final MongoDBQueryer mongodbQueryer;
-    private File outFolder;
-
-
-    
-    public static class Tuple {
-	public final String url;
-	public final String countryCode;
-	
-	public Tuple(String url, String countryCode) {
-	    this.url = url;
-	    this.countryCode = countryCode;
-	}
-
-	public String toString() {
-	    return this.url + "," + countryCode;
-	}
-    }
-    
-    
-    public WebPageURLsListing(MongoDBQueryer mongodbQueryer, File outFolder)
-    {
-	this.mongodbQueryer = mongodbQueryer;
-	this.outFolder = outFolder;
-    }
-
-    
-    public void produceURLsForPagesInMRI(File domainsFile) {
-	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
-	File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
-	writeURLsToFile(urlsList, outFile, urlsList.size());
-	
-	System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
-			   + Utility.getFilePath(outFile));
-    }
-    
-    public void produceURLsForPagesContainingMRI(File domainsFile) {
-	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.CONTAINS_MRI, domainsFile);	
-	File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
-	writeURLsToFile(urlsList, outFile, urlsList.size());
-
-	System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
-			   + Utility.getFilePath(outFile));
-    }
-    
-    private ArrayList<Tuple> getURLsForAllWebPagesInSiteListing(int filterType, File domainsFile) {
-	ArrayList<Tuple> urlsList = new ArrayList<Tuple>();
-	
-	// 1. read each url from the domainsFile
-	// 1a. do the query
-	// 1b. add the arraylist result to urls
-	
-	try (
-	     BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
-	     ) {	    
-	    
-	    String domain;
-	    
-	    while((domain = reader.readLine()) != null) {
-		domain = domain.trim();
-		if(!domain.equals("")) {
-		    
-		    String countryCode = "";
-		    int index = domain.lastIndexOf(",");
-		    if(index != -1) {
-			countryCode = domain.substring(index+1).trim();
-			domain = domain.substring(0, index);
-		    }
-		    ArrayList<String> moreURLs = mongodbQueryer.queryAllMatchingURLsFilteredBy(domain, filterType);
-
-		    // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
-		    if(moreURLs.size() == 0 && filterType == MongoDBQueryer.IS_MRI) {
-			System.out.println("   " + countryCode + " domain " + domain + " had no isMRI webpages - only containsMRI.");
-		    }
-
-		    //urlsList.addAll(moreURLs);
-		    for(int i = 0; i < moreURLs.size(); i++) {
-			urlsList.add(new Tuple(moreURLs.get(i), countryCode));
-		    }
-		    
-		}
-	    }
-	    System.err.println("");
-	} catch(Exception e) {
-	    logger.error("Unable to read URLs from file " + Utility.getFilePath(domainsFile));
-	    logger.error(e.getMessage(), e);
-	}
-
-	return urlsList;
-    }
-    
-    /** Given a hand curated list of all sites with positive numPagesContainingMRI
-     * determined by manual inspection, get a listing of all their web pages that
-     * are IN_MRI (or CONTAINS_MRI?). 
-     * Total all these pages that are inMRI (N), then work out the correct sample size (n)
-     * at 90% confidence with 5% margin of error. Then generate a random listing
-     * of n of these pages in MRI of these trusted sites and output to a file
-     * for manual inspection of the sample webpage URLs at page-level. */
-    /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
-     * of all their web pages IN_MRI (or CONTAINS_MRI).
-     * Plus a listing of all the NZ pages IN_MRI. */
-    //public void webPagesOfAllNZSitesAndDomainListing(File domainsFile) {
-    public void mriWebPageListingForDomainListing(File domainsFile) {
-
-	int filterType = MongoDBQueryer.IS_MRI;
-	
-	// for overseas websites, 
-	//produceURLsForPagesContainingMRI(handCuratedOverseasDomainsFile);
-
-	// 0. get a list of all the web pages in the given domain listing where isMRI = true
-	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
-	    // produceURLsForPagesInMRI(domainsFile);
-	
-	// 1. calculate the population size, N, the number of all webpages in the given domain
-	// site listing where isMRI = true.
-	int N_totalNumPages = urlsList.size();
-
-	// 2. write all the URLs in urlsList to a file
-	//File outFolder = domainsFile.getParentFile();
-	String fileName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI_" : "containsMRI_";
-	File outFile = new File(outFolder, fileName+domainsFile.getName());
-
-	writeURLsToFile(urlsList, outFile, N_totalNumPages);
-	System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
-			   + "\ninto file: " + Utility.getFilePath(outFile));
-		
-	// 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
-	int n_numSampleURLs = calcSampleSize(N_totalNumPages); 
-
-	System.err.println("*** N, total number of web pages that matched: " + N_totalNumPages);
-	System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);
-    
-	// 4. Shuffle all the URLs and write the first n (sample size) URLs to a file
-	// Using a constant seed for reproducibility
-	// https://stackoverflow.com/questions/6284589/setting-a-seed-to-shuffle-arraylist-in-java-deterministically
-	Collections.shuffle(urlsList, new Random(FIXED_SEED)); 
-	
-	outFile = new File(outFolder, "random"+n_numSampleURLs+"_"+domainsFile.getName());
-	writeURLsToFile(urlsList, outFile, n_numSampleURLs);
-	System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
-			   + "for the sites in input domainsFile\ninto file: " + Utility.getFilePath(outFile));
-    }
-
-    /**
-     * Calculates sample size n for binary outcomes at 90% confidence and 5% margine of error
-     * for given population size N.
-     * @return n, the sample size.
-     */
-    public int calcSampleSize(int N) {
-	
-	// calculate sample size n for population size N if using 90% confidence and 5% margin of error
-	// https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
-	// https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-sample-size/#CI1
-	// https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
-	
-	double m = 0.05; // margin of error = 5%
-	// for 90% confidence, alpha is the remainder = 10% and alpha/2 = 5%.
-	// For 90% confidence, use the table of known z_alpha/2 values from step 1 of
-	// https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
-	double z_alpha_over_2 = 1.6449;
-
-	// Formula: n = (zalpha2 ^ 2 * N) / ((z-alpha-2 ^ 2) + 4(N-1)*m^2)
-	// see https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
-	double n = (Math.pow(z_alpha_over_2, 2.0) * N) / (Math.pow(z_alpha_over_2, 2.0) + (4 * (N-1) * Math.pow(m,2.0)));
-
-	// Round up to get a whole number:
-	return (int)Math.ceil(n);
-    }
-    
-    /**
-     * Writes out the first n URLs in urlsList into outFile.
-     */
-    private void writeURLsToFile(ArrayList<Tuple> urlsList, File outFile, final int n) {
-	try (
-	     Writer writer = new BufferedWriter(new FileWriter(outFile));
-	     ) {
-
-	    for (int i=0; i < n; i++) {
-		Tuple urlInfo = urlsList.get(i);
-		
-		//System.out.println(list.get(i));
-		writer.write(urlInfo + "\n"); // calls toString() on tuple of url -> countryCode
-	    }
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + Utility.getFilePath(outFile));
-	    logger.error(e.getMessage(), e);
-	}
-    }
-
-    /* ---------------------------------------- */
-    /** 
-     * Create the file 5counts_containsMRISites_allNZGrouped.json
-     * that contains the count and domains for NZ sites (NZ origin or nz TLD) with pages
-     * that CONTAIN_MRI, followed by counts and domains listing for overseas sites
-     * that CONTAIN_MRI.
-     * @return full path of file generated
-     */
-    public String writeContainsMRISites_nzSitesAndTLDsGrouped() {
-	
-	File outFile = new File(outFolder, "5counts_containsMRISites_allNZGrouped.json");
-
-	String filename = Utility.getFilePath(outFile);
-	
-	try (
-	     Writer writer = new BufferedWriter(new FileWriter(outFile));
-	     ) {
-	    // first write out NZ sites and .nz TLD count and domains
-	    mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
-	    // next write out all overseas sites (not NZ origin or .nz TLD)
-	    // that have no "mi" in the URL path as mi.* or */mi
-	    boolean isMiInURLPath = false;
-	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI);
-	    
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + filename);
-	    logger.error(e.getMessage(), e);
-	}
-	
-	System.err.println("*** Wrote file: " + filename);
-
-	return filename;
-    }
-    
-    /** 
-     * Create the file 5a_counts_tentativeNonAutotranslatedSites.json
-     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
-     * followed by counts and domain listing for overseas sites that are either from Australia
-     * or don't contain mi in their URL path.
-     * @return full path of file generated
-     */
-    public String writeTentativeNonAutotranslatedSites() {
-	
-	File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
-
-	String filename = Utility.getFilePath(outFile);
-	
-	try (
-	     Writer writer = new BufferedWriter(new FileWriter(outFile));
-	     ) {
-	    // first write out NZ sites and .nz TLD count and domains
-	    mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
-	    // next write out all overseas sites (not NZ origin or .nz TLD)
-	    // that have no "mi" in the URL path as mi.* or */mi
-	    boolean isMiInURLPath = false;
-	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
-	    
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + filename);
-	    logger.error(e.getMessage(), e);
-	}
-	
-	System.err.println("*** Wrote file: " + filename);
-
-	return filename;
-    }
-
-    /**
-     * Create the file 5b_counts_overseasSitesWithMiInPath.json
-     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
-     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
-     * This listing is separate to allow easier weeding out of product sites/autotranslated
-     * sites when eyeballing the listing output.
-     */
-    public String writeOverseasSitesWithMiInURLPath() {
-	File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
-
-	String filename = Utility.getFilePath(outFile);
-	try (
-	     Writer writer = new BufferedWriter(new FileWriter(outFile));
-	     ) {
-	    boolean isMiInURLPath = true;
-	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
-
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + filename);
-	    logger.error(e.getMessage(), e);
-	}
-
-	System.err.println("*** Wrote file: " + filename);
-	return filename;
-    }
-	
-    public static void printUsage() {
-	System.err.println("Usage: WebPageURLsListing [domains.txt]");
-    }
-
-    /**
-     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
-     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
-     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
-     * translated and really contain at least one webpage containing at least one sentence in MRI.
-     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
-     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
-     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
-     * 90% confidence with 5% margin of error for testing binary outcomes, see
-     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
-     */
-    public static void main(String args[]) {
-	if(args.length >= 2) {
-	    printUsage();
-	    System.exit(-1);
-	}
-
-	try (
-	     MongoDBQueryer mongodb = new MongoDBQueryer();
-	     ) {
-
-	    mongodb.connectToDB();
-
-	    // output files will be stored in mongodb-data-auto
-	    File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
-	    WebPageURLsListing listing = new WebPageURLsListing(mongodb, outFolder);
-
-	    System.out.println("*************************************");
-	    
-	    
-	    if(args.length >= 1) {
-		File domainsFile = new File(args[0]);
-		if(!domainsFile.exists()) {
-		    System.err.println("File " + domainsFile + " does not exist");
-		    System.exit(-1);
-		}
-
-		//String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
-		//String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);
-
-
-		// TODO: for NZ, do IS_MRI. For overseas still CONTAINS_MRI
-		// then also do the shuffle to gen X num of random web page URLs.
-		//String filename = listing.webPagesOfAllNZSitesAndDomainListing(domainsFile);
-		listing.mriWebPageListingForDomainListing(domainsFile);
-
-		// TODO: generate the special table (6)
-		
-	    } else {		
-
-		// calculating sample size works:
-		//System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
-		//System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
-
-		// get all sites where >0 pages have containsMRI=true
-		// grouping NZ sites and .nz TLDs together and remainder under overseas
-		// geolocations.
-		String filename = listing.writeContainsMRISites_nzSitesAndTLDsGrouped();
-
-		// separately:
-		// - all NZ containsMRI + overseas tentative non-product sites with containMRI
-		// - overseas tentative product sites with containMRI
-		filename = listing.writeTentativeNonAutotranslatedSites();
-		filename = listing.writeOverseasSitesWithMiInURLPath();
-
-		// TODO: generate the tables
-
-		mongodb.writeTables(outFolder);
-	    }
-
-	    System.out.println("*************************************");
-	} catch(Exception e) {
-	    logger.error(e.getMessage(), e);
-	}
-    }
-}
