Index: other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv
===================================================================
--- other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv	(revision 33882)
+++ other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv	(revision 33883)
@@ -1,3 +1,3 @@
-﻿"_id","siteCount","numPagesInMRICount","numPagesContainingMRICount"
+﻿"_id","siteCount (numPagesContainingMRICount > 0)","numPagesInMRICount","numPagesContainingMRICount"
 "nz","176.0","4360","9641"
 "us","117.0","757","2655"
Index: other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json
===================================================================
--- other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json	(revision 33882)
+++ other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json	(revision 33883)
@@ -180,4 +180,6 @@
 
 
+First column: n pages that are in MRI / n sampled isMRI pages
+Second column: n pages that do contain MRI / n sampled pages that are not isMRI yet contain MRI
 
 /* 1 */
Index: other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java
===================================================================
--- other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33882)
+++ other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33883)
@@ -537,5 +537,5 @@
 	// newlines after every array element in the json:
 	String jsonStr = prettyPrintJson(doc.toJson());
-	System.err.println(jsonStr);
+	//System.err.println(jsonStr);
 	try {
 	    writer.write(jsonStr + NEWLINE);
@@ -553,14 +553,4 @@
     }
 
-
-    public void writeToFile(boolean append, String filename, AggregateIterable<Document> output) {
-	
-	// should only have one doc
-	for (Document doc : output) {
-	    //System.out.println(doc);
-	    System.out.println(doc.toJson());
-	}
-    }
-    
     
     /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */
Index: other-projects/maori-lang-detection/src/org/greenstone/atea/RandomURLsForDomainGenerator.java
===================================================================
--- other-projects/maori-lang-detection/src/org/greenstone/atea/RandomURLsForDomainGenerator.java	(revision 33882)
+++ other-projects/maori-lang-detection/src/org/greenstone/atea/RandomURLsForDomainGenerator.java	(revision 33883)
@@ -1,5 +1,5 @@
 package org.greenstone.atea;
 
-import java.util.*;
+import java.util.*; /* includes Random */
 import java.io.*;
 
@@ -71,5 +71,5 @@
 	     Writer writer = new BufferedWriter(new FileWriter(outFile));
 	     ) {
-	    Collections.shuffle(urlsList);
+	    Collections.shuffle(urlsList, new Random(1000));
 	    for (int i=0; i<numURLs; i++) {
 		String url = urlsList.get(i);
Index: other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java
===================================================================
--- other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33882)
+++ other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33883)
@@ -7,5 +7,7 @@
 
 /**
-* TO COMPILE OR RUN, FIRST DO:
+ * Runs some of the important mongoDB queries I ran.
+ *
+ * TO COMPILE OR RUN, FIRST DO:
  *    cd maori-lang-detection/apache-opennlp-1.9.1
  *    export OPENNLP_HOME=`pwd`
@@ -18,13 +20,15 @@
  * TO RUN:
  *    maori-lang-detection/src$
- *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt 255
+ *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt
  *
 */
 public class WebPageURLsListing {
     static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
+    static private final long FIXED_SEED = 1000;
     
     private final MongoDBAccess mongodbAccess;
     private File outFolder;
 
+    
     public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
     {
@@ -32,16 +36,32 @@
 	this.outFolder = outFolder;
     }
-    
-    public String produceURLsForPagesInMRI(File domainsFile) {
-	return writeFile(MongoDBAccess.IS_MRI, domainsFile);
-    }
-    
-    public String produceURLsForPagesContainingMRI(File domainsFile) {
-	return writeFile(MongoDBAccess.CONTAINS_MRI, domainsFile);
-    }
-
-
-    public String writeFile(int filterType, File domainsFile/*, int numURLs*/) {
-
+
+    private String getFilePath(File file) {
+	try {
+	    return file.getCanonicalPath();
+	} catch(IOException e) {
+	    return file.getAbsolutePath();
+	}
+    }
+    
+    public void produceURLsForPagesInMRI(File domainsFile) {
+	ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+	File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
+	writeURLsToFile(urlsList, outFile, urlsList.size());
+	
+	System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
+			   + getFilePath(outFile));
+    }
+    
+    public void produceURLsForPagesContainingMRI(File domainsFile) {
+	ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.CONTAINS_MRI, domainsFile);	
+	File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
+	writeURLsToFile(urlsList, outFile, urlsList.size());
+
+	System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
+			   + getFilePath(outFile));
+    }
+    
+    private ArrayList<String> getURLsForWebPages(int filterType, File domainsFile) {
 	ArrayList<String> urlsList = new ArrayList<String>();
 	
@@ -52,6 +72,5 @@
 	try (
 	     BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
-	     ) {
-	    
+	     ) {	    
 	    
 	    String domain;
@@ -62,24 +81,103 @@
 		    ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
 		    urlsList.addAll(moreURLs);
+		    if(moreURLs.size() == 0 && filterType == MongoDBAccess.IS_MRI) {
+			System.out.println("Domain " + domain + " had no isMRI webpages (only containsMRI).");
+		    }
 		}
 	    }
 	} catch(Exception e) {
-	    logger.error("Unable to read URLs from file " + domainsFile.getAbsolutePath());
-	    logger.error(e.getMessage(), e);
-	}
-
-	// Shuffle the urlsList, then write out the first numURLs into a file.	
-	// https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
-	File parentFolder = domainsFile.getParentFile();
-	//File outFile = new File(parentFolder, "random"+numURLs+"_"+domainsFile.getName());
+	    logger.error("Unable to read URLs from file " + getFilePath(domainsFile));
+	    logger.error(e.getMessage(), e);
+	}
+
+	return urlsList;
+    }
+    
+    /** Given a hand curated list of NZ sites with positive numPagesContainingMRI,
+     * get a listing of all their web pages IN_MRI (or CONTAINS_MRI?). 
+     * Total all these pages in MRI (N), then work out the correct sample size (n)
+     * at 90% confidence with 5% margin of error. Then generate a random listing
+     * of n of these pages in MRI of these trusted sites and output to a file
+     * for manual inspection. */
+    /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
+     * of all their web pages IN_MRI (or CONTAINS_MRI).
+     * Plus a listing of all the NZ pages IN_MRI. */
+    //public void webPagesOfAllNZSitesAndDomainListing(File domainsFile) {
+    public void mriWebPageListingForDomainListing(File domainsFile) {
+
+	int filterType = MongoDBAccess.IS_MRI;
+	
+	// for overseas websites, 
+	//produceURLsForPagesContainingMRI(handCuratedOverseasDomainsFile);
+
+	// 0. get a list of all the web pages in the given domain listing where isMRI = true
+	ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+	    // produceURLsForPagesInMRI(domainsFile);
+	
+	// 1. calculate the population size, N, the number of all webpages in the given domain
+	// site listing where isMRI = true.
+	int N_totalNumPages = urlsList.size();
+
+	// 2. write all the URLs in urlsList to a file
+	//File outFolder = domainsFile.getParentFile();
 	String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
-	File outFile = new File(parentFolder, fileName+domainsFile.getName());
-
-	// write out ALL the URLs
+	File outFile = new File(outFolder, fileName+domainsFile.getName());
+
+	writeURLsToFile(urlsList, outFile, N_totalNumPages);
+	System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
+			   + "\ninto file: " + getFilePath(outFile));
+		
+	// 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
+	int n_numSampleURLs = calcSampleSize(N_totalNumPages); 
+
+	System.err.println("*** N, total number of web pages that matched: " + N_totalNumPages);
+	System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);
+    
+	// 4. Shuffle all the URLs and write the first n (sample size) URLs to a file
+	// Using a constant seed for reproducibility
+	// https://stackoverflow.com/questions/6284589/setting-a-seed-to-shuffle-arraylist-in-java-deterministically
+	Collections.shuffle(urlsList, new Random(FIXED_SEED)); 
+	
+	outFile = new File(outFolder, "random"+n_numSampleURLs+"_"+domainsFile.getName());
+	writeURLsToFile(urlsList, outFile, n_numSampleURLs);
+	System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
+			   + "for the sites in input domainsFile\ninto file: " + getFilePath(outFile));
+    }
+
+    /**
+     * Calculates sample size n for binary outcomes at 90% confidence and 5% margine of error
+     * for given population size N.
+     * @return n, the sample size.
+     */
+    public int calcSampleSize(int N) {
+	
+	// calculate sample size n for population size N if using 90% confidence and 5% margin of error
+	// https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+	// https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-sample-size/#CI1
+	// https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
+	
+	double m = 0.05; // margin of error = 5%
+	// for 90% confidence, alpha is the remainder = 10% and alpha/2 = 5%.
+	// For 90% confidence, use the table of known z_alpha/2 values from step 1 of
+	// https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
+	double z_alpha_over_2 = 1.6449;
+
+	// Formula: n = (zalpha2 ^ 2 * N) / ((z-alpha-2 ^ 2) + 4(N-1)*m^2)
+	// see https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+	double n = (Math.pow(z_alpha_over_2, 2.0) * N) / (Math.pow(z_alpha_over_2, 2.0) + (4 * (N-1) * Math.pow(m,2.0)));
+
+	// Round up to get a whole number:
+	return (int)Math.ceil(n);
+    }
+    
+    /**
+     * Writes out the first n URLs in urlsList into outFile.
+     */
+    private void writeURLsToFile(ArrayList<String> urlsList, File outFile, final int n) {
 	try (
 	     Writer writer = new BufferedWriter(new FileWriter(outFile));
 	     ) {
 
-	    for (int i=0; i < urlsList.size(); i++) {
+	    for (int i=0; i < n; i++) {
 		String url = urlsList.get(i);
 		//System.out.println(list.get(i));
@@ -87,26 +185,7 @@
 	    }
 	} catch(Exception e) {
-	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
-	    logger.error(e.getMessage(), e);
-	}
-
-	/*
-	// shuffle list and take the first n - write to file
-	try (
-	     Writer writer = new BufferedWriter(new FileWriter(outFile));
-	     ) {
-	    Collections.shuffle(urlsList);
-	    for (int i=0; i<numURLs; i++) {
-		String url = urlsList.get(i);
-		//System.out.println(list.get(i));
-		writer.write(url + "\n");
-	    }
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
-	    logger.error(e.getMessage(), e);
-	}
-	*/
-
-	return outFile.getAbsolutePath();
+	    logger.error("Unable to write to file " + getFilePath(outFile));
+	    logger.error(e.getMessage(), e);
+	}
     }
 
@@ -124,5 +203,5 @@
 	File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
 
-	String filename = outFile.getAbsolutePath();
+	String filename = getFilePath(outFile);
 	
 	try (
@@ -135,8 +214,7 @@
 	    boolean isMiInURLPath = false;
 	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
-
-	    filename = outFile.getCanonicalPath();
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
+	    
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + filename);
 	    logger.error(e.getMessage(), e);
 	}
@@ -149,10 +227,12 @@
     /**
      * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
-     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path
+     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
+     * This listing is separate to allow easier weeding out of product sites/autotranslated
+     * sites when eyeballing the listing output.
      */
     public String writeOverseasSitesWithMiInURLPath() {
 	File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
 
-	String filename = outFile.getAbsolutePath();
+	String filename = getFilePath(outFile);
 	try (
 	     Writer writer = new BufferedWriter(new FileWriter(outFile));
@@ -160,7 +240,7 @@
 	    boolean isMiInURLPath = true;
 	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
-	    filename = outFile.getCanonicalPath();
-	} catch(Exception e) {
-	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
+
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + filename);
 	    logger.error(e.getMessage(), e);
 	}
@@ -171,11 +251,18 @@
 	
     public static void printUsage() {
-	System.err.println("WebPageURLsListing [domains.txt]");
-    }
-
-    // Depending on args, generates isMRI and containsMRI file listings for:
-    // 1. UNFILTERED: all (NZ + overseas) AND takes manually curated domain list file for overseas and adds all NZ
-
-    
+	System.err.println("Usage: WebPageURLsListing [domains.txt]");
+    }
+
+    /**
+     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
+     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
+     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
+     * translated and really contain at least one webpage containing at least one sentence in MRI.
+     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
+     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
+     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
+     * 90% confidence with 5% margin of error for testing binary outcomes, see
+     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+     */
     public static void main(String args[]) {
 	if(args.length >= 2) {
@@ -202,17 +289,26 @@
 		}
 
-		
-		//int genNumURLs = Integer.parseInt(args[1]);
 		//String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
 		//String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);
 
-		//listing.writeWebPagesOfAllNZSitesAndDomainListing();
+
+		// TODO: for NZ, do IS_MRI. For overseas still CONTAINS_MRI
+		// then also do the shuffle to gen X num of random web page URLs.
+		//String filename = listing.webPagesOfAllNZSitesAndDomainListing(domainsFile);
+		listing.mriWebPageListingForDomainListing(domainsFile);
+
+		// TODO: generate the special table (6)
 		
 	    } else {		
 
+		// calculating sample size works:
+		//System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
+		//System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
+		
 		String filename = listing.writeTentativeNonAutotranslatedSites();
 		filename = listing.writeOverseasSitesWithMiInURLPath();
+
+		// TODO: generate the tables
 	    }
-
 	    
 	} catch(Exception e) {
