Index: other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java
===================================================================
--- other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33906)
+++ other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33909)
@@ -29,5 +29,5 @@
     static private final long FIXED_SEED = 1000;
     
-    private final MongoDBAccess mongodbAccess;
+    private final MongoDBQueryer mongodbQueryer;
     private File outFolder;
 
@@ -49,7 +49,7 @@
     
     
-    public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
+    public WebPageURLsListing(MongoDBQueryer mongodbQueryer, File outFolder)
     {
-	this.mongodbAccess = mongodbAccess;
+	this.mongodbQueryer = mongodbQueryer;
 	this.outFolder = outFolder;
     }
@@ -57,5 +57,5 @@
     
     public void produceURLsForPagesInMRI(File domainsFile) {
-	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
+	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
 	File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
 	writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -66,5 +66,5 @@
     
     public void produceURLsForPagesContainingMRI(File domainsFile) {
-	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.CONTAINS_MRI, domainsFile);	
+	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.CONTAINS_MRI, domainsFile);	
 	File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
 	writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -97,8 +97,8 @@
 			domain = domain.substring(0, index);
 		    }
-		    ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
+		    ArrayList<String> moreURLs = mongodbQueryer.queryAllMatchingURLsFilteredBy(domain, filterType);
 
 		    // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
-		    if(moreURLs.size() == 0 && filterType == MongoDBAccess.IS_MRI) {
+		    if(moreURLs.size() == 0 && filterType == MongoDBQueryer.IS_MRI) {
 			System.out.println("   " + countryCode + " domain " + domain + " had no isMRI webpages - only containsMRI.");
 		    }
@@ -133,5 +133,5 @@
     public void mriWebPageListingForDomainListing(File domainsFile) {
 
-	int filterType = MongoDBAccess.IS_MRI;
+	int filterType = MongoDBQueryer.IS_MRI;
 	
 	// for overseas websites, 
@@ -139,5 +139,5 @@
 
 	// 0. get a list of all the web pages in the given domain listing where isMRI = true
-	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
+	ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
 	    // produceURLsForPagesInMRI(domainsFile);
 	
@@ -148,5 +148,5 @@
 	// 2. write all the URLs in urlsList to a file
 	//File outFolder = domainsFile.getParentFile();
-	String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
+	String fileName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI_" : "containsMRI_";
 	File outFile = new File(outFolder, fileName+domainsFile.getName());
 
@@ -219,7 +219,39 @@
 
     /* ---------------------------------------- */
-
     /** 
-     * Create the file 5counts_tentativeNonAutotranslatedSites.json
+     * Create the file 5counts_containsMRISites_allNZGrouped.json
+     * that contains the count and domains for NZ sites (NZ origin or nz TLD) with pages
+     * that CONTAIN_MRI, followed by counts and domains listing for overseas sites
+     * that CONTAIN_MRI.
+     * @return full path of file generated
+     */
+    public String writeContainsMRISites_nzSitesAndTLDsGrouped() {
+	
+	File outFile = new File(outFolder, "5counts_containsMRISites_allNZGrouped.json");
+
+	String filename = Utility.getFilePath(outFile);
+	
+	try (
+	     Writer writer = new BufferedWriter(new FileWriter(outFile));
+	     ) {
+	    // first write out NZ sites and .nz TLD count and domains
+	    mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
+	    // next write out all overseas sites (not NZ origin or .nz TLD)
+	    // that have no "mi" in the URL path as mi.* or */mi
+	    boolean isMiInURLPath = false;
+	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI);
+	    
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + filename);
+	    logger.error(e.getMessage(), e);
+	}
+	
+	System.err.println("*** Wrote file: " + filename);
+
+	return filename;
+    }
+    
+    /** 
+     * Create the file 5a_counts_tentativeNonAutotranslatedSites.json
      * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
      * followed by counts and domain listing for overseas sites that are either from Australia
@@ -237,9 +269,9 @@
 	     ) {
 	    // first write out NZ sites and .nz TLD count and domains
-	    mongodbAccess.aggregateContainsMRIForNZ(writer, MongoDBAccess.CONTAINS_MRI);
+	    mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
 	    // next write out all overseas sites (not NZ origin or .nz TLD)
 	    // that have no "mi" in the URL path as mi.* or */mi
 	    boolean isMiInURLPath = false;
-	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
+	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
 	    
 	} catch(Exception e) {
@@ -254,4 +286,5 @@
 
     /**
+     * Create the file 5b_counts_overseasSitesWithMiInPath.json
      * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
      * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
@@ -267,5 +300,5 @@
 	     ) {
 	    boolean isMiInURLPath = true;
-	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
+	    mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
 
 	} catch(Exception e) {
@@ -300,5 +333,5 @@
 
 	try (
-	     MongoDBAccess mongodb = new MongoDBAccess();
+	     MongoDBQueryer mongodb = new MongoDBQueryer();
 	     ) {
 
@@ -335,6 +368,14 @@
 		//System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
 		//System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
-		
-		String filename = listing.writeTentativeNonAutotranslatedSites();
+
+		// get all sites where >0 pages have containsMRI=true
+		// grouping NZ sites and .nz TLDs together and remainder under overseas
+		// geolocations.
+		String filename = listing.writeContainsMRISites_nzSitesAndTLDsGrouped();
+
+		// separately:
+		// - all NZ containsMRI + overseas tentative non-product sites with containMRI
+		// - overseas tentative product sites with containMRI
+		filename = listing.writeTentativeNonAutotranslatedSites();
 		filename = listing.writeOverseasSitesWithMiInURLPath();
 
