Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33881)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33882)
@@ -96,4 +96,8 @@
     public static final int CONTAINS_MRI = 1;
 
+    /** Some reused fieldnames in the Websites collection */
+    private static final String FILTER_NUMPAGES_IN_MRI = "numPagesInMRI";
+    private static final String FILTER_NUMPAGES_CONTAINING_MRI = "numPagesContainingMRI";
+
     // configuration details, some with fallback values
     private String HOST = "localhost";
@@ -402,10 +406,9 @@
 	]);
     */
-    public void aggregateContainsMRIForNZ(Writer writer) throws IOException {
+    public void aggregateContainsMRIForNZ(Writer writer, int filterType) throws IOException {
 	// working with the WebSites collection, not WebPages collection!
 	MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
 
-	
-	//String isMRI_filter = 
+	String mriFilterString = (filterType == CONTAINS_MRI) ? "{numPagesContainingMRI: {$gt: 0}}" : "{numPagesInMRI: {$gt: 0}}";
     
 	Bson orQuery = or(
@@ -414,5 +417,5 @@
 			  );
 	Bson andQuery = and(
-		BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
+		BasicDBObject.parse(mriFilterString),
 		orQuery);
 	
@@ -456,17 +459,21 @@
 	]);
     */
-    public void aggregateContainsMRIForOverseas(Writer writer) throws UncheckedIOException {
+    public void aggregateContainsMRIForOverseas(Writer writer, int filterType,
+						boolean isMiInURLPath) throws UncheckedIOException
+    {
 	// working with the WebSites collection, not WebPages collection!
 	MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
-
+	
+	String mriFilterString = (filterType == CONTAINS_MRI) ? "{numPagesContainingMRI: {$gt: 0}}" : "{numPagesInMRI: {$gt: 0}}";
     
 	Bson orQuery = or(
 			  BasicDBObject.parse("{geoLocationCountryCode: \"AU\"}"),
-			  BasicDBObject.parse("{urlContainsLangCodeInPath: false}")
+			  BasicDBObject.parse("{urlContainsLangCodeInPath: "+ isMiInURLPath +"}")
+			  // e.g. "{urlContainsLangCodeInPath: false}"
 			  );
 	Bson andQuery = and(
 		BasicDBObject.parse("{geoLocationCountryCode: {$ne: \"NZ\"}}"),
 		BasicDBObject.parse("{domain: {$not: /\\.nz/}}"),
-		BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
+		BasicDBObject.parse(mriFilterString),
 		orQuery);
 
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33881)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java	(revision 33882)
@@ -25,34 +25,22 @@
     
     private final MongoDBAccess mongodbAccess;
-    private int numURLs;
-    private File domainsFile;
-
-
-    
-    public WebPageURLsListing(MongoDBAccess mongodbAccess,
-					File domainsFile)
+    private File outFolder;
+
+    public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
     {
 	this.mongodbAccess = mongodbAccess;
-	this.domainsFile = domainsFile;
-    }
-    
-    public WebPageURLsListing(MongoDBAccess mongodbAccess,
-					File domainsFile,
-					int numURLs)
-    {
-	this(mongodbAccess, domainsFile);
-	this.numURLs = numURLs;
-    }
-
-    public String produceURLsForPagesInMRI() {
-	return writeFile(MongoDBAccess.IS_MRI);
-    }
-    
-    public String produceURLsForPagesContainingMRI() {
-	return writeFile(MongoDBAccess.CONTAINS_MRI);
-    }
-
-
-    public String writeFile(int filterType) {
+	this.outFolder = outFolder;
+    }
+    
+    public String produceURLsForPagesInMRI(File domainsFile) {
+	return writeFile(MongoDBAccess.IS_MRI, domainsFile);
+    }
+    
+    public String produceURLsForPagesContainingMRI(File domainsFile) {
+	return writeFile(MongoDBAccess.CONTAINS_MRI, domainsFile);
+    }
+
+
+    public String writeFile(int filterType, File domainsFile/*, int numURLs*/) {
 
 	ArrayList<String> urlsList = new ArrayList<String>();
@@ -126,10 +114,13 @@
 
     /** 
-     * Create the file 
+     * Create the file 5counts_tentativeNonAutotranslatedSites.json
+     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
+     * followed by counts and domain listing for overseas sites that are either from Australia
+     * or don't contain mi in their URL path.
      * @return full path of file generated
      */
     public String writeTentativeNonAutotranslatedSites() {
-	File outFolder = new File("../mongodb-data/").getAbsoluteFile();
-	File outFile = new File(outFolder, "5counts_tentativeNonAutotranslatedSites.json");
+	
+	File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
 
 	String filename = outFile.getAbsolutePath();
@@ -139,7 +130,9 @@
 	     ) {
 	    // first write out NZ sites and .nz TLD count and domains
-	    mongodbAccess.aggregateContainsMRIForNZ(writer);
-	    // next write out all overseas sites and .nz TLD count and domains
-	    mongodbAccess.aggregateContainsMRIForOverseas(writer);
+	    mongodbAccess.aggregateContainsMRIForNZ(writer, MongoDBAccess.CONTAINS_MRI);
+	    // next write out all overseas sites (not NZ origin or .nz TLD)
+	    // that have no "mi" in the URL path as mi.* or */mi
+	    boolean isMiInURLPath = false;
+	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
 
 	    filename = outFile.getCanonicalPath();
@@ -148,8 +141,32 @@
 	    logger.error(e.getMessage(), e);
 	}
+	
+	System.err.println("*** Wrote file: " + filename);
 
 	return filename;
     }
-    
+
+    /**
+     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
+     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path
+     */
+    public String writeOverseasSitesWithMiInURLPath() {
+	File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
+
+	String filename = outFile.getAbsolutePath();
+	try (
+	     Writer writer = new BufferedWriter(new FileWriter(outFile));
+	     ) {
+	    boolean isMiInURLPath = true;
+	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
+	    filename = outFile.getCanonicalPath();
+	} catch(Exception e) {
+	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
+	    logger.error(e.getMessage(), e);
+	}
+
+	System.err.println("*** Wrote file: " + filename);
+	return filename;
+    }
 	
     public static void printUsage() {
@@ -170,19 +187,31 @@
 	     MongoDBAccess mongodb = new MongoDBAccess();
 	     ) {
-	    File domainsFile = new File(args[0]);
-	    if(!domainsFile.exists()) {
-		System.err.println("File " + domainsFile + " does not exist");
-		System.exit(-1);
-	    }
-
-	    //int genNumURLs = Integer.parseInt(args[1]);
 
 	    mongodb.connectToDB();
-	    
-	    WebPageURLsListing listing = new WebPageURLsListing(mongodb, domainsFile);
-	    //String isMRIFile = listing.produceURLsForPagesInMRI();
-	    //String containsMRIFile = listing.produceURLsForPagesContainingMRI();
-	    String filename = listing.writeTentativeNonAutotranslatedSites();
-	    System.err.println("Check file: " + filename);
+
+	    // output files will be stored in mongodb-data-auto
+	    File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
+	    WebPageURLsListing listing = new WebPageURLsListing(mongodb, outFolder);
+
+	    
+	    if(args.length >= 1) {
+		File domainsFile = new File(args[0]);
+		if(!domainsFile.exists()) {
+		    System.err.println("File " + domainsFile + " does not exist");
+		    System.exit(-1);
+		}
+
+		
+		//int genNumURLs = Integer.parseInt(args[1]);
+		//String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
+		//String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);
+
+		//listing.writeWebPagesOfAllNZSitesAndDomainListing();
+		
+	    } else {		
+
+		String filename = listing.writeTentativeNonAutotranslatedSites();
+		filename = listing.writeOverseasSitesWithMiInURLPath();
+	    }
 
 	    
