Index: /gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt	(revision 33603)
+++ /gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt	(revision 33604)
@@ -61,4 +61,8 @@
 # special case
 mi.centr-zashity.ru,SINGLEPAGE
+
+martinvrijland.nl,martinvrijland.nl/mi/
+csunplugged.org,csunplugged.org/mi/
+gpedia.com,gpedia.com/mi/
 
 # TOP SITE BUT NOT TOP 500
Index: /gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt	(revision 33603)
+++ /gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt	(revision 33604)
@@ -2,8 +2,8 @@
 # whitelist overrides blacklist and greylist.
 # FORMAT:
-# precede URL by ^ to greylist urls that match the given prefix
-# succeed URL by $ to greylist urls that match the given suffix
-# ^url$ will greylist urls that match the given url completely
-# Without either ^ or $ symbol, urls containing the given url will get greylisted
+# precede URL by ^ to whitelist urls that match the given prefix
+# succeed URL by $ to whitelist urls that match the given suffix
+# ^url$ will whitelist urls that match the given url completely
+# Without either ^ or $ symbol, urls containing the given url will get whitelisted
 
 # Special exception for this url on yale.edu, since we needed to blacklist
@@ -15,2 +15,17 @@
 http://www.krassotkin.ru/sites/prayer.su/maori/
 https://mi.centr-zashity.ru/
+
+
+
+# WHITELIST WEBSITES THAT HAVE NON-AUTOMATED /mi/ SUBSECTIONS
+# WE CONTROL WHAT PART OF THEM WILL BE DOWNLOADED (THE /mi SUBSECTION)
+# IN sites-too-big-to-exhaustively-crawl.txt
+#https://www.martinvrijland.nl/mi/te-mana-hinengaro/Ko-te-nuinga-ake-o-nga-tangata-kei-te-timata-ki-te-kite-kei-te-noho-tatou-i-roto-i-te-whakaata-ko-te-aha-tenei/
+#https://www.csunplugged.org/mi/principles/
+#http://www.gpedia.com/mi/gpedia/Reo_M%C4%81ori
+
+https://www.martinvrijland.nl
+https://www.csunplugged.org
+http://www.gpedia.com
+
+
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33603)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33604)
@@ -324,15 +324,32 @@
 		if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
 		    if(!possibleProductDomains.contains(domainWithProtocol)) {
-			// more expensive test, so do this only if above conditions are true:
-			if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+
+			String countryCode = "";
+			try {
+			    // more expensive test, so do this only if above conditions are true:
+			    countryCode = Utility.getCountryCodeOfDomain(domainWithProtocol, geoLiteCityDatFile);
+			    System.err.println("@@@@ Got country code: " + countryCode);
+			} catch(Exception exceptObj) {
+			    countryCode = ""; // forces domain to be included for inspection
+			    
+			    error("Could not check if domain " + domainWithProtocol
+				  + " was in country: " + countryCode,
+				  exceptObj);
+			}
+
+			boolean isInNZ = countryCode.toLowerCase().equals("nz");
+			
+
+			//if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+			if(!isInNZ) {
 			    possibleProductDomains.add(domainWithProtocol);
-			    // write both domain and URL out to file
-			    possibleProductSitesWriter.write(domainWithProtocol + "\n");
+			    // write both domain and a sample URL on that site out to file
+			    possibleProductSitesWriter.write(countryCode + " : " + domainWithProtocol + "\n");			    
 			    possibleProductSitesWriter.write("\t" + url + "\n");
 			}
-		    } else {
-			// already wrote out domain to file, write just the URL out to file
+		    } /*else {
+			// already wrote out domain to file at some point, write just the URL out to file
 			possibleProductSitesWriter.write("\t" + url + "\n");
-		    }
+			}*/
 		}
 	    }
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33603)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33604)
@@ -54,5 +54,32 @@
      * version I'm using: https://github.com/maxmind/geoip-api-java
      * Newer version: https://maxmind.github.io/GeoIP2-java/
+     *
+     * @return 2 letter countrycode in uppercase or an exception
      */
+    public static String getCountryCodeOfDomain(String domainWithProtocol, File geoLiteCityDatFile)
+	throws Exception
+    {	
+	int startIndex = domainWithProtocol.indexOf("//"); // http:// or https:// prefix
+	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+	String domain = domainWithProtocol.substring(startIndex);	
+	
+	// pass in the GeoLiteCity.dat file to be able to do the location lookup for domain's IP
+	LookupService cl = new LookupService(geoLiteCityDatFile, LookupService.GEOIP_MEMORY_CACHE);
+	    
+	// get IP for domain
+	InetAddress inetAddress = InetAddress.getByName(domain);
+	String ipAddress = inetAddress.getHostAddress();
+	
+	// get location object for IP
+	Location location = cl.getLocation(ipAddress);
+	
+	if(location == null) {
+	    throw new Exception("@@@@ No location info in DB for: " + domain);
+	} else {
+	    return location.countryCode;
+	}
+	
+    }
+    
     public static boolean isDomainInCountry(String domainWithProtocol,
 					    String countryCode, File geoLiteCityDatFile)
