Index: /other-projects/maori-lang-detection/MoreReading/mongodb.txt
===================================================================
--- /other-projects/maori-lang-detection/MoreReading/mongodb.txt	(revision 33665)
+++ /other-projects/maori-lang-detection/MoreReading/mongodb.txt	(revision 33666)
@@ -346,2 +346,46 @@
 https://docs.mongodb.com/manual/reference/method/db.collection.find/
 https://docs.mongodb.com/manual/reference/method/db.collection.find/#find-projection
+
+
+-------------------
+
+Some queries with results:
+
+# Num websites
+db.getCollection('Websites').find({}).count()
+1446 
+
+# Num webpages
+db.getCollection('Webpages').find({}).count()
+75139
+
+# Find number of websites who have 1 or more pages in Maori (a positive numPagesInMRI)
+db.getCollection('Websites').find({numPagesInMRI: { $gt: 0}}).count()
+361
+
+# Find number of webpages that are deemed to be overall in MRI (pages where isMRI=true)
+db.getCollection('Webpages').find({isMRI:true}).count()
+X5224
+5215
+
+# Number of pages that contain any number of MRI sentences
+db.getCollection('Webpages').find({containsMRI: true}).count()
+12858
+
+# Number of sites with URLs containing /mi(/)
+db.getCollection('Websites').find({urlContainsLangCodeInpath:true}).count()
+153
+
+# Number of websites that are outside NZ that contain /mi(/) in any of its sub-urls
+db.getCollection('Websites').find({urlContainsLangCodeInpath:true, geoLocationCountryCode: {$ne : "NZ"} }).count()
+148
+
+# 5 sites with URLs containing /mi(/) that are in NZ
+db.getCollection('Websites').find({urlContainsLangCodeInpath:true, geoLocationCountryCode: "NZ"}).count()
+5
+
+# sort websites that contain /mi(/) in path by geoLocationCountryCode
+#    https://www.quackit.com/mongodb/tutorial/mongodb_sort_query_results.cfm
+db.getCollection('Websites').find({urlContainsLangCodeInpath:true}).sort({geoLocationCountryCode: 1})
+
+
Index: /other-projects/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt
===================================================================
--- /other-projects/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt	(revision 33665)
+++ /other-projects/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt	(revision 33666)
@@ -30,6 +30,7 @@
 #     However, if the seedurl's domain is an exact match on topsite-base-url, the seedurl will go
 #     into the file unprocessed-topsite-matches.txt and the site/page won't be crawled.
-#   - FOLLOW-LINKS-WITHIN-TOPSITE: if pages linked from the seedURL page can be followed and
-#     downloaded, as long as it's within the same subdomain matching the topsite-base-url.
+#   - FOLLOW-LINKS-WITHIN-TOPSITE: download seedURL pages and pages linked from each seedURL
+#     page should be followed and downloaded too, as long as they're within the same subdomain
+#     matching the topsite-base-url.
 #     This is different from SUBDOMAIN-COPY, as that can download all of a specific subdomain but
 #     restricts against downloading the entire domain (e.g. all pinky.blogspot.com and not anything
@@ -61,4 +62,8 @@
 # special case
 mi.centr-zashity.ru,SINGLEPAGE
+
+# we want the http://loquevendra318.com/fox/maori.html seed URL but also
+# pages within the following subsection
+loquevendra318.com,loquevendra318.com/fox/maori/
 
 martinvrijland.nl,martinvrijland.nl/mi/
Index: /other-projects/maori-lang-detection/hdfs-cc-work/conf/regex-urlfilter.GS_TEMPLATE
===================================================================
--- /other-projects/maori-lang-detection/hdfs-cc-work/conf/regex-urlfilter.GS_TEMPLATE	(revision 33665)
+++ /other-projects/maori-lang-detection/hdfs-cc-work/conf/regex-urlfilter.GS_TEMPLATE	(revision 33666)
@@ -28,5 +28,12 @@
 # skip image and other suffixes we can't yet parse
 # for a more extensive coverage use the urlfilter-suffix plugin
--\.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|CSS|sit|SIT|eps|EPS|wmf|WMF|zip|ZIP|ppt|PPT|mpg|MPG|xls|XLS|gz|GZ|rpm|RPM|tgz|TGZ|mov|MOV|exe|EXE|jpeg|JPEG|bmp|BMP|js|JS)$
+#
+# GS: Added mp3 to file types to skip, as we don't get text out of it
+# and it ends up to large to download. (Result in "key value to large"
+# error message as for site 00152, http://loquevendra318.com/fox/maori.html
+# where the audios are not in Maori anyway, being reuse for all languages
+# on the site.)
+#
+-\.(mp3|gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|CSS|sit|SIT|eps|EPS|wmf|WMF|zip|ZIP|ppt|PPT|mpg|MPG|xls|XLS|gz|GZ|rpm|RPM|tgz|TGZ|mov|MOV|exe|EXE|jpeg|JPEG|bmp|BMP|js|JS)$
 
 # skip URLs containing certain characters as probable queries, etc.
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33665)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33666)
@@ -57,6 +57,6 @@
  * e.g. (from maori-lang-detection/src)
  *    
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor ../ccrawl-data /Scratch/ak19/gs3-extensions/maori-lang-detection/to_crawl
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor ../ccrawl-data /Scratch/ak19/gs3-extensions/maori-lang-detection/to_crawl 2&gt;&amp;1 | less
+ *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor ../ccrawl-data /Scratch/ak19/maori-lang-detection/to_crawl
+ *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor ../ccrawl-data /Scratch/ak19/maori-lang-detection/to_crawl 2&gt;&amp;1 | less
  *
 */
@@ -452,5 +452,5 @@
 
 		    // Only write urls and no domain into single global seedurls file
-		    // But write domain and tabbed urls into individual sites/0000#/seedURLs.txt
+		    // But write domain and tab-spaced urls into individual sites/0000#/seedURLs.txt
 		    // files (and write regexed domain into each sites/0000#/regex-urlfilter.txt)
 		    // If we ever run nutch on a single seedURLs listing containing
@@ -515,10 +515,32 @@
 				allowedURLPatternRegex += "/";
 			    }
-			    String regexed_pattern = PROTOCOL_REGEX_PREFIX+escapeStringForRegex(allowedURLPatternRegex);
+			    String regexed_pattern = FILTER_REGEX_PREFIX+escapeStringForRegex(allowedURLPatternRegex);
 			    //String regexed_pattern = PROTOCOL_REGEX_PREFIX+allowedURLPatternRegex.replace(".", "\\.");
+			    
+			    // In case any of the seedURLs themselves are not within the
+			    // allowedURLPatternRegex part of the site, FIRST write out such
+			    // seedURLs as allowed regex patterns, so they get downloaded
+			    // as single pages.
+			    urlsForDomainSet = domainsToURLsMap.get(domainWithProtocol);
+			    for(String urlInDomain : urlsForDomainSet) {
+				
+				String urlWithoutProtocolAndWWW = Utility.stripProtocolAndWWWFromURL(urlInDomain);
+				String allowedURLPatternWithoutProtocolAndWWW = Utility.stripProtocolAndWWWFromURL(allowedURLPatternRegex);
+				if(!urlWithoutProtocolAndWWW.startsWith(allowedURLPatternWithoutProtocolAndWWW)) {
+				    // don't append slash to end this time
+				    String regexed_url = "+^"+escapeStringForRegex(urlInDomain);
+				    urlFilterWriter.write(regexed_url + "\n");
+				    siteRegexWriter.write(regexed_url + "\n");
+				}
+			    }
+
 			    siteURLsWriter.write(domainWithProtocol + "\n");
+			    // write out allowedURLPatternRegex istead of the domain
+			    //siteURLsWriter.write(allowedURLPatternRegex + "\n");
+			    
+			    // Now restrict any other URLs found to be within the allowedURLPattern
+			    // part of the site
 			    urlFilterWriter.write(regexed_pattern + "\n");
-			    siteRegexWriter.write(regexed_pattern + "\n");
-
+			    siteRegexWriter.write(regexed_pattern + "\n");			    
 			}
 		    }
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/Utility.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33665)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33666)
@@ -60,7 +60,7 @@
 	throws Exception
     {	
-	int startIndex = domainWithProtocol.indexOf("//"); // http:// or https:// prefix
-	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
-	String domain = domainWithProtocol.substring(startIndex);	
+	//int startIndex = domainWithProtocol.indexOf("//"); // http:// or https:// prefix
+	//startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+	String domain = stripProtocolFromURL(domainWithProtocol); //domainWithProtocol.substring(startIndex);	
 	
 	// pass in the GeoLiteCity.dat file to be able to do the location lookup for domain's IP
@@ -82,4 +82,21 @@
     }
     
+    public static String stripProtocolAndWWWFromURL(String url) {
+	url = stripProtocolFromURL(url);
+	
+	if(url.startsWith("www.")) { // strip any "wwww." at start as well too
+	    url = url.substring(4);
+	}
+	
+	return url;
+    }
+    
+    public static String stripProtocolFromURL(String url) {
+	int startIndex = url.indexOf("//"); // for http:// or https:// prefix
+	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+	return url.substring(startIndex);
+    }
+    
+    
     /** Work out the 'domain' for a given url.
      * This retains any www. or subdomain prefix.
@@ -88,5 +105,5 @@
 	int startIndex = startIndex = url.indexOf("//"); // for http:// or https:// prefix
 	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
-	// the keep the URL around in case param withProtocol=true
+	// keep the protocol around in case param withProtocol=true
 	String protocol = (startIndex == -1) ? "" : url.substring(0, startIndex);
 	
