Index: /other-projects/maori-lang-detection/mongodb-data/1a_counts_miInUrlPath.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/1a_counts_miInUrlPath.json	(revision 33847)
+++ /other-projects/maori-lang-detection/mongodb-data/1a_counts_miInUrlPath.json	(revision 33848)
@@ -1,2 +1,18 @@
+/*
+All crawled sites which contain mi (mi.* or */mi) in the URL path.
+
+db.Websites.aggregate([
+    { $match: {urlContainsLangCodeInPath: true} },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: "$geoLocationCountryCode",
+            count: { $sum: 1 }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+*/
+
 /* 1 */
 {
Index: /other-projects/maori-lang-detection/mongodb-data/1a_table_miInUrlPath.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/1a_table_miInUrlPath.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/1a_table_miInUrlPath.csv	(revision 33848)
@@ -0,0 +1,25 @@
+﻿"_id","count","numPagesInMRICount","numPagesContainingMRICount"
+"US","408.0","1169","3872"
+"CN","123.0","281","1144"
+"FR","34.0","754","1091"
+"UNKNOWN","19.0","115","125"
+"NZ","14.0","618","1097"
+"DE","12.0","145","212"
+"NL","8.0","76","115"
+"CA","7.0","29","119"
+"HK","7.0","3","12"
+"AU","7.0","19","117"
+"GB","5.0","3","7"
+"JP","5.0","1","3"
+"UA","4.0","9","10"
+"RU","4.0","0","14"
+"VG","2.0","0","0"
+"SG","2.0","2","13"
+"DK","2.0","0","0"
+"IE","1.0","17","21"
+"ZA","1.0","0","0"
+"TR","1.0","0","2"
+"SE","1.0","0","0"
+"EU","1.0","0","7"
+"CZ","1.0","0","0"
+"ES","1.0","4","4"
Index: /other-projects/maori-lang-detection/mongodb-data/1b_counts_noMiInUrlPath.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/1b_counts_noMiInUrlPath.json	(revision 33847)
+++ /other-projects/maori-lang-detection/mongodb-data/1b_counts_noMiInUrlPath.json	(revision 33848)
@@ -1,2 +1,18 @@
+/*
+All crawled sites which don't contain mi (mi.* or */mi) in the URL path.
+
+db.Websites.aggregate([
+    { $match: {urlContainsLangCodeInPath: false} },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: "$geoLocationCountryCode",
+            count: { $sum: 1 }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+*/
+
 /* 1 */
 {
Index: /other-projects/maori-lang-detection/mongodb-data/1b_table_noMiInUrlPath.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/1b_table_noMiInUrlPath.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/1b_table_noMiInUrlPath.csv	(revision 33848)
@@ -0,0 +1,38 @@
+﻿"_id","count","numPagesInMRICount","numPagesContainingMRICount"
+"US","288.0","2136","5452"
+"UNKNOWN","154.0","0","12"
+"NZ","101.0","2035","5129"
+"DE","40.0","1","208"
+"AU","36.0","164","313"
+"FR","35.0","22","244"
+"NL","24.0","127","265"
+"GB","13.0","1","42"
+"CA","12.0","1","209"
+"DK","8.0","4","8"
+"ES","7.0","1","50"
+"CZ","6.0","0","32"
+"JP","5.0","0","100"
+"IT","4.0","0","18"
+"RO","3.0","73","116"
+"IE","3.0","1","3"
+"RU","3.0","0","1"
+"AT","3.0","0","61"
+"IN","3.0","0","0"
+"SE","3.0","0","0"
+"IL","3.0","0","19"
+"CH","3.0","0","4"
+"CN","2.0","0","34"
+"PL","2.0","0","0"
+"CK","2.0","0","0"
+"IO","1.0","0","0"
+"SG","1.0","3","10"
+"FI","1.0","0","29"
+"UA","1.0","0","0"
+"IR","1.0","0","1"
+"ZA","1.0","0","0"
+"MX","1.0","1","21"
+"PT","1.0","0","0"
+"GR","1.0","1","3"
+"PF","1.0","0","0"
+"ME","1.0","0","0"
+"BG","1.0","2","2"
Index: /other-projects/maori-lang-detection/mongodb-data/1table_allCrawledSites.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/1table_allCrawledSites.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/1table_allCrawledSites.csv	(revision 33848)
@@ -0,0 +1,42 @@
+﻿"_id","count","numPagesInMRICount","numPagesContainingMRICount"
+"US","696.0","3305","9324"
+"UNKNOWN","173.0","115","137"
+"CN","125.0","281","1178"
+"NZ","115.0","2653","6226"
+"FR","69.0","776","1335"
+"DE","52.0","146","420"
+"AU","43.0","183","430"
+"NL","32.0","203","380"
+"CA","19.0","30","328"
+"GB","18.0","4","49"
+"DK","10.0","4","8"
+"JP","10.0","1","103"
+"ES","8.0","5","54"
+"CZ","7.0","0","32"
+"RU","7.0","0","15"
+"HK","7.0","3","12"
+"UA","5.0","9","10"
+"IE","4.0","18","24"
+"IT","4.0","0","18"
+"SE","4.0","0","0"
+"RO","3.0","73","116"
+"SG","3.0","5","23"
+"AT","3.0","0","61"
+"IN","3.0","0","0"
+"IL","3.0","0","19"
+"CH","3.0","0","4"
+"PL","2.0","0","0"
+"ZA","2.0","0","0"
+"VG","2.0","0","0"
+"CK","2.0","0","0"
+"IO","1.0","0","0"
+"FI","1.0","0","29"
+"IR","1.0","0","1"
+"TR","1.0","0","2"
+"EU","1.0","0","7"
+"PT","1.0","0","0"
+"MX","1.0","1","21"
+"GR","1.0","1","3"
+"PF","1.0","0","0"
+"ME","1.0","0","0"
+"BG","1.0","2","2"
Index: /other-projects/maori-lang-detection/mongodb-data/2table__sitesWithPagesInMRI.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/2table__sitesWithPagesInMRI.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/2table__sitesWithPagesInMRI.csv	(revision 33848)
@@ -0,0 +1,22 @@
+﻿"_id","count","numPagesInMRICount","numPagesContainingMRICount"
+"us","206.0","3305","6327"
+"nz","53.0","2653","5045"
+"cn","32.0","281","542"
+"fr","18.0","776","1101"
+"au","11.0","183","358"
+"nl","10.0","203","216"
+"de","5.0","146","190"
+"dk","4.0","4","4"
+"gb","3.0","4","13"
+"ca","3.0","30","35"
+"unknown","2.0","115","125"
+"ie","2.0","18","24"
+"ua","2.0","9","10"
+"es","2.0","5","5"
+"sg","2.0","5","23"
+"jp","1.0","1","3"
+"hk","1.0","3","8"
+"ro","1.0","73","104"
+"bg","1.0","2","2"
+"gr","1.0","1","3"
+"mx","1.0","1","21"
Index: /other-projects/maori-lang-detection/mongodb-data/3table_sitesWithPagesContainingMRI.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/3table_sitesWithPagesContainingMRI.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/3table_sitesWithPagesContainingMRI.csv	(revision 33848)
@@ -0,0 +1,32 @@
+﻿"_id","count","numPagesInMRICount","numPagesContainingMRICount"
+"us","486.0","3305","9324"
+"cn","114.0","281","1178"
+"nz","89.0","2653","6226"
+"fr","36.0","776","1335"
+"de","27.0","146","420"
+"nl","22.0","203","380"
+"au","21.0","183","430"
+"ca","12.0","30","328"
+"dk","8.0","4","8"
+"gb","7.0","4","49"
+"es","7.0","5","54"
+"cz","4.0","0","32"
+"unknown","3.0","115","137"
+"ro","3.0","73","116"
+"it","3.0","0","18"
+"at","3.0","0","61"
+"il","2.0","0","19"
+"ua","2.0","9","10"
+"ru","2.0","0","15"
+"ch","2.0","0","4"
+"hk","2.0","3","12"
+"sg","2.0","5","23"
+"ie","2.0","18","24"
+"jp","2.0","1","103"
+"tr","1.0","0","2"
+"mx","1.0","1","21"
+"fi","1.0","0","29"
+"eu","1.0","0","7"
+"gr","1.0","1","3"
+"bg","1.0","2","2"
+"ir","1.0","0","1"
Index: /other-projects/maori-lang-detection/mongodb-data/4table_tentativeNonProductSites.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/4table_tentativeNonProductSites.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/4table_tentativeNonProductSites.csv	(revision 33848)
@@ -0,0 +1,28 @@
+﻿"_id","siteCount","numPagesInMRICount","numPagesContainingMRICount"
+"us","181.0","2212","5579"
+"nz","89.0","2653","6226"
+"au","21.0","183","430"
+"de","19.0","1","208"
+"fr","17.0","22","244"
+"nl","16.0","127","265"
+"dk","8.0","4","8"
+"ca","7.0","1","209"
+"es","6.0","1","50"
+"gb","5.0","1","42"
+"cz","4.0","0","32"
+"at","3.0","0","61"
+"it","3.0","0","18"
+"ro","3.0","73","116"
+"ch","2.0","0","4"
+"il","2.0","0","19"
+"ru","1.0","0","1"
+"jp","1.0","0","100"
+"unknown","1.0","0","12"
+"ie","1.0","1","3"
+"fi","1.0","0","29"
+"sg","1.0","3","10"
+"bg","1.0","2","2"
+"cn","1.0","0","34"
+"gr","1.0","1","3"
+"ir","1.0","0","1"
+"mx","1.0","1","21"
Index: /other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv	(revision 33848)
@@ -0,0 +1,26 @@
+﻿"_id","siteCount","numPagesInMRICount","numPagesContainingMRICount"
+"nz","176.0","4360","9641"
+"us","117.0","757","2655"
+"de","19.0","1","208"
+"nl","16.0","127","265"
+"fr","16.0","22","243"
+"dk","8.0","4","8"
+"ca","7.0","1","209"
+"au","5.0","8","102"
+"cz","4.0","0","32"
+"gb","4.0","1","40"
+"es","4.0","1","7"
+"it","3.0","0","18"
+"at","3.0","0","61"
+"ro","2.0","0","12"
+"il","2.0","0","19"
+"ch","2.0","0","4"
+"ir","1.0","0","1"
+"fi","1.0","0","29"
+"ie","1.0","1","3"
+"ru","1.0","0","1"
+"jp","1.0","0","100"
+"mx","1.0","1","21"
+"unknown","1.0","0","12"
+"bg","1.0","2","2"
+"cn","1.0","0","34"
Index: /other-projects/maori-lang-detection/mongodb-data/6a_counts_manualShortlist_numPagesInMRI.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6a_counts_manualShortlist_numPagesInMRI.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6a_counts_manualShortlist_numPagesInMRI.json	(revision 33848)
@@ -0,0 +1,43 @@
+/*
+Based on the values in 6table_nonProductSites_manualShortlist.json
+
+Country codes column by numPagesInMRI counts.
+
+*/
+{
+    "_id" : "nz",
+    "count" : 4360.0
+}
+{
+    "_id" : "us",
+    "count" : 681.0
+}
+{
+    "_id" : "au",
+    "count" : 8.0
+}
+{
+    "_id" : "de",
+    "count" : 4.0
+}
+{
+    "_id" : "dk",
+    "count" : 4.0
+}
+{
+    "_id" : "bg",
+    "count" : 2.0
+}
+{
+    "_id" : "es",
+    "count" : 1.0
+}
+{
+    "_id" : "fr",
+    "count" : 1.0
+}
+{
+    "_id" : "ie",
+    "count" : 1.0
+}
+
Index: /other-projects/maori-lang-detection/mongodb-data/6a_geojson-features_manualShortlist_numPagesInMRI.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6a_geojson-features_manualShortlist_numPagesInMRI.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6a_geojson-features_manualShortlist_numPagesInMRI.json	(revision 33848)
@@ -0,0 +1,1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[46.88597100000001,-40.900557],[46.88597100000001,27.224443],[302.88597100000004,27.224443],[302.88597100000004,-40.900557],[46.88597100000001,-40.900557]]]},"properties":{"code":"NZ","count":4360,"region":"New Zealand"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-127.712891,37.09024],[-127.712891,79.65274],[-63.712891,79.65274],[-63.712891,37.09024],[-127.712891,37.09024]]]},"properties":{"code":"US","count":681,"region":"United States"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[131.775136,-25.274398],[131.775136,-17.274398],[135.775136,-17.274398],[135.775136,-25.274398],[131.775136,-25.274398]]]},"properties":{"code":"AU","count":8,"region":"Australia"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8.451526,51.165691],[8.451526,55.165691],[12.451526,55.165691],[12.451526,51.165691],[8.451526,51.165691]]]},"properties":{"code":"DE","count":4,"region":"Germany"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.501785,56.26392],[7.501785,60.26392],[11.501785,60.26392],[11.501785,56.26392],[7.501785,56.26392]]]},"properties":{"code":"DK","count":4,"region":"Denmark"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[23.48583,42.733883],[23.48583,44.733883],[27.48583,44.733883],[27.48583,42.733883],[23.48583,42.733883]]]},"properties":{"code":"BG","count":2,"region":"Bulgaria"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-5.74922,40.463667],[-5.74922,41.463667],[-1.7492200000000002,41.463667],[-1.7492200000000002,40.463667],[-5.74922,40.463667]]]},"properties":{"code":"ES","count":1,"region":"Spain"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.21374899999999997,46.227638],[0.21374899999999997,47.227638],[4.213749,47.227638],[4.213749,46.227638],[0.21374899999999997,46.227638]]]},"properties":{"code":"FR","count":1,"region":"France"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-10.24389,53.41291],[-10.24389,54.41291],[-6.24389,54.41291],[-6.24389,53.41291],[-10.24389,53.41291]]]},"properties":{"code":"IE","count":1,"region":"Ireland"}}]}
Index: /other-projects/maori-lang-detection/mongodb-data/6a_multipoint_manualShortlist_numPagesInMRI.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6a_multipoint_manualShortlist_numPagesInMRI.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6a_multipoint_manualShortlist_numPagesInMRI.json	(revision 33848)
@@ -0,0 +1,12 @@
+{"type":"MultiPoint","coordinates":
+[
+	[174.885971,-40.900557],
+	[-95.712891,37.09024],
+	[133.775136,-25.274398],
+	[10.451526,51.165691],
+	[9.501785,56.26392],
+	[25.48583,42.733883],
+	[-3.74922,40.463667],
+	[2.213749,46.227638],
+	[-8.24389,53.41291]
+]}
Index: /other-projects/maori-lang-detection/mongodb-data/6b_counts_manualShortlist_numPagesContainingMRI.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6b_counts_manualShortlist_numPagesContainingMRI.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6b_counts_manualShortlist_numPagesContainingMRI.json	(revision 33848)
@@ -0,0 +1,46 @@
+/*
+Based on the values in 6table_nonProductSites_manualShortlist.json
+
+Country codes column by numPagesContainingMRI counts.
+*/
+{
+    "_id" : "nz",
+    "count" : 9641.0
+}
+{
+    "_id" : "us",
+    "count" : 953.0
+}
+{
+    "_id" : "au",
+    "count" : 86.0
+}
+{
+    "_id" : "de",
+    "count" : 11.0
+}
+{
+    "_id" : "dk",
+    "count" : 7.0
+}
+{
+    "_id" : "bg",
+    "count" : 2.0
+}
+{
+    "_id" : "cz",
+    "count" : 1.0
+}
+{
+    "_id" : "es",
+    "count" : 1.0
+}
+{
+    "_id" : "fr",
+    "count" : 1.0
+}
+{
+    "_id" : "ie",
+    "count" : 3.0
+}
+
Index: /other-projects/maori-lang-detection/mongodb-data/6b_geojson-features_manualShortlist_numPagesContainingMRI.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6b_geojson-features_manualShortlist_numPagesContainingMRI.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6b_geojson-features_manualShortlist_numPagesContainingMRI.json	(revision 33848)
@@ -0,0 +1,1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-81.11402899999999,-40.900557],[-81.11402899999999,34.4197555],[430.88597100000004,34.4197555],[430.88597100000004,-40.900557],[-81.11402899999999,-40.900557]]]},"properties":{"code":"NZ","count":9641,"region":"New Zealand"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-159.712891,37.09024],[-159.712891,66.87149],[-31.712891,66.87149],[-31.712891,37.09024],[-159.712891,37.09024]]]},"properties":{"code":"US","count":953,"region":"United States"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[131.775136,-25.274398],[131.775136,60.725601999999995],[135.775136,60.725601999999995],[135.775136,-25.274398],[131.775136,-25.274398]]]},"properties":{"code":"AU","count":86,"region":"Australia"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8.451526,51.165691],[8.451526,62.165691],[12.451526,62.165691],[12.451526,51.165691],[8.451526,51.165691]]]},"properties":{"code":"DE","count":11,"region":"Germany"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.501785,56.26392],[7.501785,63.26392],[11.501785,63.26392],[11.501785,56.26392],[7.501785,56.26392]]]},"properties":{"code":"DK","count":7,"region":"Denmark"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[23.48583,42.733883],[23.48583,44.733883],[27.48583,44.733883],[27.48583,42.733883],[23.48583,42.733883]]]},"properties":{"code":"BG","count":2,"region":"Bulgaria"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[13.472962,49.817492],[13.472962,50.817492],[17.472962000000003,50.817492],[17.472962000000003,49.817492],[13.472962,49.817492]]]},"properties":{"code":"CZ","count":1,"region":"Czech Republic"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-5.74922,40.463667],[-5.74922,41.463667],[-1.7492200000000002,41.463667],[-1.7492200000000002,40.463667],[-5.74922,40.463667]]]},"properties":{"code":"ES","count":1,"region":"Spain"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.21374899999999997,46.227638],[0.21374899999999997,47.227638],[4.213749,47.227638],[4.213749,46.227638],[0.21374899999999997,46.227638]]]},"properties":{"code":"FR","count":1,"region":"France"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-10.24389,53.41291],[-10.24389,56.41291],[-6.24389,56.41291],[-6.24389,53.41291],[-10.24389,53.41291]]]},"properties":{"code":"IE","count":3,"region":"Ireland"}}]}
Index: /other-projects/maori-lang-detection/mongodb-data/6b_multipoint_manualShortlist_numPagesContainingMRI.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6b_multipoint_manualShortlist_numPagesContainingMRI.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6b_multipoint_manualShortlist_numPagesContainingMRI.json	(revision 33848)
@@ -0,0 +1,13 @@
+{"type":"MultiPoint","coordinates":
+[
+	[174.885971,-40.900557],
+	[-95.712891,37.09024],
+	[133.775136,-25.274398],
+	[10.451526,51.165691],
+	[9.501785,56.26392],
+	[25.48583,42.733883],
+	[15.472962,49.817492],
+	[-3.74922,40.463667],
+	[2.213749,46.227638],
+	[-8.24389,53.41291]
+]}
Index: /other-projects/maori-lang-detection/mongodb-data/6counts_nonProductSites1_manualShortlist.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6counts_nonProductSites1_manualShortlist.json	(revision 33847)
+++ /other-projects/maori-lang-detection/mongodb-data/6counts_nonProductSites1_manualShortlist.json	(revision 33848)
@@ -16,5 +16,5 @@
 {
     "_id" : "au",
-    "count" : 3.0
+    "count" : 2.0
 }
 {
Index: /other-projects/maori-lang-detection/mongodb-data/6geojson-features_nonProductSites1_manualShortlist.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6geojson-features_nonProductSites1_manualShortlist.json	(revision 33847)
+++ /other-projects/maori-lang-detection/mongodb-data/6geojson-features_nonProductSites1_manualShortlist.json	(revision 33848)
@@ -1,1 +1,1 @@
-{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[170.885971,-40.900557],[170.885971,47.099443],[178.885971,47.099443],[178.885971,-40.900557],[170.885971,-40.900557]]]},"properties":{"code":"NZ","count":176,"region":"New Zealand"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-97.712891,37.09024],[-97.712891,66.09024],[-93.712891,66.09024],[-93.712891,37.09024],[-97.712891,37.09024]]]},"properties":{"code":"US","count":29,"region":"United States"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[131.775136,-25.274398],[131.775136,-22.274398],[135.775136,-22.274398],[135.775136,-25.274398],[131.775136,-25.274398]]]},"properties":{"code":"AU","count":3,"region":"Australia"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8.451526,51.165691],[8.451526,53.165691],[12.451526,53.165691],[12.451526,51.165691],[8.451526,51.165691]]]},"properties":{"code":"DE","count":2,"region":"Germany"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.501785,56.26392],[7.501785,58.26392],[11.501785,58.26392],[11.501785,56.26392],[7.501785,56.26392]]]},"properties":{"code":"DK","count":2,"region":"Denmark"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[23.48583,42.733883],[23.48583,43.733883],[27.48583,43.733883],[27.48583,42.733883],[23.48583,42.733883]]]},"properties":{"code":"BG","count":1,"region":"Bulgaria"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[13.472962,49.817492],[13.472962,50.817492],[17.472962000000003,50.817492],[17.472962000000003,49.817492],[13.472962,49.817492]]]},"properties":{"code":"CZ","count":1,"region":"Czech Republic"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-5.74922,40.463667],[-5.74922,41.463667],[-1.7492200000000002,41.463667],[-1.7492200000000002,40.463667],[-5.74922,40.463667]]]},"properties":{"code":"ES","count":1,"region":"Spain"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.21374899999999997,46.227638],[0.21374899999999997,47.227638],[4.213749,47.227638],[4.213749,46.227638],[0.21374899999999997,46.227638]]]},"properties":{"code":"FR","count":1,"region":"France"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-10.24389,53.41291],[-10.24389,54.41291],[-6.24389,54.41291],[-6.24389,53.41291],[-10.24389,53.41291]]]},"properties":{"code":"IE","count":1,"region":"Ireland"}}]}
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[170.885971,-40.900557],[170.885971,47.099443],[178.885971,47.099443],[178.885971,-40.900557],[170.885971,-40.900557]]]},"properties":{"code":"NZ","count":176,"region":"New Zealand"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-97.712891,37.09024],[-97.712891,66.09024],[-93.712891,66.09024],[-93.712891,37.09024],[-97.712891,37.09024]]]},"properties":{"code":"US","count":29,"region":"United States"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[131.775136,-25.274398],[131.775136,-23.274398],[135.775136,-23.274398],[135.775136,-25.274398],[131.775136,-25.274398]]]},"properties":{"code":"AU","count":2,"region":"Australia"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8.451526,51.165691],[8.451526,53.165691],[12.451526,53.165691],[12.451526,51.165691],[8.451526,51.165691]]]},"properties":{"code":"DE","count":2,"region":"Germany"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.501785,56.26392],[7.501785,58.26392],[11.501785,58.26392],[11.501785,56.26392],[7.501785,56.26392]]]},"properties":{"code":"DK","count":2,"region":"Denmark"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[23.48583,42.733883],[23.48583,43.733883],[27.48583,43.733883],[27.48583,42.733883],[23.48583,42.733883]]]},"properties":{"code":"BG","count":1,"region":"Bulgaria"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[13.472962,49.817492],[13.472962,50.817492],[17.472962000000003,50.817492],[17.472962000000003,49.817492],[13.472962,49.817492]]]},"properties":{"code":"CZ","count":1,"region":"Czech Republic"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-5.74922,40.463667],[-5.74922,41.463667],[-1.7492200000000002,41.463667],[-1.7492200000000002,40.463667],[-5.74922,40.463667]]]},"properties":{"code":"ES","count":1,"region":"Spain"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.21374899999999997,46.227638],[0.21374899999999997,47.227638],[4.213749,47.227638],[4.213749,46.227638],[0.21374899999999997,46.227638]]]},"properties":{"code":"FR","count":1,"region":"France"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-10.24389,53.41291],[-10.24389,54.41291],[-6.24389,54.41291],[-6.24389,53.41291],[-10.24389,53.41291]]]},"properties":{"code":"IE","count":1,"region":"Ireland"}}]}
Index: /other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json	(revision 33848)
@@ -0,0 +1,68 @@
+/*
+For sites originating in NZ or with nz TLD, none of the URLs are manually inspected and all URLs are accepted.
+
+For all but NZ, get final column results with:
+	db.getCollection('Websites').find({domain:/coggle\.it/})
+And can check for URLs with:
+	db.getCollection('Webpages').find({URL: /coggle\.it/, isMRI: true})
+
+
+NOTES:
+1. DE:
+
+"de","2.0","0+1","9+35 misdetected", http://www.cartogiraffe.com, https://www.cartogiraffe.com, 
+Ought to be 2+2 numPagesInMRICount and 9+2 numPagesContainingMRICount:
+- both cartogiraffe.com pages were identical and had mostly MRI sentences with one name not being MRI. So isMRI should have been true for both pages.
+- Only one of the 2 MRI translations of the universal declaration of human rights at http://www.udhr.de got downloaded. A total of 75 pages were downloaded, but more translated pages appeared to be on the webpage. Not sure why the crawl had a _SUCCESS file to indicate completed download.
+- Then http://www.udhr.de had 35-1 non-MRI language translations of the universal declaration of human rights where one or more sentences were misdetected as MRI. With the additional MRI page that didn't get downloaded, should have 9+2 = 11 pages containing MRI.
+
+So instead of 
+"de","2.0","1","44", http://www.cartogiraffe.com, https://www.cartogiraffe.com, http://www.udhr.de
+"de","2.0","4","11", http://www.cartogiraffe.com, https://www.cartogiraffe.com, http://www.udhr.de
+
+
+"au","3.0",7+0+1,83+1+3,https://www.kiwiproperty.com, https://infogram.com/te-marautanga-o-aotearoa-moe-pld-allocations-2012-1go502ygvn562jd,https://koreromaori.com
+
+2. US:
+aclhokiangarocks.blogspot.com contains at least a page with MRI paragraphs. See http://aclhokiangarocks.blogspot.com/feeds/posts/default under section "Nga Tuhinga o tatou Tupuna"
+Although this page has been crawled by Nutch, the contents were presented in the blog in a complex way and therefore the text wasn't retrieved here. See also the dedicated page this text should have been in http://aclhokiangarocks.blogspot.com/2012/05/nga-tuhinga-o-tatou-tupuna.html
+
+ï»¿"_id","siteCount","numPagesInMRICount","numPagesContainingMRICount","URLs of pages detected as inMRI"
+"nz","176.0","4360","9641"
+"us","29.0",
+	1+2+0+0+4+166+0+39 +257+2+21+12+25+13+53+0+1+0+1+11 +32+37+4 +0+0+0 = 681,
+	31+2+2+20+58+166+3+91 +258+2+25+12+66+22+53+6+1+1+2+10 +58+54+6 +1+2+1 = 953,
+	anglicanhistory.org,unicode.org,static-promote.weebly.com,aclhokiangarocks.blogspot.com,bahaiprayers.net,biblehub.com,muhammad.com,godrules.net,m.biblepub.com, krassotkin.ru,gotquestions.org, 
+      maorinews.com,maaori.com,kiaorahola.blogspot.com,kjohnsonnz.blogspot.com,pumanawawhangara.blogspot.com,dannykahei.tripod.com,burkekm001.tripod.com,tkkpipipaopao.blogspot.com, manateina.blogspot.com,
+	tatai09.blogspot.com,twttoa.com,tuhua2010.blogspot.com,
+	breaker.audio,drive.google.com/file/d/1NwuzafjddaP8gxI7O_Zapts5bM7mrtwn/preview,in.pinterest.com/pin/317363104978423418/
+"au","2.0","8","86", https://www.kiwiproperty.com, https://koreromaori.com
+"de","2.0","4","11", http://www.cartogiraffe.com, https://www.cartogiraffe.com, http://www.udhr.de
+"dk","2.0","4","7", *.ngapuhitelevision.com, *.ngapuhiradio.com
+"bg","1.0","2","2", http://anitra.net/activism/humanrights/UDHR/mbf_print.htm, http://anitra.net/activism/humanrights/UDHR/rrt_print.htm
+"cz","1.0","0","1", http://www.henryklahola.nazory.cz/094.Maori.htm, http://henryklahola.nazory.cz/094.Maori.htm
+"es","1.0","1","1", https://www.uv.es/~pla/red.net/intmaori.html
+"fr","1.0","1","1", http://chantsdeluttes.free.fr/versionsinter/page%20maori.html
+"ie","1.0","1","3", https://coggle.it/diagram/WSYB0mLA2QABD5BH/t/ko-au-ko-koe
+
+*/
+
+
+
+ï»¿"_id","siteCount","numPagesInMRICount","numPagesContainingMRICount","URLs of pages detected as inMRI"
+"nz","176.0","4360","9641"
+"us","29.0","681","953"
+"au","2.0","8","86"
+"de","2.0","4","11"
+"dk","2.0","4","7"
+"bg","1.0","2","2"
+"cz","1.0","0","1"
+"es","1.0","1","1"
+"fr","1.0","1","1"
+"ie","1.0","1","3"
+
+Total sites containing MRI: 216
+Total pages detected as being in MRI: 5062
+Total pages detected as containing MRI sentences: 10706
+
+
Index: /other-projects/maori-lang-detection/mongodb-data/tables.txt
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/tables.txt	(revision 33848)
+++ /other-projects/maori-lang-detection/mongodb-data/tables.txt	(revision 33848)
@@ -0,0 +1,180 @@
+Instructions for producing the tables:
+a. Copy the Javascript version of results for each mongodb query listed below into a text editor.
+b. Then regex replace \/\*\s*\d+\s*\*\/ with "," and embed all the JS inside [].
+c. Paste that Javascript into https://json-csv.com/ to get the CSV tables
+
+Note that for step 6, there are 2 mongodb queries. The results of both have to be merged into a single csv file.
+-----------
+1. Table 1
+
+db.Websites.aggregate([
+   
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: "$geoLocationCountryCode",
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+
+1a.
+
+db.Websites.aggregate([
+    { $match: {urlContainsLangCodeInPath: true} },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: "$geoLocationCountryCode",
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+
+1b.
+
+db.Websites.aggregate([
+   {$match: {urlContainsLangCodeInPath: false} },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: "$geoLocationCountryCode",
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+-----------
+2. Table 2
+
+db.Websites.aggregate([
+    {
+        $match: {
+            numPagesInMRI: {$gt: 0}
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+-----------
+3. Table 3
+
+db.Websites.aggregate([
+    {
+        $match: {
+            numPagesContainingMRI: {$gt: 0}
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+-----------
+
+4. Table 4
+db.Websites.aggregate([
+    {
+        $match: {
+            $and: [
+                {numPagesContainingMRI: {$gt: 0}},
+                {$or: [{geoLocationCountryCode: /(NZ|AU)/}, {domain: /\.nz$/}, {urlContainsLangCodeInPath: false}]}
+            ]
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 },
+             /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+-----------
+5. Table 5
+
+Outside of NZ:
+
+db.Websites.aggregate([
+    {
+        $match: {
+            $and: [
+                {geoLocationCountryCode: {$ne: "NZ"}},
+                {domain: {$not: /\.nz/}},
+                {numPagesContainingMRI: {$gt: 0}},
+                {$or: [{geoLocationCountryCode: "AU"}, {urlContainsLangCodeInPath: false}]}            
+            ]
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+NZ:
+db.Websites.aggregate([
+    {
+        $match: {
+            $and: [
+                {numPagesContainingMRI: {$gt: 0}},
+                {$or: [{geoLocationCountryCode:"NZ"},{domain: /\.nz/}]}
+            ]
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: "nz",
+            count: { $sum: 1 },
+            /*domain: { $addToSet: '$domain' },*/
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
