Changeset 33847

Timestamp:

2020-01-17T19:32:16+13:00 (4 years ago)

Author:

ak19

Message:

indigenousblogs.com did have one page actually in Maori (an XML feed). So adding 1 to the table of counts for US sites with mi in the URL path that contained actual MRI.

Location:

other-projects/maori-lang-detection

Files:

• MoreReading/mongodb.txt (modified) (4 diffs)
• mongodb-data/6counts_nonProductSites1_manualShortlist.json (modified) (1 diff)
• mongodb-data/6geojson-features_nonProductSites1_manualShortlist.json (modified) (1 diff)
• mongodb-data/6map_exclAutotranslatedSites1_manualShortlist.png (modified) ( previous)

other-projects/maori-lang-detection/MoreReading/mongodb.txt

@@ -1488 +1488 @@
 TIDIED:
 NZ: 176
-US: 25+3 from US with mi in URL path = 28
+US: 25+4 from US with mi in URL path = 29
 AU: 3
 DE: 2
@@ -1497 +1497 @@
 FR: 1
 IE: 1
-TOTAL: 213+3 from US with mi in URL path = 216
+TOTAL: 213+4 from US with mi in URL path = 217
 
 
@@ -1525 +1525 @@
 Of interest or possible interest:
 US: 
-!! http://indigenousblogs.com [15/18 blogs work]
+!! http://indigenousblogs.com [15/18 blogs work] - has one page in Maori (http://indigenousblogs.com/feeds/mi.xml)
 X https://biblia.gospelprime.com.br - misdetection (containsMRI)
 X ?https://follow3rs.com - seems dodgy and possibly auto-translated. Can't spell account, misspelled as accout
@@ -1559 +1559 @@
 db.getCollection('Webpages').find({$and: [{isMRI: true}, {URL: /indigenousblogs\.com/}]})
 => http://indigenousblogs.com/mi/
+
+--------------------------
+
+
+db.Websites.aggregate([
+    {
+        $match: {
+            $and: [
+                {geoLocationCountryCode: {$ne: "NZ"}},
+                {domain: {$not: /\.nz/}},
+                {numPagesContainingMRI: {$gt: 0}},
+                {$or: [{geoLocationCountryCode: "AU"}, {urlContainsLangCodeInPath: false}]}            
+            ]
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 },
+            domain: { $addToSet: '$domain' },
+            numPagesInMRI: { $addToSet: '$numPagesInMRI' },
+            numPagesContainingMRI: { $addToSet: '$numPagesContainingMRI' },
+            numPagesInMRICount: { $sum: '$numPagesInMRI' },
+            numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+
+To convert json to csv
+In gedit replace
+\/\*\s*\d+\s*\*\/ => ,

other-projects/maori-lang-detection/mongodb-data/6counts_nonProductSites1_manualShortlist.json

@@ -12 +12 @@
 {
     "_id" : "us",
-    "count" : 28.0
+    "count" : 29.0
 }
 {

other-projects/maori-lang-detection/mongodb-data/6geojson-features_nonProductSites1_manualShortlist.json

@@ -1 @@
-{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[170.885971,-40.900557],[170.885971,47.099443],[178.885971,47.099443],[178.885971,-40.900557],[170.885971,-40.900557]]]},"properties":{"code":"NZ","count":176,"region":"New Zealand"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-97.712891,37.09024],[-97.712891,65.09024],[-93.712891,65.09024],[-93.712891,37.09024],[-97.712891,37.09024]]]},"properties":{"code":"US","count":28,"region":"United States"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[131.775136,-25.274398],[131.775136,-22.274398],[135.775136,-22.274398],[135.775136,-25.274398],[131.775136,-25.274398]]]},"properties":{"code":"AU","count":3,"region":"Australia"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8.451526,51.165691],[8.451526,53.165691],[12.451526,53.165691],[12.451526,51.165691],[8.451526,51.165691]]]},"properties":{"code":"DE","count":2,"region":"Germany"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.501785,56.26392],[7.501785,58.26392],[11.501785,58.26392],[11.501785,56.26392],[7.501785,56.26392]]]},"properties":{"code":"DK","count":2,"region":"Denmark"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[23.48583,42.733883],[23.48583,43.733883],[27.48583,43.733883],[27.48583,42.733883],[23.48583,42.733883]]]},"properties":{"code":"BG","count":1,"region":"Bulgaria"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[13.472962,49.817492],[13.472962,50.817492],[17.472962000000003,50.817492],[17.472962000000003,49.817492],[13.472962,49.817492]]]},"properties":{"code":"CZ","count":1,"region":"Czech Republic"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-5.74922,40.463667],[-5.74922,41.463667],[-1.7492200000000002,41.463667],[-1.7492200000000002,40.463667],[-5.74922,40.463667]]]},"properties":{"code":"ES","count":1,"region":"Spain"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.21374899999999997,46.227638],[0.21374899999999997,47.227638],[4.213749,47.227638],[4.213749,46.227638],[0.21374899999999997,46.227638]]]},"properties":{"code":"FR","count":1,"region":"France"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-10.24389,53.41291],[-10.24389,54.41291],[-6.24389,54.41291],[-6.24389,53.41291],[-10.24389,53.41291]]]},"properties":{"code":"IE","count":1,"region":"Ireland"}}]}
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[170.885971,-40.900557],[170.885971,47.099443],[178.885971,47.099443],[178.885971,-40.900557],[170.885971,-40.900557]]]},"properties":{"code":"NZ","count":176,"region":"New Zealand"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-97.712891,37.09024],[-97.712891,66.09024],[-93.712891,66.09024],[-93.712891,37.09024],[-97.712891,37.09024]]]},"properties":{"code":"US","count":29,"region":"United States"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[131.775136,-25.274398],[131.775136,-22.274398],[135.775136,-22.274398],[135.775136,-25.274398],[131.775136,-25.274398]]]},"properties":{"code":"AU","count":3,"region":"Australia"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8.451526,51.165691],[8.451526,53.165691],[12.451526,53.165691],[12.451526,51.165691],[8.451526,51.165691]]]},"properties":{"code":"DE","count":2,"region":"Germany"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.501785,56.26392],[7.501785,58.26392],[11.501785,58.26392],[11.501785,56.26392],[7.501785,56.26392]]]},"properties":{"code":"DK","count":2,"region":"Denmark"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[23.48583,42.733883],[23.48583,43.733883],[27.48583,43.733883],[27.48583,42.733883],[23.48583,42.733883]]]},"properties":{"code":"BG","count":1,"region":"Bulgaria"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[13.472962,49.817492],[13.472962,50.817492],[17.472962000000003,50.817492],[17.472962000000003,49.817492],[13.472962,49.817492]]]},"properties":{"code":"CZ","count":1,"region":"Czech Republic"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-5.74922,40.463667],[-5.74922,41.463667],[-1.7492200000000002,41.463667],[-1.7492200000000002,40.463667],[-5.74922,40.463667]]]},"properties":{"code":"ES","count":1,"region":"Spain"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0.21374899999999997,46.227638],[0.21374899999999997,47.227638],[4.213749,47.227638],[4.213749,46.227638],[0.21374899999999997,46.227638]]]},"properties":{"code":"FR","count":1,"region":"France"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-10.24389,53.41291],[-10.24389,54.41291],[-6.24389,54.41291],[-6.24389,53.41291],[-10.24389,53.41291]]]},"properties":{"code":"IE","count":1,"region":"Ireland"}}]}