Changeset 33804

Timestamp:

2019-12-13T20:00:53+13:00 (4 years ago)

Author:

ak19

Message:

1. Updated results from mongodb querying after yesterday's modifications to NutchTextDumpToMongoDB.java. Mods included removing another adult site (moved to blacklist) and replacing country codes of sites whose origin countries couldn't be detected with its TLD suffix else UNKNOWN instead of the previous null. Adding extra field to Websites mongodb collection called numPagesContainingMRI which counts the number of webpages in a website which has at least one sentence for which OpenNLP predicted its best detectable language as MRI 2. Added new Mongodb queries to work with the numPagesContainingMRI field.

File:

• other-projects/maori-lang-detection/MoreReading/mongodb.txt (modified) (4 diffs)

other-projects/maori-lang-detection/MoreReading/mongodb.txt

@@ -454 +454 @@
 # Num websites
 db.getCollection('Websites').find({}).count()
-1446 
+1445 
 
 # Num webpages
@@ -464 +464 @@
 db.getCollection('Websites').find({numPagesInMRI: { $gt: 0}}).count()
 361
+
+# Number of sites containing at least one sentence for which OpenNLP detected the best language = MRI
+db.getCollection('Websites').find({numPagesContainingMRI: {$gt: 0}}).count()
+868
+
+# Obviously, the union of the above two will be identical to numPagesContainingMRI:
+db.getCollection('Websites').find({ $or: [ { numPagesInMRI: { $gt: 0 } }, { numPagesContainingMRI: {$gt: 0} } ] } ).count()
+868
 
 # Find number of webpages that are deemed to be overall in MRI (pages where isMRI=true)
@@ -484 +492 @@
 # Number of websites that are outside NZ that contain /mi(/) in any of its sub-urls
 db.getCollection('Websites').find({urlContainsLangCodeInPath:true, geoLocationCountryCode: {$ne : "NZ"} }).count()
-148
+147
 
 # 5 sites with URLs containing /mi(/) that are in NZ
 db.getCollection('Websites').find({urlContainsLangCodeInPath:true, geoLocationCountryCode: "NZ"}).count()
-5
+6
+
 
 # sort websites that contain /mi(/) in path by geoLocationCountryCode
@@ -557 +566 @@
         $group: {
             _id: "$geoLocationCountryCode",
+            count: { $sum: 1 }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+// count of country codes for sites that have at least one page detected as MRI 
+
+db.Websites.aggregate([
+    {
+        $match: {
+            numPagesInMRI: {$gt: 0}
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+// count of country codes for sites that have at least one page containing at least one sentence detected as MRI 
+db.Websites.aggregate([
+    {
+        $match: {
+            numPagesContainingMRI: {$gt: 0}
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
             count: { $sum: 1 }
         }