Changeset 33876 for other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

Timestamp:

2020-01-29T21:48:52+13:00 (4 years ago)

Author:

ak19

Message:

Some missteps, but have got complex collection.aggregate() working at last.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java (modified) (9 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -3 +3 @@
 //import org.bson.BSONObject;
 
+import com.mongodb.client.AggregateIterable;
 import com.mongodb.client.MongoCollection; 
 import com.mongodb.client.MongoDatabase; 
@@ -11 +12 @@
 // to use collection.find().projection() filters like include() etc
 import static com.mongodb.client.model.Projections.*;
+// to use aggregation functions like unwind(), match(), sort() etc
+import static com.mongodb.client.model.Aggregates.*;
+// to use functions like sum() and addToSet() within aggregation functions
+import static com.mongodb.client.model.Accumulators.*;
 
 //import org.bson.conversions.Bson;
@@ -22 +27 @@
 
 import org.bson.Document;
+import org.bson.conversions.Bson;
+    
+import com.mongodb.util.JSON;
+//import com.mongodb.DBObject;
 
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
+import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.List;
@@ -68 +78 @@
     static final String PROPS_FILENAME = "config.properties";
     public static final String WEBPAGES_COLLECTION = "Webpages";
-    public static final String WEBSITES_COLLECTION = "Websites";    
+    public static final String WEBSITES_COLLECTION = "Websites";
+    
+    /** mongodb filter types to execute */
+    public static final int IS_MRI = 0;
+    public static final int CONTAINS_MRI = 1;
 
     // configuration details, some with fallback values
@@ -264 +278 @@
     }
     */
-
+    
+    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
+    return queryAllMatchingURLsFilteredBy(domain, IS_MRI);
+    }
+    public ArrayList<String> queryAllMatchingcontainsMRIURLs(String domain) {
+    return queryAllMatchingURLsFilteredBy(domain, CONTAINS_MRI);
+    }
+    
     /** 
      * Java mongodb find: https://mongodb.github.io/mongo-java-driver/3.4/driver/getting-started/quick-start/ 
@@ -275 +296 @@
      * https://stackoverflow.com/questions/55029222/parse-mongodb-query-to-java
      * http://pingax.com/trick-convert-mongo-shell-query-equivalent-java-objects/
-*/
-    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
+     */
+    public ArrayList<String> queryAllMatchingURLsFilteredBy(String domain, int filterType) {
     
     final ArrayList<String> urlsList = new ArrayList<String>();
@@ -300 +321 @@
         };
 
-
     
     // Run the following mongodb query:
@@ -309 +329 @@
 
     // 2. Another way:
-    String query = "{URL: /DOMAIN/, isMRI: true}";
+    //String query = "{URL: /DOMAIN/, isMRI: true}";
+    String query = "{URL: /DOMAIN/, ";
+    if(filterType == IS_MRI) {
+        query += "isMRI: true}";
+    } else if(filterType == CONTAINS_MRI) {
+        query += "containsMRI: true}";
+    }
+    
     domain = domain.replace(".", "\\."); // escape dots in domain for regex
     query = query.replace("DOMAIN", domain);
@@ -324 +351 @@
     }
 
-    
+    /**
+
+       db.Websites.aggregate([
+    {
+        $match: {
+            $and: [
+                {geoLocationCountryCode: {$ne: "NZ"}},
+                {domain: {$not: /\.nz/}},
+                {numPagesContainingMRI: {$gt: 0}},
+                {$or: [{geoLocationCountryCode: "AU"}, {urlContainsLangCodeInPath: false}]}            
+            ]
+        }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+        $group: {
+            _id: {$toLower: '$geoLocationCountryCode'},
+            count: { $sum: 1 },
+            domain: { $addToSet: '$domain' }
+        }
+    },
+    { $sort : { count : -1} }
+]);
+
+       https://stackoverflow.com/questions/31643109/mongodb-aggregation-with-java-driver
+       https://stackoverflow.com/questions/48000891/parse-mongodb-json-query-in-java-with-multiple-criteria
+       Not Java: https://stackoverflow.com/questions/39060221/a-pipeline-stage-specification-object-must-contain-exactly-one-field-with-php-mo
+
+       (https://stackoverflow.com/questions/55029222/parse-mongodb-query-to-java)
+       https://www.programcreek.com/java-api-examples/?api=com.mongodb.client.model.Aggregates
+       On using group(TExpression) inside collection.aggregate().
+    */
+    public String aggregateContainsMRIForOverseas() {
+    // working with the WebSites collection, not WebPages collection!
+    MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
+
+    /*String matchQuery =
+    "$and: ["
+        + "{geoLocationCountryCode: {$ne: \"NZ\"}},"
+        + "{domain: {$not: /\\.nz/}},"
+        + "{numPagesContainingMRI: {$gt: 0}},"
+        + "{$or: [{geoLocationCountryCode: \"AU\"}, {urlContainsLangCodeInPath: false}]}"
+            + "]";*/
+
+
+    
+    
+    Bson orQuery = or(
+              BasicDBObject.parse("{geoLocationCountryCode: \"AU\"}"),
+              BasicDBObject.parse("{urlContainsLangCodeInPath: false}")
+              );
+    Bson andQuery = and(
+        BasicDBObject.parse("{geoLocationCountryCode: {$ne: \"NZ\"}}"),
+        BasicDBObject.parse("{domain: {$not: /\\.nz/}}"),
+        BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
+        orQuery);
+    
+    AggregateIterable<Document> output
+        = collection.aggregate(Arrays.asList(
+         match(andQuery),  //match(BasicDBObject.parse(matchQuery))
+         // match((List<DBObject>)JSON.parse(matchQuery)),
+         unwind("$geoLocationCountryCode"),
+         group("$geoLocationCountryCode", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
+         sort(BasicDBObject.parse("{count : -1}"))
+                         ));
+    
+    for (Document doc : output) {
+        //System.out.println(doc);
+        System.out.println(doc.toJson());
+    }
+
+    return "";
+    }
     
     /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */