Changeset 33881 for other-projects

Timestamp:

2020-01-30T22:08:00+13:00 (4 years ago)

Author:

ak19

Message:

Uses lambda expression to process each doc in a mongodb aggregate result. Hopefully doing so means it processes each result Document as the result is obtained, meaning increased efficiency, rather than still storing all the results and processing them, as that code is more legible.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java (modified) (8 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -42 +42 @@
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.io.Writer;
 
@@ -321 +322 @@
 
     // code we'll execute in Iterable.forEach() below
+    // see also https://www.baeldung.com/foreach-java
     Block<Document> storeURL = new Block<Document>() {
         @Override
@@ -362 +364 @@
     }
 
-    /**       
-       The mongodb aggregate() we want to run this time:
-
+    /**     
+     * RUNNING A MONGODB COLLECTION.AGGREGATE() in JAVA:
+     *
+     * https://stackoverflow.com/questions/31643109/mongodb-aggregation-with-java-driver
+     * https://stackoverflow.com/questions/48000891/parse-mongodb-json-query-in-java-with-multiple-criteria
+     * Not Java: https://stackoverflow.com/questions/39060221/a-pipeline-stage-specification-object-must-contain-exactly-one-field-with-php-mo
+     *
+     * (https://stackoverflow.com/questions/55029222/parse-mongodb-query-to-java)
+     * https://www.programcreek.com/java-api-examples/?api=com.mongodb.client.model.Aggregates
+     * On using group(TExpression) inside collection.aggregate().
+     *
+     *  For forEach lamba expressions, see also https://www.baeldung.com/foreach-java
+     *  and https://www.javatpoint.com/java-8-foreach
+     *  and https://stackoverflow.com/questions/47979978/ambiguous-reference-to-foreach-when-listing-mongodbs-database-in-java
+     *
+     *
+     * The mongodb aggregate() we want to run this time:
+     *
        db.Websites.aggregate([
        {
@@ -400 +417 @@
         orQuery);
     
-    AggregateIterable<Document> output
-        = collection.aggregate(Arrays.asList(
+    // Hopefully the lambda expression (forEach()) at end means
+    // we write out each result Document as we get it
+    collection.aggregate(Arrays.asList(
          match(andQuery),
          unwind("$geoLocationCountryCode"),
          group("NZ", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
          sort(BasicDBObject.parse("{count : -1}"))
-     ));
-
-    // should only have one doc
-    for (Document doc : output) {
-        //System.out.println(doc);
-        System.out.println(doc.toJson());
-        // https://mongodb.github.io/mongo-java-driver/3.9/javadoc/index.html?org/bson/json/JsonWriterSettings.html
-        //writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true/*indent*/)) + NEWLINE);
-        /*
-        JsonWriterSettings writeSettings = new JsonWriterSettings();
-        writeSettings.builder().outputMode(JsonMode.SHELL).indent(true).build();        
-        writer.write(doc.toJson(writeSettings) + NEWLINE);
-        */
-        writer.write(prettyPrintJson(doc.toJson()) + NEWLINE);
-    }
+     )).forEach((Block<Document>)doc -> writeDoc(doc, writer));
+
+    // should only have one doc for NZ since it's a count by geolocation.
 
     return;
@@ -426 +432 @@
     
     /**
-       RUNNING A MONGODB COLLECTION.AGGREGATE() in JAVA:
-
-       https://stackoverflow.com/questions/31643109/mongodb-aggregation-with-java-driver
-       https://stackoverflow.com/questions/48000891/parse-mongodb-json-query-in-java-with-multiple-criteria
-       Not Java: https://stackoverflow.com/questions/39060221/a-pipeline-stage-specification-object-must-contain-exactly-one-field-with-php-mo
-
-       (https://stackoverflow.com/questions/55029222/parse-mongodb-query-to-java)
-       https://www.programcreek.com/java-api-examples/?api=com.mongodb.client.model.Aggregates
-       On using group(TExpression) inside collection.aggregate().
-
-       
-       The aggregate() we want to run:
-
+     * The aggregate() we want to run this time:
+     *
        db.Websites.aggregate([
        {
@@ -460 +455 @@
      { $sort : { count : -1} }
     ]);
-
-
-    https://stackoverflow.com/questions/54746814/jsonwriter-add-a-new-line
     */
-    public void aggregateContainsMRIForOverseas(Writer writer) throws IOException {
+    public void aggregateContainsMRIForOverseas(Writer writer) throws UncheckedIOException {
     // working with the WebSites collection, not WebPages collection!
     MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
 
-    /*String matchQuery =
-    "$and: ["
-        + "{geoLocationCountryCode: {$ne: \"NZ\"}},"
-        + "{domain: {$not: /\\.nz/}},"
-        + "{numPagesContainingMRI: {$gt: 0}},"
-        + "{$or: [{geoLocationCountryCode: \"AU\"}, {urlContainsLangCodeInPath: false}]}"
-            + "]";*/
-
-
-    
     
     Bson orQuery = or(
@@ -488 +470 @@
         BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
         orQuery);
-    
+
+
+    collection.aggregate(Arrays.asList(
+         match(andQuery),  //match(BasicDBObject.parse(matchQuery))
+         // match((List<DBObject>)JSON.parse(matchQuery)),
+         unwind("$geoLocationCountryCode"),
+         group("$geoLocationCountryCode", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
+         sort(BasicDBObject.parse("{count : -1}"))
+       )).forEach((Block<Document>)doc -> writeDoc(doc, writer));
+
+    // casting to Block<Document> necessary because otherwise we see the error at
+    // https://stackoverflow.com/questions/47979978/ambiguous-reference-to-foreach-when-listing-mongodbs-database-in-java
+
+    // Less efficient way is to keep all the results in memory and then
+    // write them out one at a time
+    /*
     AggregateIterable<Document> output
         = collection.aggregate(Arrays.asList(
@@ -497 +494 @@
          sort(BasicDBObject.parse("{count : -1}"))
      ));
+
     
     for (Document doc : output) {
         //System.out.println(doc);
         System.out.println(doc.toJson());
-        // https://mongodb.github.io/mongo-java-driver/3.9/javadoc/index.html?org/bson/json/JsonWriterSettings.html
-        //writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true/*indent*/)) + NEWLINE);
-        /*
-        JsonWriterSettings writeSettings = new JsonWriterSettings();
-        writeSettings.builder().outputMode(JsonMode.SHELL).indent(true).build();
-        writer.write(doc.toJson(writeSettings) + NEWLINE);
-        */
-        writer.write(prettyPrintJson(doc.toJson()) + NEWLINE);
-    }
-
+        
+    }
+    */
     return;
     }
 
-
+    /**
+     * called by lambda forEach() call on Document objects to write them out to a file.
+     * Have to deal with unreported exceptions here that can't be dealt with when doing
+     * the actual forEach(). See
+     * https://stackoverflow.com/questions/39090292/how-to-cleanly-deal-with-unreported-exception-ioexception-in-stream-foreach
+     */
+    
+    public void writeDoc(Document doc, Writer writer) throws UncheckedIOException {
+    //OLD WAY: writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true)) + NEWLINE);
+    // Can't control json output to add newlines after each array element,
+    // no matter which JsonMode is used.
+    
+    // https://mongodb.github.io/mongo-java-driver/3.9/javadoc/index.html?org/bson/json/JsonWriterSettings.html
+    // Still can't control array element output,
+    // but this way uses newer mongo java driver 3.9(.1). Tried its various JsonModes too:
+    //JsonWriterSettings writeSettings = new JsonWriterSettings();
+    //writeSettings.builder().outputMode(JsonMode.SHELL).indent(true).build();
+    //writer.write(doc.toJson(writeSettings) + NEWLINE);
+
+    // Not the JsonWriter of mongodb java driver:
+    // https://stackoverflow.com/questions/54746814/jsonwriter-add-a-new-line
+    
+    // Have to use gson's pretty print to produce a json string that contains
+    // newlines after every array element in the json:
+    String jsonStr = prettyPrintJson(doc.toJson());
+    System.err.println(jsonStr);
+    try {
+        writer.write(jsonStr + NEWLINE);
+    } catch (IOException ex) {
+        //throw ex;
+        throw new UncheckedIOException(ex);
+    }       
+    }
     public String prettyPrintJson(String jsonStr) {
     Gson gson = new GsonBuilder().setPrettyPrinting().create();