Changeset 33880 for other-projects

Timestamp:

2020-01-30T21:17:40+13:00 (4 years ago)

Author:

ak19

Message:

Write out the 5counts_tentativeNonAutotranslatedSites.json file with proper pretty printing: needed gson's capabilities to convert mongodb's document result into json string output, as this allows newlines between array elements whereas mongodb's JsonWriter didn't.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (6 diffs)
• WebPageURLsListing.java (modified) (3 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -17 +17 @@
 import static com.mongodb.client.model.Accumulators.*;
 
+
 //import org.bson.conversions.Bson;
 import com.mongodb.BasicDBObject;
@@ -28 +29 @@
 import org.bson.Document;
 import org.bson.conversions.Bson;
+import org.bson.json.JsonMode;
+import org.bson.json.JsonWriterSettings;
     
 import com.mongodb.util.JSON;
 //import com.mongodb.DBObject;
+
+
+import com.google.gson.*; // for pretty printing
 
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
+import java.io.IOException;
+import java.io.Writer;
+
 import java.util.Arrays;
 import java.util.ArrayList;
@@ -79 +88 @@
     public static final String WEBPAGES_COLLECTION = "Webpages";
     public static final String WEBSITES_COLLECTION = "Websites";
+
+    public static final String NEWLINE = System.getProperty("line.separator");
     
     /** mongodb filter types to execute */
@@ -351 +362 @@
     }
 
+    /**       
+       The mongodb aggregate() we want to run this time:
+
+       db.Websites.aggregate([
+       {
+        $match: {
+            $and: [
+                {numPagesContainingMRI: {$gt: 0}},
+                {$or: [{geoLocationCountryCode:"NZ"},{domain: /\.nz/}]}
+            ]
+          }
+    },
+    { $unwind: "$geoLocationCountryCode" },
+    {
+          $group: {
+            _id: "nz",
+            count: { $sum: 1 },
+            domain: { $addToSet: '$domain' }
+          }
+    },
+    { $sort : { count : -1} }
+    ]);
+    */
+    public void aggregateContainsMRIForNZ(Writer writer) throws IOException {
+    // working with the WebSites collection, not WebPages collection!
+    MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
+
+    
+    //String isMRI_filter = 
+    
+    Bson orQuery = or(
+              BasicDBObject.parse("{geoLocationCountryCode: \"NZ\"}"),
+              BasicDBObject.parse("{domain: /\\.nz/}")
+              );
+    Bson andQuery = and(
+        BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
+        orQuery);
+    
+    AggregateIterable<Document> output
+        = collection.aggregate(Arrays.asList(
+         match(andQuery),
+         unwind("$geoLocationCountryCode"),
+         group("NZ", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
+         sort(BasicDBObject.parse("{count : -1}"))
+     ));
+
+    // should only have one doc
+    for (Document doc : output) {
+        //System.out.println(doc);
+        System.out.println(doc.toJson());
+        // https://mongodb.github.io/mongo-java-driver/3.9/javadoc/index.html?org/bson/json/JsonWriterSettings.html
+        //writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true/*indent*/)) + NEWLINE);
+        /*
+        JsonWriterSettings writeSettings = new JsonWriterSettings();
+        writeSettings.builder().outputMode(JsonMode.SHELL).indent(true).build();        
+        writer.write(doc.toJson(writeSettings) + NEWLINE);
+        */
+        writer.write(prettyPrintJson(doc.toJson()) + NEWLINE);
+    }
+
+    return;
+    }
+    
     /**
        RUNNING A MONGODB COLLECTION.AGGREGATE() in JAVA:
@@ -387 +461 @@
     ]);
 
+
+    https://stackoverflow.com/questions/54746814/jsonwriter-add-a-new-line
     */
-    public String aggregateContainsMRIForOverseas() {
+    public void aggregateContainsMRIForOverseas(Writer writer) throws IOException {
     // working with the WebSites collection, not WebPages collection!
     MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
@@ -425 +501 @@
         //System.out.println(doc);
         System.out.println(doc.toJson());
-    }
-
-    return "";
-    }
-
-
-    /**       
-       The mongodb aggregate() we want to run this time:
-
-       db.Websites.aggregate([
-       {
-        $match: {
-            $and: [
-                {numPagesContainingMRI: {$gt: 0}},
-                {$or: [{geoLocationCountryCode:"NZ"},{domain: /\.nz/}]}
-            ]
-          }
-    },
-    { $unwind: "$geoLocationCountryCode" },
-    {
-          $group: {
-            _id: "nz",
-            count: { $sum: 1 },
-            domain: { $addToSet: '$domain' }
-          }
-    },
-    { $sort : { count : -1} }
-    ]);
-    */
-    public String aggregateContainsMRIForNZ() {
-    // working with the WebSites collection, not WebPages collection!
-    MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
-
-    
-    //String isMRI_filter = 
-    
-    Bson orQuery = or(
-              BasicDBObject.parse("{geoLocationCountryCode: \"NZ\"}"),
-              BasicDBObject.parse("{domain: /\\.nz/}")
-              );
-    Bson andQuery = and(
-        BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
-        orQuery);
-    
-    AggregateIterable<Document> output
-        = collection.aggregate(Arrays.asList(
-         match(andQuery),
-         unwind("$geoLocationCountryCode"),
-         group("NZ", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
-         sort(BasicDBObject.parse("{count : -1}"))
-     ));
-
-    // should only have one doc
-    for (Document doc : output) {
-        //System.out.println(doc);
-        System.out.println(doc.toJson());
-    }
-
-    return "";
-    }
+        // https://mongodb.github.io/mongo-java-driver/3.9/javadoc/index.html?org/bson/json/JsonWriterSettings.html
+        //writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true/*indent*/)) + NEWLINE);
+        /*
+        JsonWriterSettings writeSettings = new JsonWriterSettings();
+        writeSettings.builder().outputMode(JsonMode.SHELL).indent(true).build();
+        writer.write(doc.toJson(writeSettings) + NEWLINE);
+        */
+        writer.write(prettyPrintJson(doc.toJson()) + NEWLINE);
+    }
+
+    return;
+    }
+
+
+    public String prettyPrintJson(String jsonStr) {
+    Gson gson = new GsonBuilder().setPrettyPrinting().create();
+    JsonParser jp = new JsonParser();
+    JsonElement je = jp.parse(jsonStr);
+    String prettyJsonString = gson.toJson(je);
+    return prettyJsonString;
+    }
+
 
     public void writeToFile(boolean append, String filename, AggregateIterable<Document> output) {

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -27 +27 @@
     private int numURLs;
     private File domainsFile;
+
+
     
     public WebPageURLsListing(MongoDBAccess mongodbAccess,
@@ -120 +122 @@
     return outFile.getAbsolutePath();
     }
+
+    /* ---------------------------------------- */
+
+    /** 
+     * Create the file 
+     * @return full path of file generated
+     */
+    public String writeTentativeNonAutotranslatedSites() {
+    File outFolder = new File("../mongodb-data/").getAbsoluteFile();
+    File outFile = new File(outFolder, "5counts_tentativeNonAutotranslatedSites.json");
+
+    String filename = outFile.getAbsolutePath();
+    
+    try (
+         Writer writer = new BufferedWriter(new FileWriter(outFile));
+         ) {
+        // first write out NZ sites and .nz TLD count and domains
+        mongodbAccess.aggregateContainsMRIForNZ(writer);
+        // next write out all overseas sites and .nz TLD count and domains
+        mongodbAccess.aggregateContainsMRIForOverseas(writer);
+
+        filename = outFile.getCanonicalPath();
+    } catch(Exception e) {
+        logger.error("Unable to write to file " + outFile.getAbsolutePath());
+        logger.error(e.getMessage(), e);
+    }
+
+    return filename;
+    }
     
     
@@ -152 +183 @@
         //String isMRIFile = listing.produceURLsForPagesInMRI();
         //String containsMRIFile = listing.produceURLsForPagesContainingMRI();
-        mongodb.aggregateContainsMRIForNZ();
-        mongodb.aggregateContainsMRIForOverseas();
+        String filename = listing.writeTentativeNonAutotranslatedSites();
+        System.err.println("Check file: " + filename);
+
         
     } catch(Exception e) {