Changeset 33885 for other-projects

Timestamp:

2020-01-31T22:54:15+13:00 (4 years ago)

Author:

ak19

Message:

Attempting to write the tables. csv not yet supported. Table 1 done.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (4 diffs)
• WebPageURLsListing.java (modified) (2 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -39 +39 @@
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -512 +514 @@
     }
 
+    /** Do the aggregates for writing out tables.
+       Table1:
+       db.Websites.aggregate([
+       
+       { $unwind: "$geoLocationCountryCode" },
+       {
+       $group: {
+       _id: "$geoLocationCountryCode",
+       count: { $sum: 1 },
+       //domain: { $addToSet: '$domain' },
+       numPagesInMRICount: { $sum: '$numPagesInMRI' },
+       numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+       }
+       },
+       { $sort : { count : -1} }
+       ]);
+    */
+    public void writeTables(File outFolder) {
+    // In this function, we're always dealing with the Websites mongodb collection.
+    MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
+
+    // table 1
+    File outFile = new File(outFolder, "1table_allCrawledSites.csv");
+    try (
+         Writer writer = new BufferedWriter(new FileWriter(outFile));
+         ) {
+        
+        
+        AggregateIterable<Document> output = collection.aggregate(Arrays.asList(
+                           //match(BasicDBObject.parse("{urlContainsLangCodeInPath:true}")),
+         unwind("$geoLocationCountryCode"),
+         group("$geoLocationCountryCode", Arrays.asList(
+                                sum("count", 1),
+                                /*addToSet("domain", "$domain"),*/
+                                sum("numPagesInMRICount", "$numPagesInMRI"),
+                                sum("numPagesContainingMRICount", "$numPagesContainingMRI"))),
+         sort(BasicDBObject.parse("{count : -1}"))
+        ));//.forEach((Block<Document>)doc -> writeDoc(doc, writer));
+
+        int docNum = 0;
+        for (Document doc : output) {       
+        //System.out.println(doc);
+        writeDoc(++docNum, doc, writer);        
+        }
+        
+        
+    } catch(UncheckedIOException ioe) {
+        logger.error("Caught UncheckedIOException: " + ioe.getMessage(), ioe);
+    }
+    catch(Exception e) {
+        logger.error("Could not write table to file " + outFile, e);
+    }
+    }
+
+    public void doTable1() {
+    
+    }
+    
+    
+    
     /**
      * called by lambda forEach() call on Document objects to write them out to a file.
@@ -517 +579 @@
      * the actual forEach(). See
      * https://stackoverflow.com/questions/39090292/how-to-cleanly-deal-with-unreported-exception-ioexception-in-stream-foreach
-     */
-    
+     */    
     public void writeDoc(Document doc, Writer writer) throws UncheckedIOException {
     //OLD WAY: writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true)) + NEWLINE);
@@ -545 +606 @@
     }       
     }
+    public void writeDoc(int docNum, Document doc, Writer writer) throws UncheckedIOException {
+    String jsonStr = prettyPrintJson(doc.toJson());
+    //System.err.println(jsonStr);
+    try {
+        writer.write("/* " + docNum + " */\n" + jsonStr + NEWLINE);
+    } catch (IOException ex) {
+        //throw ex;
+        throw new UncheckedIOException(ex);
+    }
+    }
+    
+    
     public String prettyPrintJson(String jsonStr) {
     Gson gson = new GsonBuilder().setPrettyPrinting().create();

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -20 +20 @@
  * TO RUN:
  *    maori-lang-detection/src$
+ *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing
+ * OR:
  *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt
  *
@@ -344 +346 @@
 
         // TODO: generate the tables
+
+        mongodb.writeTables(outFolder);
         }