Changeset 33887 for other-projects

Timestamp:

2020-01-31T23:49:11+13:00 (4 years ago)

Author:

ak19

Message:

1. Added support for writing out tables in csv format too. 2. Second table written out now. 3. Moved getFilePath() into Utility.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (8 diffs)
• Utility.java (modified) (1 diff)
• WebPageURLsListing.java (modified) (9 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -17 +17 @@
 import static com.mongodb.client.model.Accumulators.*;
 
-
 //import org.bson.conversions.Bson;
 import com.mongodb.BasicDBObject;
@@ -57 +56 @@
 import org.greenstone.atea.morphia.*;
 import dev.morphia.*;
+
+import org.apache.commons.csv.*;
 
 /** 
@@ -83 +84 @@
  *   - https://www.mongodb.com/blog/post/6-rules-of-thumb-for-mongodb-schema-design-part-1
  *
+ * API:
+ * - https://mongodb.github.io/mongo-java-driver/3.4/javadoc/?com/mongodb/client/MongoCollection.html#find--
+ * - examples: https://mongodb.github.io/mongo-java-driver/3.4/driver/getting-started/quick-start/
  */
 public class MongoDBAccess implements AutoCloseable {
@@ -516 +520 @@
     /** Do the aggregates for writing out tables.
        Table1:
-       db.Websites.aggregate([
        
-       { $unwind: "$geoLocationCountryCode" },
-       {
-       $group: {
-       _id: "$geoLocationCountryCode",
-       count: { $sum: 1 },
-       //domain: { $addToSet: '$domain' },
-       numPagesInMRICount: { $sum: '$numPagesInMRI' },
-       numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
-       }
-       },
-       { $sort : { count : -1} }
-       ]);
     */
     public void writeTables(File outFolder) {
@@ -535 +526 @@
     MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
 
-    // table 1
-    File outFile = new File(outFolder, "1table_allCrawledSites.csv");
-    try (
-         Writer writer = new BufferedWriter(new FileWriter(outFile));
-         ) {
+    String[] tableNames = { "", "1table_allCrawledSites", "2table_sitesWithPagesInMRI"};
+    for (int tableNum = 1; tableNum < tableNames.length; tableNum++) {
+        File outFile = new File(outFolder, tableNames[tableNum] + ".json");
+        File csvFile = new File(outFolder, tableNames[tableNum] + ".csv");
+        try (
+         Writer writer = new BufferedWriter(new FileWriter(outFile));
+         CSVPrinter csvWriter = new CSVPrinter(new FileWriter(csvFile), CSVFormat.DEFAULT);
+         ) {
+
+        // Write out the CSV column headings
+        // https://commons.apache.org/proper/commons-csv/apidocs/org/apache/commons/csv/CSVPrinter.html
+        csvWriter.printRecord("countryCode", "siteCount",
+              "numPagesInMRI count","numPagesContainingMRICount"/*, "domain"*/);
+        
+        AggregateIterable<Document> output = getTable(collection, tableNum); //doTable1().forEach((Block<Document>)doc -> writeDoc(doc, writer));
+        
+        int docNum = 0;
+        for (Document doc : output) {       
+            //System.out.println(doc);
+            writeDocAsJsonRecord(++docNum, doc, writer);
+            writeDocAsCSVRecord(++docNum, doc, csvWriter);
+        }       
+        logger.info("@@@ Wrote out table into file: " + Utility.getFilePath(outFile) + " and .csv");
+        } catch(UncheckedIOException ioe) {
+        logger.error("Caught UncheckedIOException: " + ioe.getMessage(), ioe);
+        }
+        catch(Exception e) {
+        logger.error("Could not write table to file " + outFile + " or .csv equivalent" , e);
+        }
+    }
+    }
+
+    public AggregateIterable<Document> getTable(MongoCollection<Document> collection, int tableNum) {
+    
+    AggregateIterable<Document> output = null;
+
+    switch(tableNum) {
         
-        
-        AggregateIterable<Document> output = collection.aggregate(Arrays.asList(
+    case 1:
+        /* 1table_allCrawledSites -
+
+           db.Websites.aggregate([       
+           { $unwind: "$geoLocationCountryCode" },
+           {
+           $group: {
+           _id: "$geoLocationCountryCode",
+           count: { $sum: 1 },
+           //domain: { $addToSet: '$domain' },
+           numPagesInMRICount: { $sum: '$numPagesInMRI' },
+           numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+           }
+           },
+           { $sort : { count : -1} }
+           ]);
+         */
+        output = collection.aggregate(Arrays.asList(
                            //match(BasicDBObject.parse("{urlContainsLangCodeInPath:true}")),
          unwind("$geoLocationCountryCode"),
@@ -550 +589 @@
                                 sum("numPagesInMRICount", "$numPagesInMRI"),
                                 sum("numPagesContainingMRICount", "$numPagesContainingMRI"))),
-         sort(BasicDBObject.parse("{count : -1}"))
-        ));//.forEach((Block<Document>)doc -> writeDoc(doc, writer));
-
-        int docNum = 0;
-        for (Document doc : output) {       
-        //System.out.println(doc);
-        writeDoc(++docNum, doc, writer);        
-        }
+         sort(BasicDBObject.parse("{count : -1}"))       
+        ));
+        break;
+
+    case 2:
+        /*
+          db.Websites.aggregate([
+          { $match: { numPagesInMRI: {$gt: 0} } },
+          { $unwind: "$geoLocationCountryCode" },
+          {
+          $group: {
+          _id: {$toLower: '$geoLocationCountryCode'}, // ignore toLower
+          count: { $sum: 1 },
+          //domain: { $addToSet: '$domain' },
+          numPagesInMRICount: { $sum: '$numPagesInMRI' },
+          numPagesContainingMRICount: { $sum: '$numPagesContainingMRI' }
+          }
+          },
+          { $sort : { count : -1} }
+          ]);
+         */
+         output = collection.aggregate(Arrays.asList(
+                           match(BasicDBObject.parse("{ numPagesInMRI: {$gt: 0} }")),
+         unwind("$geoLocationCountryCode"),
+         group("$geoLocationCountryCode", Arrays.asList(
+                                sum("count", 1),
+                                /*addToSet("domain", "$domain"),*/
+                                sum("numPagesInMRICount", "$numPagesInMRI"),
+                                sum("numPagesContainingMRICount", "$numPagesContainingMRI"))),
+         sort(BasicDBObject.parse("{count : -1}"))       
+        ));
+        break;
         
+    default: logger.error("Unknown table number: " + tableNum);
         
-    } catch(UncheckedIOException ioe) {
-        logger.error("Caught UncheckedIOException: " + ioe.getMessage(), ioe);
-    }
-    catch(Exception e) {
-        logger.error("Could not write table to file " + outFile, e);
-    }
-    }
-
-    public void doTable1() {
+    }
+
+     return output;
     
     }
@@ -606 +664 @@
     }       
     }
-    public void writeDoc(int docNum, Document doc, Writer writer) throws UncheckedIOException {
+    
+    public void writeDocAsJsonRecord(int docNum, Document doc, Writer writer) throws UncheckedIOException {
     String jsonStr = prettyPrintJson(doc.toJson());
     //System.err.println(jsonStr);
@@ -616 +675 @@
     }
     }
-    
+
+    // TODO
+    //public void writeDocToJsonAndCSV(int docNum, Document doc, Writer writer, CSVPrinter csvWriter) throws UncheckedIOException {
+    public void writeDocAsCSVRecord(int docNum, Document doc, CSVPrinter csvWriter) throws UncheckedIOException {
+    String jsonStr = doc.toJson();
+    JsonParser parser = new JsonParser();
+    JsonElement json = parser.parse(jsonStr);
+
+    JsonObject jsonObj = (JsonObject)json;
+    
+    String countryCode = jsonObj.get("_id").getAsString();
+    int siteCount = jsonObj.get("count").getAsInt();
+    int numPagesInMRICount = jsonObj.get("numPagesInMRICount").getAsInt();
+    int numPagesContainingMRICount = jsonObj.get("numPagesContainingMRICount").getAsInt();  
+    
+    //System.err.println(jsonStr);
+    try {
+        //writer.write("/* " + docNum + " */\n" + prettyPrintJson(jsonStr) + NEWLINE);
+        csvWriter.printRecord(countryCode, siteCount, numPagesInMRICount, numPagesContainingMRICount);
+    } catch (IOException ex) {
+        //throw ex;
+        throw new UncheckedIOException(ex);
+    }
+    }
     
     public String prettyPrintJson(String jsonStr) {

other-projects/maori-lang-detection/src/org/greenstone/atea/Utility.java

@@ -11 +11 @@
 public class Utility {
     private static Logger logger = Logger.getLogger(org.greenstone.atea.Utility.class.getName());
+    
+    public static String getFilePath(File file) {
+    try {
+        return file.getCanonicalPath();
+    } catch(IOException e) {
+        return file.getAbsolutePath();
+    }
+    }
     
     // Run gunzip

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -55 +55 @@
     }
 
-    private String getFilePath(File file) {
-    try {
-        return file.getCanonicalPath();
-    } catch(IOException e) {
-        return file.getAbsolutePath();
-    }
-    }
     
     public void produceURLsForPagesInMRI(File domainsFile) {
@@ -69 +62 @@
     
     System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
-               + getFilePath(outFile));
+               + Utility.getFilePath(outFile));
     }
     
@@ -78 +71 @@
 
     System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
-               + getFilePath(outFile));
+               + Utility.getFilePath(outFile));
     }
     
@@ -120 +113 @@
         System.err.println("");
     } catch(Exception e) {
-        logger.error("Unable to read URLs from file " + getFilePath(domainsFile));
+        logger.error("Unable to read URLs from file " + Utility.getFilePath(domainsFile));
         logger.error(e.getMessage(), e);
     }
@@ -159 +152 @@
     writeURLsToFile(urlsList, outFile, N_totalNumPages);
     System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
-               + "\ninto file: " + getFilePath(outFile));
+               + "\ninto file: " + Utility.getFilePath(outFile));
         
     // 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
@@ -175 +168 @@
     writeURLsToFile(urlsList, outFile, n_numSampleURLs);
     System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
-               + "for the sites in input domainsFile\ninto file: " + getFilePath(outFile));
+               + "for the sites in input domainsFile\ninto file: " + Utility.getFilePath(outFile));
     }
 
@@ -219 +212 @@
         }
     } catch(Exception e) {
-        logger.error("Unable to write to file " + getFilePath(outFile));
+        logger.error("Unable to write to file " + Utility.getFilePath(outFile));
         logger.error(e.getMessage(), e);
     }
@@ -237 +230 @@
     File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
 
-    String filename = getFilePath(outFile);
+    String filename = Utility.getFilePath(outFile);
     
     try (
@@ -268 +261 @@
     File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
 
-    String filename = getFilePath(outFile);
+    String filename = Utility.getFilePath(outFile);
     try (
          Writer writer = new BufferedWriter(new FileWriter(outFile));