Changeset 33917 for other-projects

Timestamp:

2020-02-13T18:18:13+13:00 (4 years ago)

Author:

ak19

Message:

Added some better reporting when confirming sample size was correct

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBQueryer.java (modified) (1 diff)
• SummaryTool.java (modified) (2 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBQueryer.java

@@ -177 +177 @@
     return urlsList;
     }
-
+    
+    /**
+     * Does a mongoDB query like the following, depending on filter type:
+     *    db.getCollection('Webpages').find({isMRI: true}).count()
+     * @param filterType can be either IS_MRI or CONTAINS_MRI.
+     * @return the number of webpages that matched the filterType setting.
+    */
+    public long countOfWebpagesMatching(int filterType) {
+    String query = (filterType == IS_MRI) ? "{isMRI: true}" : "{containsMRI: true}";
+    long result = -1;   
+    MongoCollection<Document> collection = getWebpagesCollection();
+
+    
+    try {
+        BasicDBObject queryObj = BasicDBObject.parse(query);
+        //result = collection.find(queryObj).count();
+        // https://stackoverflow.com/questions/32683458/how-to-call-count-operation-after-find-with-mongodb-java-driver
+        result = collection.countDocuments(queryObj);
+        
+    } catch(Exception e) {
+        logger.error("MongoDB couldn't parse provided query " + query);
+    }
+    
+    return result;
+    }
+    
     /**     
      * RUNNING A MONGODB COLLECTION.AGGREGATE() in JAVA:

other-projects/maori-lang-detection/src/org/greenstone/atea/SummaryTool.java

@@ -148 +148 @@
     // 2. write all the URLs in urlsList to a file
     //File outFolder = domainsFile.getParentFile();
-    String fileName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI_" : "containsMRI_";
-    File outFile = new File(outFolder, fileName+domainsFile.getName());
+    String filterName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI" : "containsMRI";
+    File outFile = new File(outFolder, filterName+"_"+domainsFile.getName());
 
     writeURLsToFile(urlsList, outFile, N_totalNumPages);
@@ -156 +156 @@
         
     // 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
-    int n_numSampleURLs = calcSampleSize(N_totalNumPages); 
-
-    System.err.println("*** N, total number of web pages that matched: " + N_totalNumPages);
+    int n_numSampleURLs = calcSampleSize(N_totalNumPages);
+
+    System.err.println("*** N, total number of web pages for which " + filterName + "=true from domain shortlist: " + N_totalNumPages);
+    System.err.println("    (out of " + mongodbQueryer.countOfWebpagesMatching(filterType)
+               + " web pages across ALL sites for which " + filterName + " = true)");
     System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);