Changeset 33883

Timestamp:

2020-01-31T21:50:34+13:00 (4 years ago)

Author:

ak19

Message:

Clarifications

Location:

other-projects/maori-lang-detection

Files:

• mongodb-data/5table_tentativeNonProductSites1.csv (modified) (1 diff)
• mongodb-data/6table_nonProductSites1_manualShortlist.json (modified) (1 diff)
• src/org/greenstone/atea/MongoDBAccess.java (modified) (2 diffs)
• src/org/greenstone/atea/RandomURLsForDomainGenerator.java (modified) (2 diffs)
• src/org/greenstone/atea/WebPageURLsListing.java (modified) (12 diffs)

other-projects/maori-lang-detection/mongodb-data/5table_tentativeNonProductSites1.csv

@@ -1 @@
-"_id","siteCount","numPagesInMRICount","numPagesContainingMRICount"
+"_id","siteCount (numPagesContainingMRICount > 0)","numPagesInMRICount","numPagesContainingMRICount"
 "nz","176.0","4360","9641"
 "us","117.0","757","2655"

other-projects/maori-lang-detection/mongodb-data/6table_nonProductSites1_manualShortlist.json

@@ -180 +180 @@
 
 
+First column: n pages that are in MRI / n sampled isMRI pages
+Second column: n pages that do contain MRI / n sampled pages that are not isMRI yet contain MRI
 
 /* 1 */

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -537 +537 @@
     // newlines after every array element in the json:
     String jsonStr = prettyPrintJson(doc.toJson());
-    System.err.println(jsonStr);
+    //System.err.println(jsonStr);
     try {
         writer.write(jsonStr + NEWLINE);
@@ -553 +553 @@
     }
 
-
-    public void writeToFile(boolean append, String filename, AggregateIterable<Document> output) {
-    
-    // should only have one doc
-    for (Document doc : output) {
-        //System.out.println(doc);
-        System.out.println(doc.toJson());
-    }
-    }
-    
     
     /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */

other-projects/maori-lang-detection/src/org/greenstone/atea/RandomURLsForDomainGenerator.java

@@ -1 +1 @@
 package org.greenstone.atea;
 
-import java.util.*;
+import java.util.*; /* includes Random */
 import java.io.*;
 
@@ -71 +71 @@
          Writer writer = new BufferedWriter(new FileWriter(outFile));
          ) {
-        Collections.shuffle(urlsList);
+        Collections.shuffle(urlsList, new Random(1000));
         for (int i=0; i<numURLs; i++) {
         String url = urlsList.get(i);

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -7 +7 @@
 
 /**
-* TO COMPILE OR RUN, FIRST DO:
+ * Runs some of the important mongoDB queries I ran.
+ *
+ * TO COMPILE OR RUN, FIRST DO:
  *    cd maori-lang-detection/apache-opennlp-1.9.1
  *    export OPENNLP_HOME=`pwd`
@@ -18 +20 @@
  * TO RUN:
  *    maori-lang-detection/src$
- *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt 255
+ *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt
  *
 */
 public class WebPageURLsListing {
     static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
+    static private final long FIXED_SEED = 1000;
     
     private final MongoDBAccess mongodbAccess;
     private File outFolder;
 
+    
     public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
     {
@@ -32 +36 @@
     this.outFolder = outFolder;
     }
-    
-    public String produceURLsForPagesInMRI(File domainsFile) {
-    return writeFile(MongoDBAccess.IS_MRI, domainsFile);
-    }
-    
-    public String produceURLsForPagesContainingMRI(File domainsFile) {
-    return writeFile(MongoDBAccess.CONTAINS_MRI, domainsFile);
-    }
-
-
-    public String writeFile(int filterType, File domainsFile/*, int numURLs*/) {
-
+
+    private String getFilePath(File file) {
+    try {
+        return file.getCanonicalPath();
+    } catch(IOException e) {
+        return file.getAbsolutePath();
+    }
+    }
+    
+    public void produceURLsForPagesInMRI(File domainsFile) {
+    ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+    File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
+    writeURLsToFile(urlsList, outFile, urlsList.size());
+    
+    System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
+               + getFilePath(outFile));
+    }
+    
+    public void produceURLsForPagesContainingMRI(File domainsFile) {
+    ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.CONTAINS_MRI, domainsFile);   
+    File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
+    writeURLsToFile(urlsList, outFile, urlsList.size());
+
+    System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
+               + getFilePath(outFile));
+    }
+    
+    private ArrayList<String> getURLsForWebPages(int filterType, File domainsFile) {
     ArrayList<String> urlsList = new ArrayList<String>();
     
@@ -52 +72 @@
     try (
          BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
-         ) {
-        
+         ) {        
         
         String domain;
@@ -62 +81 @@
             ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
             urlsList.addAll(moreURLs);
+            if(moreURLs.size() == 0 && filterType == MongoDBAccess.IS_MRI) {
+            System.out.println("Domain " + domain + " had no isMRI webpages (only containsMRI).");
+            }
         }
         }
     } catch(Exception e) {
-        logger.error("Unable to read URLs from file " + domainsFile.getAbsolutePath());
-        logger.error(e.getMessage(), e);
-    }
-
-    // Shuffle the urlsList, then write out the first numURLs into a file.  
-    // https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
-    File parentFolder = domainsFile.getParentFile();
-    //File outFile = new File(parentFolder, "random"+numURLs+"_"+domainsFile.getName());
+        logger.error("Unable to read URLs from file " + getFilePath(domainsFile));
+        logger.error(e.getMessage(), e);
+    }
+
+    return urlsList;
+    }
+    
+    /** Given a hand curated list of NZ sites with positive numPagesContainingMRI,
+     * get a listing of all their web pages IN_MRI (or CONTAINS_MRI?). 
+     * Total all these pages in MRI (N), then work out the correct sample size (n)
+     * at 90% confidence with 5% margin of error. Then generate a random listing
+     * of n of these pages in MRI of these trusted sites and output to a file
+     * for manual inspection. */
+    /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
+     * of all their web pages IN_MRI (or CONTAINS_MRI).
+     * Plus a listing of all the NZ pages IN_MRI. */
+    //public void webPagesOfAllNZSitesAndDomainListing(File domainsFile) {
+    public void mriWebPageListingForDomainListing(File domainsFile) {
+
+    int filterType = MongoDBAccess.IS_MRI;
+    
+    // for overseas websites, 
+    //produceURLsForPagesContainingMRI(handCuratedOverseasDomainsFile);
+
+    // 0. get a list of all the web pages in the given domain listing where isMRI = true
+    ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+        // produceURLsForPagesInMRI(domainsFile);
+    
+    // 1. calculate the population size, N, the number of all webpages in the given domain
+    // site listing where isMRI = true.
+    int N_totalNumPages = urlsList.size();
+
+    // 2. write all the URLs in urlsList to a file
+    //File outFolder = domainsFile.getParentFile();
     String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
-    File outFile = new File(parentFolder, fileName+domainsFile.getName());
-
-    // write out ALL the URLs
+    File outFile = new File(outFolder, fileName+domainsFile.getName());
+
+    writeURLsToFile(urlsList, outFile, N_totalNumPages);
+    System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
+               + "\ninto file: " + getFilePath(outFile));
+        
+    // 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
+    int n_numSampleURLs = calcSampleSize(N_totalNumPages); 
+
+    System.err.println("*** N, total number of web pages that matched: " + N_totalNumPages);
+    System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);
+    
+    // 4. Shuffle all the URLs and write the first n (sample size) URLs to a file
+    // Using a constant seed for reproducibility
+    // https://stackoverflow.com/questions/6284589/setting-a-seed-to-shuffle-arraylist-in-java-deterministically
+    Collections.shuffle(urlsList, new Random(FIXED_SEED)); 
+    
+    outFile = new File(outFolder, "random"+n_numSampleURLs+"_"+domainsFile.getName());
+    writeURLsToFile(urlsList, outFile, n_numSampleURLs);
+    System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
+               + "for the sites in input domainsFile\ninto file: " + getFilePath(outFile));
+    }
+
+    /**
+     * Calculates sample size n for binary outcomes at 90% confidence and 5% margine of error
+     * for given population size N.
+     * @return n, the sample size.
+     */
+    public int calcSampleSize(int N) {
+    
+    // calculate sample size n for population size N if using 90% confidence and 5% margin of error
+    // https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+    // https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-sample-size/#CI1
+    // https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
+    
+    double m = 0.05; // margin of error = 5%
+    // for 90% confidence, alpha is the remainder = 10% and alpha/2 = 5%.
+    // For 90% confidence, use the table of known z_alpha/2 values from step 1 of
+    // https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
+    double z_alpha_over_2 = 1.6449;
+
+    // Formula: n = (zalpha2 ^ 2 * N) / ((z-alpha-2 ^ 2) + 4(N-1)*m^2)
+    // see https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+    double n = (Math.pow(z_alpha_over_2, 2.0) * N) / (Math.pow(z_alpha_over_2, 2.0) + (4 * (N-1) * Math.pow(m,2.0)));
+
+    // Round up to get a whole number:
+    return (int)Math.ceil(n);
+    }
+    
+    /**
+     * Writes out the first n URLs in urlsList into outFile.
+     */
+    private void writeURLsToFile(ArrayList<String> urlsList, File outFile, final int n) {
     try (
          Writer writer = new BufferedWriter(new FileWriter(outFile));
          ) {
 
-        for (int i=0; i < urlsList.size(); i++) {
+        for (int i=0; i < n; i++) {
         String url = urlsList.get(i);
         //System.out.println(list.get(i));
@@ -87 +185 @@
         }
     } catch(Exception e) {
-        logger.error("Unable to write to file " + outFile.getAbsolutePath());
-        logger.error(e.getMessage(), e);
-    }
-
-    /*
-    // shuffle list and take the first n - write to file
-    try (
-         Writer writer = new BufferedWriter(new FileWriter(outFile));
-         ) {
-        Collections.shuffle(urlsList);
-        for (int i=0; i<numURLs; i++) {
-        String url = urlsList.get(i);
-        //System.out.println(list.get(i));
-        writer.write(url + "\n");
-        }
-    } catch(Exception e) {
-        logger.error("Unable to write to file " + outFile.getAbsolutePath());
-        logger.error(e.getMessage(), e);
-    }
-    */
-
-    return outFile.getAbsolutePath();
+        logger.error("Unable to write to file " + getFilePath(outFile));
+        logger.error(e.getMessage(), e);
+    }
     }
 
@@ -124 +203 @@
     File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
 
-    String filename = outFile.getAbsolutePath();
+    String filename = getFilePath(outFile);
     
     try (
@@ -135 +214 @@
         boolean isMiInURLPath = false;
         mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
-
-        filename = outFile.getCanonicalPath();
-    } catch(Exception e) {
-        logger.error("Unable to write to file " + outFile.getAbsolutePath());
+        
+    } catch(Exception e) {
+        logger.error("Unable to write to file " + filename);
         logger.error(e.getMessage(), e);
     }
@@ -149 +227 @@
     /**
      * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
-     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path
+     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
+     * This listing is separate to allow easier weeding out of product sites/autotranslated
+     * sites when eyeballing the listing output.
      */
     public String writeOverseasSitesWithMiInURLPath() {
     File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
 
-    String filename = outFile.getAbsolutePath();
+    String filename = getFilePath(outFile);
     try (
          Writer writer = new BufferedWriter(new FileWriter(outFile));
@@ -160 +240 @@
         boolean isMiInURLPath = true;
         mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
-        filename = outFile.getCanonicalPath();
-    } catch(Exception e) {
-        logger.error("Unable to write to file " + outFile.getAbsolutePath());
+
+    } catch(Exception e) {
+        logger.error("Unable to write to file " + filename);
         logger.error(e.getMessage(), e);
     }
@@ -171 +251 @@
     
     public static void printUsage() {
-    System.err.println("WebPageURLsListing [domains.txt]");
-    }
-
-    // Depending on args, generates isMRI and containsMRI file listings for:
-    // 1. UNFILTERED: all (NZ + overseas) AND takes manually curated domain list file for overseas and adds all NZ
-
-    
+    System.err.println("Usage: WebPageURLsListing [domains.txt]");
+    }
+
+    /**
+     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
+     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
+     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
+     * translated and really contain at least one webpage containing at least one sentence in MRI.
+     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
+     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
+     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
+     * 90% confidence with 5% margin of error for testing binary outcomes, see
+     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
+     */
     public static void main(String args[]) {
     if(args.length >= 2) {
@@ -202 +289 @@
         }
 
-        
-        //int genNumURLs = Integer.parseInt(args[1]);
         //String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
         //String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);
 
-        //listing.writeWebPagesOfAllNZSitesAndDomainListing();
+
+        // TODO: for NZ, do IS_MRI. For overseas still CONTAINS_MRI
+        // then also do the shuffle to gen X num of random web page URLs.
+        //String filename = listing.webPagesOfAllNZSitesAndDomainListing(domainsFile);
+        listing.mriWebPageListingForDomainListing(domainsFile);
+
+        // TODO: generate the special table (6)
         
         } else {        
 
+        // calculating sample size works:
+        //System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
+        //System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
+        
         String filename = listing.writeTentativeNonAutotranslatedSites();
         filename = listing.writeOverseasSitesWithMiInURLPath();
+
+        // TODO: generate the tables
         }
-
         
     } catch(Exception e) {