source: other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java@ 33906

package org.greenstone.atea;

import java.util.*;
import java.io.*;

import org.apache.log4j.Logger;

/**
 * Runs some of the important mongoDB queries I ran.
 *
 * TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing
 * OR:
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt
 *
*/
public class WebPageURLsListing {
    static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
    static private final long FIXED_SEED = 1000;
    
    private final MongoDBAccess mongodbAccess;
    private File outFolder;


    
    public static class Tuple {
    public final String url;
    public final String countryCode;
    
    public Tuple(String url, String countryCode) {
        this.url = url;
        this.countryCode = countryCode;
    }

    public String toString() {
        return this.url + "," + countryCode;
    }
    }
    
    
    public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
    {
    this.mongodbAccess = mongodbAccess;
    this.outFolder = outFolder;
    }

    
    public void produceURLsForPagesInMRI(File domainsFile) {
    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
    File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
    writeURLsToFile(urlsList, outFile, urlsList.size());
    
    System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
               + Utility.getFilePath(outFile));
    }
    
    public void produceURLsForPagesContainingMRI(File domainsFile) {
    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.CONTAINS_MRI, domainsFile);    
    File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
    writeURLsToFile(urlsList, outFile, urlsList.size());

    System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
               + Utility.getFilePath(outFile));
    }
    
    private ArrayList<Tuple> getURLsForAllWebPagesInSiteListing(int filterType, File domainsFile) {
    ArrayList<Tuple> urlsList = new ArrayList<Tuple>();
    
    // 1. read each url from the domainsFile
    // 1a. do the query
    // 1b. add the arraylist result to urls
    
    try (
         BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
         ) {        
        
        String domain;
        
        while((domain = reader.readLine()) != null) {
        domain = domain.trim();
        if(!domain.equals("")) {
            
            String countryCode = "";
            int index = domain.lastIndexOf(",");
            if(index != -1) {
            countryCode = domain.substring(index+1).trim();
            domain = domain.substring(0, index);
            }
            ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);

            // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
            if(moreURLs.size() == 0 && filterType == MongoDBAccess.IS_MRI) {
            System.out.println("   " + countryCode + " domain " + domain + " had no isMRI webpages - only containsMRI.");
            }

            //urlsList.addAll(moreURLs);
            for(int i = 0; i < moreURLs.size(); i++) {
            urlsList.add(new Tuple(moreURLs.get(i), countryCode));
            }
            
        }
        }
        System.err.println("");
    } catch(Exception e) {
        logger.error("Unable to read URLs from file " + Utility.getFilePath(domainsFile));
        logger.error(e.getMessage(), e);
    }

    return urlsList;
    }
    
    /** Given a hand curated list of all sites with positive numPagesContainingMRI
     * determined by manual inspection, get a listing of all their web pages that
     * are IN_MRI (or CONTAINS_MRI?). 
     * Total all these pages that are inMRI (N), then work out the correct sample size (n)
     * at 90% confidence with 5% margin of error. Then generate a random listing
     * of n of these pages in MRI of these trusted sites and output to a file
     * for manual inspection of the sample webpage URLs at page-level. */
    /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
     * of all their web pages IN_MRI (or CONTAINS_MRI).
     * Plus a listing of all the NZ pages IN_MRI. */
    //public void webPagesOfAllNZSitesAndDomainListing(File domainsFile) {
    public void mriWebPageListingForDomainListing(File domainsFile) {

    int filterType = MongoDBAccess.IS_MRI;
    
    // for overseas websites, 
    //produceURLsForPagesContainingMRI(handCuratedOverseasDomainsFile);

    // 0. get a list of all the web pages in the given domain listing where isMRI = true
    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
        // produceURLsForPagesInMRI(domainsFile);
    
    // 1. calculate the population size, N, the number of all webpages in the given domain
    // site listing where isMRI = true.
    int N_totalNumPages = urlsList.size();

    // 2. write all the URLs in urlsList to a file
    //File outFolder = domainsFile.getParentFile();
    String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
    File outFile = new File(outFolder, fileName+domainsFile.getName());

    writeURLsToFile(urlsList, outFile, N_totalNumPages);
    System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
               + "\ninto file: " + Utility.getFilePath(outFile));
        
    // 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
    int n_numSampleURLs = calcSampleSize(N_totalNumPages); 

    System.err.println("*** N, total number of web pages that matched: " + N_totalNumPages);
    System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);
    
    // 4. Shuffle all the URLs and write the first n (sample size) URLs to a file
    // Using a constant seed for reproducibility
    // https://stackoverflow.com/questions/6284589/setting-a-seed-to-shuffle-arraylist-in-java-deterministically
    Collections.shuffle(urlsList, new Random(FIXED_SEED)); 
    
    outFile = new File(outFolder, "random"+n_numSampleURLs+"_"+domainsFile.getName());
    writeURLsToFile(urlsList, outFile, n_numSampleURLs);
    System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
               + "for the sites in input domainsFile\ninto file: " + Utility.getFilePath(outFile));
    }

    /**
     * Calculates sample size n for binary outcomes at 90% confidence and 5% margine of error
     * for given population size N.
     * @return n, the sample size.
     */
    public int calcSampleSize(int N) {
    
    // calculate sample size n for population size N if using 90% confidence and 5% margin of error
    // https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
    // https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-sample-size/#CI1
    // https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
    
    double m = 0.05; // margin of error = 5%
    // for 90% confidence, alpha is the remainder = 10% and alpha/2 = 5%.
    // For 90% confidence, use the table of known z_alpha/2 values from step 1 of
    // https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
    double z_alpha_over_2 = 1.6449;

    // Formula: n = (zalpha2 ^ 2 * N) / ((z-alpha-2 ^ 2) + 4(N-1)*m^2)
    // see https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
    double n = (Math.pow(z_alpha_over_2, 2.0) * N) / (Math.pow(z_alpha_over_2, 2.0) + (4 * (N-1) * Math.pow(m,2.0)));

    // Round up to get a whole number:
    return (int)Math.ceil(n);
    }
    
    /**
     * Writes out the first n URLs in urlsList into outFile.
     */
    private void writeURLsToFile(ArrayList<Tuple> urlsList, File outFile, final int n) {
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {

        for (int i=0; i < n; i++) {
        Tuple urlInfo = urlsList.get(i);
        
        //System.out.println(list.get(i));
        writer.write(urlInfo + "\n"); // calls toString() on tuple of url -> countryCode
        }
    } catch(Exception e) {
        logger.error("Unable to write to file " + Utility.getFilePath(outFile));
        logger.error(e.getMessage(), e);
    }
    }

    /* ---------------------------------------- */

    /** 
     * Create the file 5counts_tentativeNonAutotranslatedSites.json
     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
     * followed by counts and domain listing for overseas sites that are either from Australia
     * or don't contain mi in their URL path.
     * @return full path of file generated
     */
    public String writeTentativeNonAutotranslatedSites() {
    
    File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");

    String filename = Utility.getFilePath(outFile);
    
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        // first write out NZ sites and .nz TLD count and domains
        mongodbAccess.aggregateContainsMRIForNZ(writer, MongoDBAccess.CONTAINS_MRI);
        // next write out all overseas sites (not NZ origin or .nz TLD)
        // that have no "mi" in the URL path as mi.* or */mi
        boolean isMiInURLPath = false;
        mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
        
    } catch(Exception e) {
        logger.error("Unable to write to file " + filename);
        logger.error(e.getMessage(), e);
    }
    
    System.err.println("*** Wrote file: " + filename);

    return filename;
    }

    /**
     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
     * This listing is separate to allow easier weeding out of product sites/autotranslated
     * sites when eyeballing the listing output.
     */
    public String writeOverseasSitesWithMiInURLPath() {
    File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");

    String filename = Utility.getFilePath(outFile);
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        boolean isMiInURLPath = true;
        mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);

    } catch(Exception e) {
        logger.error("Unable to write to file " + filename);
        logger.error(e.getMessage(), e);
    }

    System.err.println("*** Wrote file: " + filename);
    return filename;
    }
    
    public static void printUsage() {
    System.err.println("Usage: WebPageURLsListing [domains.txt]");
    }

    /**
     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
     * translated and really contain at least one webpage containing at least one sentence in MRI.
     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
     * 90% confidence with 5% margin of error for testing binary outcomes, see
     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
     */
    public static void main(String args[]) {
    if(args.length >= 2) {
        printUsage();
        System.exit(-1);
    }

    try (
         MongoDBAccess mongodb = new MongoDBAccess();
         ) {

        mongodb.connectToDB();

        // output files will be stored in mongodb-data-auto
        File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
        WebPageURLsListing listing = new WebPageURLsListing(mongodb, outFolder);

        System.out.println("*************************************");
        
        
        if(args.length >= 1) {
        File domainsFile = new File(args[0]);
        if(!domainsFile.exists()) {
            System.err.println("File " + domainsFile + " does not exist");
            System.exit(-1);
        }

        //String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
        //String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);


        // TODO: for NZ, do IS_MRI. For overseas still CONTAINS_MRI
        // then also do the shuffle to gen X num of random web page URLs.
        //String filename = listing.webPagesOfAllNZSitesAndDomainListing(domainsFile);
        listing.mriWebPageListingForDomainListing(domainsFile);

        // TODO: generate the special table (6)
        
        } else {        

        // calculating sample size works:
        //System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
        //System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
        
        String filename = listing.writeTentativeNonAutotranslatedSites();
        filename = listing.writeOverseasSitesWithMiInURLPath();

        // TODO: generate the tables

        mongodb.writeTables(outFolder);
        }

        System.out.println("*************************************");
    } catch(Exception e) {
        logger.error(e.getMessage(), e);
    }
    }
}