source: other-projects/maori-lang-detection/src/org/greenstone/atea/SummaryTool.java@ 33925

package org.greenstone.atea;

import java.util.*;
import java.io.*;

import org.apache.log4j.Logger;

import org.greenstone.util.SafeProcess;

/**
 * Runs some of the important mongoDB queries I ran.
 *
 * This program expects a folder ../mongo-data-auto to exist.
 *
 * TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/SummaryTool.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/SummaryTool
 * OR:
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/SummaryTool ../mongodb-data/domainsNZ_IsMRI.txt
 *
*/
public class SummaryTool {
    
    static Logger logger = Logger.getLogger(org.greenstone.atea.SummaryTool.class.getName());

    
    static private final String GEOJSON_FEATURES_FILE_PREFIX = "geojson-features_";
    
    static private final long FIXED_SEED = 1000;
    
    private final MongoDBQueryer mongodbQueryer;
    private File outFolder;

    
    public static class Tuple {
    public final String url;
    public final String countryCode;
    
    public Tuple(String url, String countryCode) {
        this.url = url;
        this.countryCode = countryCode;
    }

    public String toString() {
        return this.url + "," + countryCode;
    }
    }
    
    
    public SummaryTool(MongoDBQueryer mongodbQueryer, File outFolder)
    {
    this.mongodbQueryer = mongodbQueryer;
    this.outFolder = outFolder;
    }

    
    public void produceURLsForPagesInMRI(File domainsFile) {
    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
    File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
    writeURLsToFile(urlsList, outFile, urlsList.size());
    
    System.out.println("Wrote all isMRI web page URLs for the sites in input domainsFile\ninto file: "
               + Utility.getFilePath(outFile));
    }
    
    public void produceURLsForPagesContainingMRI(File domainsFile) {
    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.CONTAINS_MRI, domainsFile);   
    File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
    writeURLsToFile(urlsList, outFile, urlsList.size());

    System.out.println("Wrote all containsMRI web page URLs for the sites in input domainsFile\ninto file: "
               + Utility.getFilePath(outFile));
    }
    
    private ArrayList<Tuple> getURLsForAllWebPagesInSiteListing(int filterType, File domainsFile) {
    ArrayList<Tuple> urlsList = new ArrayList<Tuple>();
    
    // 1. read each url from the domainsFile
    // 1a. do the query
    // 1b. add the arraylist result to urls
    
    try (
         BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
         ) {        
        
        String domain;
        
        while((domain = reader.readLine()) != null) {
        domain = domain.trim();
        if(!domain.equals("")) {
            
            String countryCode = "";
            int index = domain.lastIndexOf(",");
            if(index != -1) {
            countryCode = domain.substring(index+1).trim();
            domain = domain.substring(0, index);
            }
            ArrayList<String> moreURLs = mongodbQueryer.queryAllMatchingURLsFilteredBy(domain, filterType);

            // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
            if(moreURLs.size() == 0 && filterType == MongoDBQueryer.IS_MRI) {
            System.out.println("   " + countryCode + " domain " + domain + " had no webpages where isMRI=true - only containsMRI.");
            }

            //urlsList.addAll(moreURLs);
            for(int i = 0; i < moreURLs.size(); i++) {
            urlsList.add(new Tuple(moreURLs.get(i), countryCode));
            }
            
        }
        }
        System.err.println("");
    } catch(Exception e) {
        logger.error("Unable to read URLs from file " + Utility.getFilePath(domainsFile));
        logger.error(e.getMessage(), e);
    }

    return urlsList;
    }
    
    /** Given a hand curated list of all sites with positive numPagesContainingMRI
     * determined by manual inspection, get a listing of all their web pages that
     * are IN_MRI (or CONTAINS_MRI?). 
     * Total all these pages that are inMRI (N), then work out the correct sample size (n)
     * at 90% confidence with 5% margin of error. Then generate a random listing
     * of n of these pages in MRI of these trusted sites and output to a file
     * for manual inspection of the sample webpage URLs at page-level. */
    /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
     * of all their web pages IN_MRI (or CONTAINS_MRI).
     * Plus a listing of all the NZ pages IN_MRI. */
    //public void webPagesOfAllNZSitesAndDomainListing(File domainsFile) {
    public void mriWebPageListingForDomainListing(File domainsFile) {

    int filterType = MongoDBQueryer.IS_MRI;
    
    // for overseas websites, 
    //produceURLsForPagesContainingMRI(handCuratedOverseasDomainsFile);

    // 0. get a list of all the web pages in the given domain listing where isMRI = true
    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
        // produceURLsForPagesInMRI(domainsFile);
    
    // 1. calculate the population size, N, the number of all webpages in the given domain
    // site listing where isMRI = true.
    int N_totalNumPages = urlsList.size();

    // 2. write all the URLs in urlsList to a file
    //File outFolder = domainsFile.getParentFile();
    String filterName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI" : "containsMRI";
    File outFile = new File(outFolder, filterName+"_full_"+domainsFile.getName());

    writeURLsToFile(urlsList, outFile, N_totalNumPages);
    System.out.println("Wrote out full listing of web page URLs for sites in input domainsFile"
               + "\ninto file: " + Utility.getFilePath(outFile));
        
    // 3. calculate sample size n for population size N if using 90% confidence and 5% margin of error
    int n_numSampleURLs = calcSampleSize(N_totalNumPages);

    System.err.println("*** N, total number of web pages for which " + filterName + "=true from domain shortlist file: " + N_totalNumPages);
    System.err.println("    (out of " + mongodbQueryer.countOfWebpagesMatching(filterType)
               + " web pages across ALL sites for which " + filterName + " = true)");
    System.err.println("*** n, sample size of web page URLs: " + n_numSampleURLs);
    
    // 4. Shuffle all the URLs and write the first n (sample size) URLs to a file
    // Using a constant seed for reproducibility
    // https://stackoverflow.com/questions/6284589/setting-a-seed-to-shuffle-arraylist-in-java-deterministically
    Collections.shuffle(urlsList, new Random(FIXED_SEED)); 
    
    outFile = new File(outFolder, "random"+n_numSampleURLs+"_"+domainsFile.getName());
    writeURLsToFile(urlsList, outFile, n_numSampleURLs);
    System.out.println("Wrote a sample of n=" + n_numSampleURLs + " of web page URLs "
               + "for the sites in input domainsFile\ninto file: " + Utility.getFilePath(outFile));

    // For N = 6557, z-alpha-over-2 = 1.6449 and m = 0.05 (5%),
    // n = (z-alpha-over-2^2 x N) / (z-alpha-over-2^2 + 4 x (N-1) x m^2)
    // = (1.6449^2×6557) ÷ (1.6449^2 + 4 × 6556×0.05^2) = 259.88526851 => 260 rounded up. Check.
    }

    /**
     * Calculates sample size n for binary outcomes at 90% confidence and 5% margine of error
     * for given population size N.
     * @return n, the sample size.
     */
    public int calcSampleSize(int N) {
    
    // calculate sample size n for population size N if using 90% confidence and 5% margin of error
    // https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
    // https://www.statisticshowto.datasciencecentral.com/probability-and-statistics/find-sample-size/#CI1
    // https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
    
    double m = 0.05; // margin of error = 5%
    // for 90% confidence, alpha is the remainder = 10% and alpha/2 = 5%.
    // For 90% confidence, use the table of known z_alpha/2 values from step 1 of
    // https://www.statisticshowto.datasciencecentral.com/z-alpha2-za2/
    double z_alpha_over_2 = 1.6449;

    // Formula: n = (zalpha2 ^ 2 * N) / ((z-alpha-2 ^ 2) + 4(N-1)*m^2)
    // see https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
    double n = (Math.pow(z_alpha_over_2, 2.0) * N) / (Math.pow(z_alpha_over_2, 2.0) + (4 * (N-1) * Math.pow(m,2.0)));

    // Round up to get a whole number:
    return (int)Math.ceil(n);
    }
    
    /**
     * Writes out the first n URLs in urlsList into outFile.
     */
    private void writeURLsToFile(ArrayList<Tuple> urlsList, File outFile, final int n) {
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {

        for (int i=0; i < n; i++) {
        Tuple urlInfo = urlsList.get(i);
        
        //System.out.println(list.get(i));
        writer.write(urlInfo + "\n"); // calls toString() on tuple of url -> countryCode
        }
    } catch(Exception e) {
        logger.error("Unable to write to file " + Utility.getFilePath(outFile));
        logger.error(e.getMessage(), e);
    }
    }

    /* ---------------------------------------- */
    /** 
     * Create the file 5counts_containsMRISites_allNZGrouped.json
     * that contains the count and domains for NZ sites (NZ origin or nz TLD) with pages
     * that CONTAIN_MRI, followed by counts and domains listing for overseas sites
     * that CONTAIN_MRI.
     * @return full path of file generated
     */
    public String writeContainsMRISites_nzSitesAndTLDsGrouped() {
    
    File outFile = new File(outFolder, "5counts_containsMRISites_allNZGrouped.json");

    String filename = Utility.getFilePath(outFile);
    
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        // first write out NZ sites and .nz TLD count and domains
        mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
        // next write out all overseas sites (not NZ origin or .nz TLD)
        // that have no "mi" in the URL path as mi.* or */mi
        boolean isMiInURLPath = false;
        mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI);
        
    } catch(Exception e) {
        logger.error("Unable to write to file " + filename);
        logger.error(e.getMessage(), e);
    }
    
    System.err.println("*** Wrote file: " + filename);

    return filename;
    }
    
    /** 
     * Create the file 5a_counts_tentativeNonAutotranslatedSites.json
     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
     * followed by counts and domain listing for overseas sites that are either from Australia
     * or don't contain mi in their URL path.
     * @return full path of file generated
     */
    public String writeTentativeNonAutotranslatedSites() {
    
    File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");

    String filename = Utility.getFilePath(outFile);
    
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        // first write out NZ sites and .nz TLD count and domains
        mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
        // next write out all overseas sites (not NZ origin or .nz TLD)
        // that have no "mi" in the URL path as mi.* or */mi
        boolean isMiInURLPath = false;
        mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
        
    } catch(Exception e) {
        logger.error("Unable to write to file " + filename);
        logger.error(e.getMessage(), e);
    }
    
    System.err.println("*** Wrote file: " + filename);

    return filename;
    }

    /**
     * Create the file 5b_counts_overseasSitesWithMiInPath.json
     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
     * This listing is separate to allow easier weeding out of product sites/autotranslated
     * sites when eyeballing the listing output.
     */
    public String writeOverseasSitesWithMiInURLPath() {
    File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");

    String filename = Utility.getFilePath(outFile);
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        boolean isMiInURLPath = true;
        mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);

    } catch(Exception e) {
        logger.error("Unable to write to file " + filename);
        logger.error(e.getMessage(), e);
    }

    System.err.println("*** Wrote file: " + filename);
    return filename;
    }
    
    public static void printUsage() {
    System.err.println("Usage: SummaryTool [domains.txt]");
    }

    /**
     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
     * translated and really contain at least one webpage containing at least one sentence in MRI.
     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
     * 90% confidence with 5% margin of error for testing binary outcomes, see
     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
     */
    public static void main(String args[]) {
    if(args.length >= 2) {
        printUsage();
        System.exit(-1);
    }

    try (
         MongoDBQueryer mongodb = new MongoDBQueryer();
         ) {

        mongodb.connectToDB();

        // output files will be stored in mongodb-data-auto
        File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
        SummaryTool listing = new SummaryTool(mongodb, outFolder);

        System.out.println("*************************************");
        
        
        if(args.length >= 1) { // web page urls listing mode
        File domainsFile = new File(args[0]);
        if(!domainsFile.exists()) {
            System.err.println("File " + domainsFile + " does not exist");
            System.exit(-1);
        }

        //String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
        //String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);


        // TODO: for NZ, do IS_MRI. For overseas still CONTAINS_MRI
        // then also do the shuffle to gen X num of random web page URLs.
        //String filename = listing.webPagesOfAllNZSitesAndDomainListing(domainsFile);
        listing.mriWebPageListingForDomainListing(domainsFile);

        // TODO: generate the special table (6)
        
        } else {        

        // calculating sample size works:
        //System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
        //System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));

        // get all sites where >0 pages have containsMRI=true
        // grouping NZ sites and .nz TLDs together and remainder under overseas
        // geolocations.
        String filename = listing.writeContainsMRISites_nzSitesAndTLDsGrouped();

        // separately:
        // - all NZ containsMRI + overseas tentative non-product sites with containMRI
        // - overseas tentative product sites with containMRI
        filename = listing.writeTentativeNonAutotranslatedSites();
        filename = listing.writeOverseasSitesWithMiInURLPath();

        // TODO: generate the tables

        String[] tableFileNames = mongodb.writeTables(outFolder);
        // for each table file name, generate the geojson-features .json file
        // that GEOJSON_MAP_TOOL_URL takes as input to produce a map.

        for(int i = 1; i < tableFileNames.length; i++) { // empty element at 0
            String tablefilename = tableFileNames[i] + ".json"; // filenames have no suffix
            
            File countsTableFile = new File(outFolder, tablefilename);
            if(!countsTableFile.exists()) {
            logger.error("@@@ File " + countsTableFile + " does not exist!");
            logger.error("@@@ Can't generate map date for this.");
            continue;
            }
            String countsTableFilename = outFolder + File.separator + tablefilename;
            CountryCodeCountsMapData mapData
            = new CountryCodeCountsMapData(countsTableFilename);
            String geoJsonFilename = mapData.writeFeaturesGeoJsonToFile(CountryCodeCountsMapData.SUPPRESS_MAPDATA_DISPLAY);
            

            /*
            // Ensure the geo-json file generated exists
            //String geoJsonFilename = outFolder + File.separator
            //+ GEOJSON_FEATURES_FILE_PREFIX + tablefilename;
            
            File geoJsonFile = new File(geoJsonFilename);
            if(!geoJsonFile.exists()) {
            System.err.println("@@@ geoJson file " + geoJsonFilename + " not generated!");
            continue;
            }
            */
            System.err.println("**** Wrote mapdata to file " + geoJsonFilename);
            //System.err.println("     Paste the file's geojson contents into "
            //+ "the editor at " + CountryCodeCountsMapData.GEOJSON_MAP_TOOL_URL
            //+ " to see the data arranged on a world map");            
            System.err.println("Total count for query: " + mapData.getTotalCount());
            
            // Running the command:
            //    python -mjson.tool file.json
            // to confirm if the JSON in the provided json file parses/is valid JSON.

            String[] cmdArgs = {"python", "-mjson.tool", geoJsonFilename};
            SafeProcess p = new SafeProcess(cmdArgs);
            int retVal = p.runProcess();
            logger.info("\nJson validation tool returned with value: " + retVal);
            if(retVal == 0) {
            logger.info("Valid geo json.");
            } else {
            logger.error("   std out: " + p.getStdOutput());
            logger.error("   std err: " + p.getStdError());
            logger.error("INVALID geo json generated. Not attempting screenshot of map.");
            continue;
            }

            
            /*boolean uriEncoded = true;
            String mapDataEncodedStr = mapData.getFeaturesGeoJsonString(uriEncoded);
            System.err.println("Encoded string: " + mapDataEncodedStr);
            */
            
            System.err.println("Data URL string: " + mapData.getAsMapURL());
            System.err.println();
            mapData.geoJsonMapScreenshot(outFolder, tablefilename);
            System.err.println("---");

            // TODO: breaks after first table -> map conversion
            break;
        }
        
        }

        System.out.println("*************************************");
    } catch(Exception e) {
        logger.error(e.getMessage(), e);
    }
    }
}