source: other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java@ 33880

package org.greenstone.atea;

import java.util.*;
import java.io.*;

import org.apache.log4j.Logger;

/**
* TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt 255
 *
*/
public class WebPageURLsListing {
    static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
    
    private final MongoDBAccess mongodbAccess;
    private int numURLs;
    private File domainsFile;


    
    public WebPageURLsListing(MongoDBAccess mongodbAccess,
                    File domainsFile)
    {
    this.mongodbAccess = mongodbAccess;
    this.domainsFile = domainsFile;
    }
    
    public WebPageURLsListing(MongoDBAccess mongodbAccess,
                    File domainsFile,
                    int numURLs)
    {
    this(mongodbAccess, domainsFile);
    this.numURLs = numURLs;
    }

    public String produceURLsForPagesInMRI() {
    return writeFile(MongoDBAccess.IS_MRI);
    }
    
    public String produceURLsForPagesContainingMRI() {
    return writeFile(MongoDBAccess.CONTAINS_MRI);
    }


    public String writeFile(int filterType) {

    ArrayList<String> urlsList = new ArrayList<String>();
    
    // 1. read each url from the domainsFile
    // 1a. do the query
    // 1b. add the arraylist result to urls
    
    try (
         BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
         ) {
        
        
        String domain;
        
        while((domain = reader.readLine()) != null) {
        domain = domain.trim();
        if(!domain.equals("")) {
            ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
            urlsList.addAll(moreURLs);
        }
        }
    } catch(Exception e) {
        logger.error("Unable to read URLs from file " + domainsFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }

    // Shuffle the urlsList, then write out the first numURLs into a file.  
    // https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
    File parentFolder = domainsFile.getParentFile();
    //File outFile = new File(parentFolder, "random"+numURLs+"_"+domainsFile.getName());
    String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
    File outFile = new File(parentFolder, fileName+domainsFile.getName());

    // write out ALL the URLs
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {

        for (int i=0; i < urlsList.size(); i++) {
        String url = urlsList.get(i);
        //System.out.println(list.get(i));
        writer.write(url + "\n");
        }
    } catch(Exception e) {
        logger.error("Unable to write to file " + outFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }

    /*
    // shuffle list and take the first n - write to file
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        Collections.shuffle(urlsList);
        for (int i=0; i<numURLs; i++) {
        String url = urlsList.get(i);
        //System.out.println(list.get(i));
        writer.write(url + "\n");
        }
    } catch(Exception e) {
        logger.error("Unable to write to file " + outFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }
    */

    return outFile.getAbsolutePath();
    }

    /* ---------------------------------------- */

    /** 
     * Create the file 
     * @return full path of file generated
     */
    public String writeTentativeNonAutotranslatedSites() {
    File outFolder = new File("../mongodb-data/").getAbsoluteFile();
    File outFile = new File(outFolder, "5counts_tentativeNonAutotranslatedSites.json");

    String filename = outFile.getAbsolutePath();
    
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        // first write out NZ sites and .nz TLD count and domains
        mongodbAccess.aggregateContainsMRIForNZ(writer);
        // next write out all overseas sites and .nz TLD count and domains
        mongodbAccess.aggregateContainsMRIForOverseas(writer);

        filename = outFile.getCanonicalPath();
    } catch(Exception e) {
        logger.error("Unable to write to file " + outFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }

    return filename;
    }
    
    
    public static void printUsage() {
    System.err.println("WebPageURLsListing [domains.txt]");
    }

    // Depending on args, generates isMRI and containsMRI file listings for:
    // 1. UNFILTERED: all (NZ + overseas) AND takes manually curated domain list file for overseas and adds all NZ

    
    public static void main(String args[]) {
    if(args.length >= 2) {
        printUsage();
        System.exit(-1);
    }

    try (
         MongoDBAccess mongodb = new MongoDBAccess();
         ) {
        File domainsFile = new File(args[0]);
        if(!domainsFile.exists()) {
        System.err.println("File " + domainsFile + " does not exist");
        System.exit(-1);
        }

        //int genNumURLs = Integer.parseInt(args[1]);

        mongodb.connectToDB();
        
        WebPageURLsListing listing = new WebPageURLsListing(mongodb, domainsFile);
        //String isMRIFile = listing.produceURLsForPagesInMRI();
        //String containsMRIFile = listing.produceURLsForPagesContainingMRI();
        String filename = listing.writeTentativeNonAutotranslatedSites();
        System.err.println("Check file: " + filename);

        
    } catch(Exception e) {
        logger.error(e.getMessage(), e);
    }
    }
}