source: other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java@ 33873

package org.greenstone.atea;

import java.util.*;
import java.io.*;

import org.apache.log4j.Logger;

/**
* TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt  255
 *
*/
public class WebPageURLsListing {
    static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
    
    private final MongoDBAccess mongodbAccess;
    private final int numURLs;

    //private String[] urls;
    ArrayList<String> urlsList = new ArrayList<String>();
    
    public WebPageURLsListing(MongoDBAccess mongodbAccess,
                    File domainsFile,
                    int numURLs)
    {
    this.mongodbAccess = mongodbAccess;
    this.numURLs = numURLs;

    // 1. read each url from the domainsFile
      // 1a. do the query
      // 1b. add the arraylist result to urls

    try (
         BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
         ) {
        
        
        String domain;
        
        while((domain = reader.readLine()) != null) {
        domain = domain.trim();
        if(!domain.equals("")) {
            ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingIsMRIURLs(domain);
            urlsList.addAll(moreURLs);
        }
        }
    } catch(Exception e) {
        logger.error("Unable to read URLs from file " + domainsFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }

    // Shuffle the urlsList, then write out the first numURLs into a file.  
    // https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
    File parentFolder = domainsFile.getParentFile();
    String fileName = domainsFile.getName();
    //File outFile = new File(parentFolder, "random"+numURLs+"_"+fileName);
    File fullSetOutFile = new File(parentFolder, "allPages_"+fileName);

    // write out ALL the URLs
    try (
         Writer writer = new BufferedWriter(new FileWriter(fullSetOutFile));
         ) {

        for (int i=0; i < urlsList.size(); i++) {
        String url = urlsList.get(i);
        //System.out.println(list.get(i));
        writer.write(url + "\n");
        }
    } catch(Exception e) {
        logger.error("Unable to write to file " + fullSetOutFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }

    /*
    // shuffle list and take the first n - write to file
    try (
         Writer writer = new BufferedWriter(new FileWriter(outFile));
         ) {
        Collections.shuffle(urlsList);
        for (int i=0; i<numURLs; i++) {
        String url = urlsList.get(i);
        //System.out.println(list.get(i));
        writer.write(url + "\n");
        }
    } catch(Exception e) {
        logger.error("Unable to write to file " + outFile.getAbsolutePath());
        logger.error(e.getMessage(), e);
    }
    */
    }
    
    public static void printUsage() {
    System.err.println("WebPageURLsListing [domains.txt]");
    }

    // Depending on args, generates isMRI and containsMRI file listings for:
    // 1. UNFILTERED: all (NZ + overseas) AND takes manually curated domain list file for overseas and adds all NZ

    
    
    public static void main(String args[]) {
    if(args.length >= 2) {
        printUsage();
        System.exit(-1);
    }

    try (
         MongoDBAccess mongodb = new MongoDBAccess();
         ) {
        File domainsFile = new File(args[0]);
        if(!domainsFile.exists()) {
        System.err.println("File " + domainsFile + " does not exist");
        System.exit(-1);
        }

        int genNumURLs = Integer.parseInt(args[1]);

        mongodb.connectToDB();
        
        WebPageURLsListing listing = new WebPageURLsListing(mongodb, domainsFile);
        
    } catch(Exception e) {
        logger.error(e.getMessage(), e);
    }
    }
}