package org.greenstone.atea;

import java.util.*;
import java.io.*;

import org.apache.log4j.Logger;

/**
* TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/WebPageURLsListing ../mongodb-data/domainsNZ_IsMRI.txt 255
 *
*/
public class WebPageURLsListing {
    static Logger logger = Logger.getLogger(org.greenstone.atea.WebPageURLsListing.class.getName());
    
    private final MongoDBAccess mongodbAccess;
    private File outFolder;

    public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
    {
	this.mongodbAccess = mongodbAccess;
	this.outFolder = outFolder;
    }
    
    public String produceURLsForPagesInMRI(File domainsFile) {
	return writeFile(MongoDBAccess.IS_MRI, domainsFile);
    }
    
    public String produceURLsForPagesContainingMRI(File domainsFile) {
	return writeFile(MongoDBAccess.CONTAINS_MRI, domainsFile);
    }


    public String writeFile(int filterType, File domainsFile/*, int numURLs*/) {

	ArrayList<String> urlsList = new ArrayList<String>();
	
	// 1. read each url from the domainsFile
	// 1a. do the query
	// 1b. add the arraylist result to urls
	
	try (
	     BufferedReader reader = new BufferedReader(new FileReader(domainsFile));
	     ) {
	    
	    
	    String domain;
	    
	    while((domain = reader.readLine()) != null) {
		domain = domain.trim();
		if(!domain.equals("")) {
		    ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
		    urlsList.addAll(moreURLs);
		}
	    }
	} catch(Exception e) {
	    logger.error("Unable to read URLs from file " + domainsFile.getAbsolutePath());
	    logger.error(e.getMessage(), e);
	}

	// Shuffle the urlsList, then write out the first numURLs into a file.	
	// https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
	File parentFolder = domainsFile.getParentFile();
	//File outFile = new File(parentFolder, "random"+numURLs+"_"+domainsFile.getName());
	String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
	File outFile = new File(parentFolder, fileName+domainsFile.getName());

	// write out ALL the URLs
	try (
	     Writer writer = new BufferedWriter(new FileWriter(outFile));
	     ) {

	    for (int i=0; i < urlsList.size(); i++) {
		String url = urlsList.get(i);
		//System.out.println(list.get(i));
		writer.write(url + "\n");
	    }
	} catch(Exception e) {
	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
	    logger.error(e.getMessage(), e);
	}

	/*
	// shuffle list and take the first n - write to file
	try (
	     Writer writer = new BufferedWriter(new FileWriter(outFile));
	     ) {
	    Collections.shuffle(urlsList);
	    for (int i=0; i<numURLs; i++) {
		String url = urlsList.get(i);
		//System.out.println(list.get(i));
		writer.write(url + "\n");
	    }
	} catch(Exception e) {
	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
	    logger.error(e.getMessage(), e);
	}
	*/

	return outFile.getAbsolutePath();
    }

    /* ---------------------------------------- */

    /** 
     * Create the file 5counts_tentativeNonAutotranslatedSites.json
     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
     * followed by counts and domain listing for overseas sites that are either from Australia
     * or don't contain mi in their URL path.
     * @return full path of file generated
     */
    public String writeTentativeNonAutotranslatedSites() {
	
	File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");

	String filename = outFile.getAbsolutePath();
	
	try (
	     Writer writer = new BufferedWriter(new FileWriter(outFile));
	     ) {
	    // first write out NZ sites and .nz TLD count and domains
	    mongodbAccess.aggregateContainsMRIForNZ(writer, MongoDBAccess.CONTAINS_MRI);
	    // next write out all overseas sites (not NZ origin or .nz TLD)
	    // that have no "mi" in the URL path as mi.* or */mi
	    boolean isMiInURLPath = false;
	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);

	    filename = outFile.getCanonicalPath();
	} catch(Exception e) {
	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
	    logger.error(e.getMessage(), e);
	}
	
	System.err.println("*** Wrote file: " + filename);

	return filename;
    }

    /**
     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path
     */
    public String writeOverseasSitesWithMiInURLPath() {
	File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");

	String filename = outFile.getAbsolutePath();
	try (
	     Writer writer = new BufferedWriter(new FileWriter(outFile));
	     ) {
	    boolean isMiInURLPath = true;
	    mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
	    filename = outFile.getCanonicalPath();
	} catch(Exception e) {
	    logger.error("Unable to write to file " + outFile.getAbsolutePath());
	    logger.error(e.getMessage(), e);
	}

	System.err.println("*** Wrote file: " + filename);
	return filename;
    }
	
    public static void printUsage() {
	System.err.println("WebPageURLsListing [domains.txt]");
    }

    // Depending on args, generates isMRI and containsMRI file listings for:
    // 1. UNFILTERED: all (NZ + overseas) AND takes manually curated domain list file for overseas and adds all NZ

    
    public static void main(String args[]) {
	if(args.length >= 2) {
	    printUsage();
	    System.exit(-1);
	}

	try (
	     MongoDBAccess mongodb = new MongoDBAccess();
	     ) {

	    mongodb.connectToDB();

	    // output files will be stored in mongodb-data-auto
	    File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
	    WebPageURLsListing listing = new WebPageURLsListing(mongodb, outFolder);

	    
	    if(args.length >= 1) {
		File domainsFile = new File(args[0]);
		if(!domainsFile.exists()) {
		    System.err.println("File " + domainsFile + " does not exist");
		    System.exit(-1);
		}

		
		//int genNumURLs = Integer.parseInt(args[1]);
		//String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
		//String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);

		//listing.writeWebPagesOfAllNZSitesAndDomainListing();
		
	    } else {		

		String filename = listing.writeTentativeNonAutotranslatedSites();
		filename = listing.writeOverseasSitesWithMiInURLPath();
	    }

	    
	} catch(Exception e) {
	    logger.error(e.getMessage(), e);
	}
    }
}