source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NZTLDProcessor.java@ 33402

package org.greenstone.atea;

import java.io.*;
import java.util.Properties;

import org.apache.log4j.Logger;
import org.greenstone.util.SafeProcess; // from GS3's gutil.jar. For safely running wget/any process from Java.



/**
 * Ensure you have OPENNLP_HOME set to apache-opennlp's full path.
 * 
 * 
 * Compile as:
 *     maori-lang-detection/src$ javac -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar:../lib/*" org/greenstone/atea/NZTLDProcessor.java
 * 
 * Run as:
 *     maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar:../conf:../lib/*" org.greenstone.atea.NZTLDProcessor ../uniq-tld-nz-urls-2019-08-09.txt
 */
public class NZTLDProcessor
{

    static Logger logger = Logger.getLogger(org.greenstone.atea.NZTLDProcessor.class.getName());
    
    Properties configProps = null;

    boolean DEBUG_MODE = true;


    private void log(String msg) {
    System.err.println(msg);
    logger.info(msg);
    }

    private void error(String msg) {
    System.err.println("ERROR: " + msg);
    logger.error(msg);
    }
    
    private void warn(String msg) {
    System.err.println("WARN: " + msg);
    logger.warn(msg);
    }
    
    public NZTLDProcessor(File seedURLsFile) throws Exception { 

    if(DEBUG_MODE) {
        warn("**** Debugger is turned ON!!!");
    }
    
    // TODO: why do I have to provide the path to config.props when this path is already on
    // the classpath?
    // In fact, I don't need it on the classpath for the following to work.
    // How do I get it to work by specifying its path on the classpath and not here?
    //try (FileInputStream infile = new FileInputStream("../lib/config.properties")) {
    //try (InputStream infile = Class.forName("org.greenstone.atea.NZTLDProcessor").getClassLoader().getResourceAsStream("config.properties")) {
    try (InputStream infile = this.getClass().getClassLoader().getResourceAsStream("config.properties")) {
        configProps = new Properties();
        configProps.load(infile);
        infile.close();
        
    } catch(Exception e) {
        System.err.println("Exception attempting to read properties from config.properties.");
        //e.printStackTrace();
        throw e;
    }

    
    try(BufferedReader br = new BufferedReader(new FileReader(seedURLsFile))) {
        String url = null;

        int maxURLs = 1;
        int urlCounter = 0;
        
        while((url = br.readLine()) != null) {
        
        log("Got URL: " + url);

        processURL(url);
        
        if(DEBUG_MODE) {
            urlCounter++;
            if(urlCounter == maxURLs) break;
        }
        
        }

        br.close();
        
    } catch (IOException ioe) {
        System.err.println("Exception attempting to read from seedURLsFile.");
        //ioe.printStackTrace();
        throw ioe;
    }
    }

    public boolean processURL(String nzDomainURL) {
    // launch wget on URL
    // when download done, recurse through downloaded dir
    // for each htm/html/no-extension file: parse out text from paragraphs (TODO: later Tika on non html/htm/no-extension files)
    // For each para, run MaoriTextDetector and store results in db
    // Remove download directory (or only remove if site doesn't contain Maori txt)?


    if(DEBUG_MODE) {
        nzDomainURL = "www.waikato.ac.nz";
    }
    log("Processing seed URL: " + nzDomainURL);

    wgetURL(nzDomainURL);
    
    return true;
    }

    // TODO: Still need to get wget to be launched from a specified directory (../dwn_sites).
    public boolean wgetURL(String nzDomainURL) {
    // use SafeProcess and wget command in lib/config.properties
    // DONE: set up logging

    
    String wgetCmd = configProps.getProperty("wget.cmd");
    if(wgetCmd == null || wgetCmd.equals("")) {
        System.err.println("Invalid or empty wget.cmd in config.properties");
        return false;
    }

    // replace the placeholder in the wget cmd for the seed url
    wgetCmd = wgetCmd.replace("%%BASE_URL%%", nzDomainURL);
    
    log("Will launch wget with the command: " + wgetCmd);

    String[] wgetCommandArgs = wgetCmd.split(" ");
    SafeProcess wgetProcess = new SafeProcess(wgetCommandArgs, null, new File("../dwn_sites"));
    
    SafeProcess.LineByLineHandler outLineHandler = new ProcessLineHandler(logger, SafeProcess.STDOUT);
    SafeProcess.LineByLineHandler errLineHandler = new ProcessLineHandler(logger, SafeProcess.STDERR);
        
    int success = wgetProcess.runProcess(outLineHandler, errLineHandler);

    if(success != 0) {
        System.err.println("Wget cmd \"" + wgetCmd  + "\" returned unsuccessfully with the value \"" + success + "\"");
        return false;
    }
    
    return true;
    }

    public boolean processDownloadedDir() {
    // recursion then process each file

    return true;
    }

    public boolean processFile() {
    // skip if js, css, robots.txt
    // if no-extension or htm or html, call processHTMLFile()
    // else: skip for now, TODO: later call Tika on other file types

    // TODO: update db with url info, page location and para?
    
    return true;
    }


    public boolean processHTMLFile() {
    // Use JSoup to get paras

    // call processParas(text);

    return true;
    }

    public boolean processNonHTMLFile() {
    // Use Tika to get text 

    // call processParas(text)

    return true;
    }

    public boolean processParas(String text) {

    // Split on (double) new line for paras? If too short, then add text from next para unless there's no more. Else add to previous?
    
    // for each para, call MaoriTextDetector.java

    return true;
    }
    
    // ----------- inner class for SafeProcess to Run Wget ----------
    public class ProcessLineHandler extends SafeProcess.LineByLineHandler
    {
    //static Logger logger = Logger.getLogger(this.getClass().getName());;
    Logger logger;
    
    public ProcessLineHandler(Logger logger, int src)
    {
        super(src); // will set this.source to STDERR or STDOUT
        //logger = Logger.getLogger(this.getClass().getName());
        this.logger = logger;
    }

    public void gotLine(String line) { // first non-null line

        // String form of "src" will be "stderr" or "stdout"
        String msg = SafeProcess.streamToString(source) + ": " + line;
        System.err.println(msg);
        //NZTLDProcessor.this.logger.info(msg);
        logger.info(msg);
    }
    public void gotException(Exception e) {
        //NZTLDProcessor.this.logger.error("Error in reading process' " + SafeProcess.streamToString(source), e);
        logger.error("Error in reading process' " + SafeProcess.streamToString(source), e);
    }

    }
    // --------------------------- End inner class --------------------

    public static void main(String[] args) {
    if(args.length <= 0) {
        System.err.println("ERROR: Must provide input file of unique nz website addresses");
        System.exit(-1);
    }
    
    String infilePath = args[0];

    System.err.println("Requested to process URLS file: "  + infilePath);

    File infile = new File(infilePath);
    if(!infile.exists()) {
        System.err.println("ERROR: File " + infilePath + " did not exist");
        System.exit(-1);
    }


    try {
        NZTLDProcessor processor = new NZTLDProcessor(infile);
    } catch(Exception e) {
        e.printStackTrace();
    }
    
    }
}