source: other-projects/maori-lang-detection/src/org/greenstone/atea/ManualURLInspection.java@ 33941

package org.greenstone.atea;

import java.util.*;
import java.io.*;

import org.apache.commons.csv.*;
import org.apache.log4j.Logger;

import org.greenstone.util.SafeProcess;

/**
 * Program to help going through the n number of random sample web page URLs stored in input
 * csv file, to eyeball whether the full text (stored in mongodb for each) is indeed inMRI
 * or not. User can enter Y|N|? and ctr-D or ctrl-C to continue working on this later.
 * The output file is the input filename + .tmp suffix.
 * When user continues later, the output file from last time must be used as input file.
 * Any csv records not completed earlier or with ? entered will be presented for input
 * on (re-)running this program.
 *
 * TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/ManualURLInspection.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*" org/greenstone/atea/ManualURLInspection ../mongodb-data/random260_manualList_globalDomains_whereAPageContainsMRI.txt
 *
*/
public class ManualURLInspection {
    
    static Logger logger = Logger.getLogger(org.greenstone.atea.ManualURLInspection.class.getName());

    private final MongoDBQueryer mongodbQueryer;
    private final File outFolder;
    private final File webPageURLsCSVFile;
    private final File tmpOutFile;

    /** csv column numbers */
    public static final int URL_COLUMN = 0;
    public static final int COUNTRY_CODE_COLUMN = 1;
    public static final int IS_REALLY_IN_MRI_COLUMN = 2;

    
    public ManualURLInspection(MongoDBQueryer mongodbQueryer, File csvFile)
    {
    this.mongodbQueryer = mongodbQueryer;
    this.webPageURLsCSVFile = csvFile;
    this.outFolder = csvFile.getParentFile();
    
    String tmpFilename = Utility.getFilePath(webPageURLsCSVFile);
    this.tmpOutFile = new File(tmpFilename+".tmp");
    }

    public String getCSVOutputFilename() {
    return (tmpOutFile == null) ? "" : Utility.getFilePath(tmpOutFile);
    }
    
    /**
     * Read .csv input file one line at a time.
     * For each line, 
     * - if empty line empty, skip it.
     * - If the 3rd column of line is already filled in with Y|N, write out identical line
     * into tmp output file. 
     * - If third column contains ? or if 3rd column is empty, run a MongoDBQuery to get
     * the full text of the page and display it on screen.
     * Wait for user input.
     * - If Enter hit or Y input, write out Y in 3rd field of line into tmp file.
     * - If N or ? entered, write out N/? as 3rd field.
     * 
     * Loop through input csv until finished or until Ctrl-C or Ctrl-D pressed.
     * Ctrl-D here means end of all user interaction, signalling user wants
     * to stop entering data and continue later.
     * 
     * When finished or Ctrl-D entered or Ctrl C pressed, all data entered must have been written
     * out. So to avoid losing data on Ctrl-Ck, write out each processed csv record (whether
     * already complete or whether user entry made it complete) and flush writer.
     * When program terminates in any manner, print message that the file has been created.
    */    
    public String processCSV() {    
    
    final String USER_PROMPT = "Enter isMRI value of Y|N|? for (%d): %s - %s > ";
        //"Enter isMRI value of Y|N|? for (" + count + "): " + url + " - " + countryCode + " > ";
    
    boolean terminate = false;
    CSVParser parser = null;
    
    try {
        parser = CSVParser.parse(webPageURLsCSVFile, java.nio.charset.Charset.forName("US-ASCII"), CSVFormat.RFC4180);
    } catch(Exception e) {
        logger.error("Failed to parse input CSV file " + Utility.getFilePath(webPageURLsCSVFile), e);
        return "Failed";
    }
    
    try (        
         //BufferedWriter writer = new BufferedWriter(new FileWriter(tmpOutFile));
         CSVPrinter csvWriter = new CSVPrinter(new FileWriter(tmpOutFile), CSVFormat.DEFAULT.withQuoteMode(QuoteMode.MINIMAL));
         ) {

        int recordCount = 0;
        for (CSVRecord csvRecord : parser) {        
        //if(terminate) condition handled further below
        
        //logger.debug("Got record: " + csvRecord.toString());
        
        //int recordNo = csvRecord.RecordNumber(); // will count empty lines!

        //if(csvRecord.size() != 0) {
        String url = csvRecord.get(URL_COLUMN);
        if(url.equals("")) { // skip empty lines
            continue;
        }
        
        recordCount++;
        String countryCode = csvRecord.get(COUNTRY_CODE_COLUMN);
        String isReallyInMRI = "";
        //String isReallyInMRI = csvRecord.get(IS_REALLY_IN_MRI_COLUMN);
        //if(!isReallyInMRI.equals("")) {
        if(csvRecord.isSet(IS_REALLY_IN_MRI_COLUMN)) {
            isReallyInMRI = csvRecord.get(IS_REALLY_IN_MRI_COLUMN);
        }
        
        if(terminate || (!isReallyInMRI.equals("") && !isReallyInMRI.equals("?"))) {
            // if(terminate) on Ctrl-D, don't stop processing csv records
            // Instead, copy remaining records of input csv file into output csv file
            isReallyInMRI = isReallyInMRI.toUpperCase();
            csvWriter.printRecord(url, countryCode, isReallyInMRI);
            csvWriter.flush();
            logger.info("Got record " + recordCount + ": " + url + " - " + countryCode
                + " - " + isReallyInMRI);
        }
        else {
            
            // First, display full text for web page record with matching url
            // so the user can look at it to decide whether it is indeed overall in MRI or not.
            String fulltext = mongodbQueryer.displayFullTextOfPage(url);
            System.err.println(String.format("FULL-TEXT for record %d:\n%s\n", recordCount, fulltext));
            
            //logger.info("Got record " + recordCount + ": " + url + " - " + countryCode);
            
            // Read Input until Ctrl-D: read System.In as bufferedReader
            // https://stackoverflow.com/questions/5837823/read-input-until-controld
            // Ctrl-C is already taken care if, see
            // https://coderanch.com/t/279136/java/terminated-program-Control-close-open
            // "Whenever a process is terminated/killed(CTRL-C), the file descriptors are released. You really do not need to close the stream in such cases."
            // So I just need to flush the csv print writer after every record is written
            // and Ctrl-C won't lose any of the data thus far entered by the user.
            
            BufferedReader systemIn = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));            
            
            boolean done = false;           
            
            System.out.println(String.format(USER_PROMPT, recordCount, url, countryCode));
            boolean previouslyQuestionMark = false;
            if(isReallyInMRI.equals("?")) {
            previouslyQuestionMark = true;
            System.err.println("\t? entered last time");
            }
            while(!done && ((isReallyInMRI = systemIn.readLine()) != null)) {
            
            isReallyInMRI = isReallyInMRI.toUpperCase();
            
            //logger.debug("@@ Got: |" + isReallyInMRI + "|");

            // if user hit enter, it means they accepted
            // - that the full text displayed is really in MRI: Y
            // - the previous value entered if it was a ?
            if(isReallyInMRI.equals("")) {
                if(previouslyQuestionMark) {
                isReallyInMRI = "?";
                } else {
                isReallyInMRI = "Y";
                }
            }           
            
            
            if(isReallyInMRI.equals("Y") || isReallyInMRI.equals("N") || isReallyInMRI.equals("?")) {
                done = true;
                //break;
            } else {
                System.out.println("@@ UNRECOGNISED. "
                   + String.format(USER_PROMPT, recordCount, url, countryCode));
            }
            }
            
            // save the record
            csvWriter.printRecord(url, countryCode, isReallyInMRI);
            csvWriter.flush();
            
            if(isReallyInMRI == null) {
            terminate = true;
            System.out.println("User entered Ctrl-D (Lin)/Ctrl-Z (Win) - terminating.");
            } else {
            System.out.println("User entered: " + isReallyInMRI);
            }
            
        }
        }
        //}
        
    } catch(Exception e) {
        e.printStackTrace();
        logger.error("Exception occurred when processing CSV file or writing out file:\n"
             + Utility.getFilePath(tmpOutFile));
        logger.error(e.getMessage(), e);
    }

    //return urlsList;
    return Utility.getFilePath(tmpOutFile);
    }
    
    
    
    public static void printUsage() {
    System.err.println("Usage: ManualURLInspection webPageURLs.txt");
    }

    /**
     * If no args are passed in, generates complete containsMRI file listings for NZ and overseas web SITES (domains),
     * with overseas web sites that have mi (mi.* or *\/mi) in the URL path listed separately.
     * You can then manually inspect the domains in this listing to shortlist which of these sites are not automatically
     * translated and really contain at least one webpage containing at least one sentence in MRI.
     * If a file is passed in containing a list of domains, then this first generates a full listing of all webpages 
     * matching isMRI for each site in the domain list. It then generates a smaller set of random webpages matching
     * isMRI for the pooled sites in the domain list where the sample size of URLs produced is sufficient for giving
     * 90% confidence with 5% margin of error for testing binary outcomes, see
     * https://stats.stackexchange.com/questions/207584/sample-size-choice-with-binary-outcome
     */
    public static void main(String args[]) {
    SafeProcess.DEBUG = 1;
    
    if(args.length != 1) {
        printUsage();
        System.exit(-1);
    }   
    
        
    try (
         MongoDBQueryer mongodb = new MongoDBQueryer();
         ) {

        mongodb.connectToDB();

        // output files will be stored in mongodb-data-auto
        File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
        

        logger.info("*************************************");
        
        
        
        final File inputFile = new File(args[0]);
        if(!inputFile.exists()) {
        logger.info("File " + inputFile + " does not exist");
        System.exit(-1);
        }

        final ManualURLInspection inspector = new ManualURLInspection(mongodb, inputFile);


        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        public void run() {
            logger.info("@@@@@@@@@@@@@@@@@@@@@@@@");
            logger.info("WARNING!!!");
            logger.info("Got Ctrl-C. INCOMPLETE generated temp CSV file: " +
                inspector.getCSVOutputFilename());
            logger.info(String.format("Copy remaining records from input file %s into this file.",
                          Utility.getFilePath(inputFile)));
            logger.info("@@@@@@@@@@@@@@@@@@@@@@@@");
        }
        }));
        
        String filename = inspector.processCSV();
        
        logger.info("Generated temp CSV file: " + filename);
        logger.info("*************************************");
    } catch(Exception e) {
        logger.error(e.getMessage(), e);
    }
    }
}