source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java@ 33615

package org.greenstone.atea;

import java.io.*;
import java.lang.ArrayIndexOutOfBoundsException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;

import org.apache.commons.csv.*;
import org.apache.log4j.Logger;


/**
 * Class to process the dump text files produced FOR EACH SITE (e.g. site "00001") that
 * Nutch has finished crawling and whose text has been dumped out to a file called dump.txt.
 * This reads in the dump.txt file contained in each site folder within the input folder.
 * (e.g. input folder "crawled" could contain folders 00001 to 01465. Each contains a dump.txt)
 * Each dump.txt could contain the text contents for an entire site, or for individual pages.
 * This class then uses class TextDumpPage to parse each webpage within a dump.txt,
 * which parses out the actual text body content of each webpage's section within a dump.txt.
 * Finally, MaoriTextDetector is run over that to determine whether the full body text is
 * likely to be in Maori or not.
 *
 * Potential issues: since a web page's text is dumped out by nutch with neither paragraph
 * nor even newline separator, it's hard to be sure that the entire page is in language.
 * If it's in multiple languages, there's no way to be sure there aren't promising Maori language
 * paragraphs contained in a page, if the majority/the remainder happen to be in English.
 * 
 * So if we're looking for any paragraphs in Maori to store in a DB, perhaps it's better to run
 * the MaoriTextDetector.isTextInMaori(BufferedReader reader) over two "lines" at a time,
 * instead of running it over the entire html body's text.
 *
 * TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled-small
 *
 * or:
 *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled-small > ../crawled-small/bla.txt 2>&1
 *
*/
public class NutchTextDumpProcessor {
    static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());

    static boolean DEBUG_MODE = true;

    /** Counter for number of sites.
     * Should be equal to number of times NutchTextDumpProcessor constructor
     * is called: once per site.
     */
    static private int SITE_COUNTER = 0;
    static private long WEBPAGE_COUNTER = 0;
    static private long MRI_SENTENCE_COUNTER = 0;
    
    private final MaoriTextDetector maoriTxtDetector;
    
    public final String siteID;
    public final boolean siteCrawlUnfinished;
    public final long siteCrawledTimestamp; /** When the crawl of the site terminated */
    
    private String domainOfSite;
    
    /** keep a list to store the text of each page */
    private ArrayList<TextDumpPage> pages;

    /** list of pages in this site which were detected as being in MRI */
    private ArrayList<MRIWebPageStats> pagesInMRI;
    /** 
     * list of pages in this site which were NOT detected as being in MRI but nevertheless
     * contain one or more sentences in MRI 
     */
    private ArrayList<MRIWebPageStats> pagesContainingMRI;
    
    private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
    // The start of a new web page's record in nutch's text dump of an entire site
    // is denoted by a newline followed by a URL (protocol)
    // or the very start of the file with a URL (protocol)
    return ((prevLine == null || prevLine.equals(""))
        && (line.startsWith("http://") || line.startsWith("https://")));
    }

    public void debugPageDump(StringBuilder pageDump) {
    if(DEBUG_MODE) {
        // START DEBUG
        logger.debug("__________________________________________");
        logger.debug("@@@ Found page entry: ");
        logger.debug("__________________________________________");
        logger.debug(pageDump.toString());
        logger.debug("------------------------------------------");
        // END DEBUG
    }
    }

    /** A NutchTextDumpProcessor processes the dump.txt for one site */
    public NutchTextDumpProcessor(CSVPrinter webpagesCSVPrinter, CSVPrinter mriSentencesCSVPrinter,
                  MaoriTextDetector maoriTxtDetector, String siteID,
                  File txtDumpFile, long lastModified, boolean siteCrawlUnfinished)
    throws IOException
    {   
    // increment static counter of sites processed by a NutchTextDumpProcessor instance
    SITE_COUNTER++;
    
    // siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
    this.siteID = siteID;
    this.siteCrawlUnfinished = siteCrawlUnfinished;
    this.siteCrawledTimestamp = lastModified;
    
    this.maoriTxtDetector = maoriTxtDetector;
        
    pages = new ArrayList<TextDumpPage>();

    String line = null;
    StringBuilder pageDump = null;
    try (
         BufferedReader reader = new BufferedReader(new FileReader(txtDumpFile));
         ) {

        boolean readingText = false;
        String prevLine = null;
        
        while((line = reader.readLine()) != null) { // readLine removes newline separator
        line = line.trim();
        // iff outside of a page's body text, then an empty line marks the end of a page
        // in nutch's text dump of a site.
        // But note, there can be an empty line (or more?) between the start and end
        // markers of a page's text, though.

        if(isStartOfNewWebPageRecord(prevLine, line)) {

            if(pageDump != null) { // should also be the case then: if(prevLine != null)
            // finish old pageDump and begin new one            

            //debugPageDump(pageDump);
                
            TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
            // parses the fields and body text of a webpage in nutch's txt dump of entire site
            //page.parseFields();
            //page.getText();
            pages.add(page);
            pageDump = null;

            }           

            // begin new webpage dump
            pageDump = new StringBuilder();
            pageDump.append(line);
            pageDump.append("\n");
            
        }
        else if(!line.equals("")) {
            pageDump.append(line);
            pageDump.append("\n");

        }
        // can throw away any newlines between text start and end markers.
        
        prevLine = line;        
        }

        // process final webpage record:
        //debugPageDump(pageDump);
        
        TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
        pages.add(page);
        pageDump = null;
        
    } catch (IOException ioe) {
        logger.error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
    }

    // Just do this once: get and store domain of site.
    // Passing true to get domain with protocol prefix
    if(pages.size() > 0) {
        TextDumpPage firstPage = pages.get(0);
        String url = firstPage.getPageURL();
        this.domainOfSite = CCWETProcessor.getDomainForURL(url, true);
    }
    else {
        this.domainOfSite = "UNKNOWN";
    }
    
    prepareSiteStats(webpagesCSVPrinter, mriSentencesCSVPrinter);
    }
    
    /** pageID: id into pages array */
    public boolean isPageInMaori(int pageID) throws ArrayIndexOutOfBoundsException {    

    String text = getTextForPage(pageID);
    
    // QTODO: what to do when page body text is empty?
    if(text.equals("")) return false;
    return maoriTxtDetector.isTextInMaori(text);
    }

    private TextDumpPage getPage(int pageID) throws ArrayIndexOutOfBoundsException {
    if(pageID < 0 || pageID >= pages.size()) {
        throw new ArrayIndexOutOfBoundsException();
    }

    TextDumpPage page = pages.get(pageID);
    return page;
    }

    public String getTextForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageText();  
    }
    public String getURLForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageURL();
    }   
       
    public int totalNumPages() {
    return pages.size();
    }
    public int getNumPagesInMRI() {
    return pagesInMRI.size();
    }    
    public int getNumPagesContainingMRI() {
    return pagesContainingMRI.size();
    }
    
    private void prepareSiteStats(CSVPrinter webpageCSVPrinter, CSVPrinter mriSentencesCSVPrinter) throws IOException {
    pagesInMRI = new ArrayList<MRIWebPageStats>();
    pagesContainingMRI = new ArrayList<MRIWebPageStats>();

    TextDumpPage page = null;
    for(int i = 0; i < pages.size(); i++) {
        
        page = pages.get(i);
        
        String text = page.getPageText();
        
        if(text.equals("")) {
        page.addMRILanguageStatus(false);
        continue;
        }
        else {
        boolean isMRI = maoriTxtDetector.isTextInMaori(text);
        
        page.addMRILanguageStatus(isMRI);

        // Even if the entire page is not found to be overall in Māori,
        // let's still inspect the sentences of the page and count how many (if any)
        // are in te reo.
        ArrayList<String> mriSentences = maoriTxtDetector.getAllSentencesInMaori(text);
        // first element of ArrayList returned is always total num sentences on page
        // remaining elements are the actual sentences that were detected as being Māori
        int totalSentences = Integer.parseInt(mriSentences.get(0));
        int numSentencesInMRI = mriSentences.size() - 1;        

        // Add page to list of MRI pages if the page's body text overall was detected
        // as Māori
        // Add page to list of pages containing MRI if >= 1 sentences in the page
        // were detected as being in MRI
        if(isMRI || numSentencesInMRI >= 1) {
            String url = page.getPageURL();
            MRIWebPageStats MRIpageStats = new MRIWebPageStats(this.siteID, url, i, isMRI,
                           totalSentences, numSentencesInMRI);
            if(isMRI) {   
            pagesInMRI.add(MRIpageStats);
            } else if(numSentencesInMRI >= 1) {
            pagesContainingMRI.add(MRIpageStats);
            }

            // Only write to webpages csv file for those pages that had any MRI
            // language content.
            // column headers:
            //    pageID, siteID, URL, isMRI, numSentences, numSentencesInMRI
            //int pageID = i; // not primary key by itself,
                    // must be combined with siteID to form primary key
            
            webpageCSVPrinter.printRecord(WEBPAGE_COUNTER++,
                          SITE_COUNTER, /* alternative: this.siteID */
                          url, isMRI, totalSentences, numSentencesInMRI);

            // Write the sentences that are in te reo into the mri-sentences CSV file
            // whether from webpages that are MRI overall or only those that containing
            // any sentences in MRI
            for (int j = 1; j < mriSentences.size(); j++) { // 1st element not a sentence
            //int sentenceID = j; // combine with siteID and pageID to form primary key
            String mriSentence = mriSentences.get(j);
            // sentenceID, pageID, sentence
            //mriSentencesCSVPrinter.printRecord(sentenceID, pageID, mriSentence);
            mriSentencesCSVPrinter.printRecord(MRI_SENTENCE_COUNTER++, WEBPAGE_COUNTER, mriSentence);
            }
        }       

        }       
    }
    }
    
    public void printSiteStats() {
    
    
    logger.info("------------- " + this.siteID + " SITE STATS -----------");

    logger.info("SITE DOMAIN: " + this.domainOfSite);
    logger.info("Total number of web pages in site: " + pages.size());
    logger.info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
    
    if(pagesInMRI.size() > 0) {
        logger.info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
        for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
        logger.info(mriWebPageInfo.toString());
        }
    }

    logger.info("                      -----------                   ");
    if(pagesContainingMRI.size() > 0) {     
        logger.info("The following pages weren't detected as primarily being in Māori");
        logger.info("But still contained sentences detected as Māori");
        for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
        logger.info(mriWebPageInfo.toString());
        }
        
    } else {
        logger.info("No further pages detected as containing any sentences in MRI");       
    }
    logger.info("                      -----------                   ");
    }

    
    public void writeSiteRecordToCSV(CSVPrinter websitesCSVPrinter) throws IOException {

    // https://stackoverflow.com/questions/35183146/how-can-i-create-a-java-8-localdate-from-a-long-epoch-time-in-milliseconds
    // LocalDateTime date =
    //     LocalDateTime.ofInstant(Instant.ofEpochMilli(this.siteCrawledTimestamp), ZoneId.systemDefault());
    // String crawlTimestamp =
    //     date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + " " + date.format(DateTimeFormatter.ofPattern("HH:mm:ss"));

    boolean redoCrawl = false;
    int numPagesInMRI = pagesInMRI.size();
    int numPagesContainingMRI = pagesContainingMRI.size();
        
    if(this.siteCrawlUnfinished) {
        // arbitrary decision, but need some indication that the MRI content was not close to one-off in the website
        if(numPagesInMRI > 2 || numPagesContainingMRI > 2) { 
        redoCrawl = true;
        }
    }
    
    // site.csv CSV file row:
    // ID, siteID, domainURL, totalPagesInSite, numPagesInMRI, numPagesContainingMRI, crawlUnfinished, redoCrawl
    websitesCSVPrinter.printRecord(SITE_COUNTER, this.siteID, this.domainOfSite,
                       pages.size(), numPagesInMRI, numPagesContainingMRI,
                   this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl);
    }

    
    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
   
    public static void printUsage() {
    System.err.println("Run this program as:");
    System.err.println("\tNutchTextDumpProcessor <path to 'crawled' folder>");
    }
    
    public static void main(String[] args) {
    if(args.length != 1) {
        printUsage();
        return;
    }

    File sitesDir = new File(args[0]);
    if(!sitesDir.exists() || !sitesDir.isDirectory()) {
        logger.error("Error: " + args[0] + " does not exist or is not a directory");
        return;
    }

    NutchTextDumpProcessor.DEBUG_MODE = false;
    
    File websitesCSVFile = new File(sitesDir, "websites.csv");
    File webpagesCSVFile = new File(sitesDir, "webpages.csv");
    File mriSentencesCSVFile = new File(sitesDir, "mri-sentences.csv");
    
    try (
         CSVPrinter websitesCSVPrinter = new CSVPrinter(new FileWriter(websitesCSVFile), CSVFormat.DEFAULT);
         CSVPrinter webpagesCSVPrinter = new CSVPrinter(new FileWriter(webpagesCSVFile), CSVFormat.DEFAULT);
         CSVPrinter mriSentencesCSVPrinter = new CSVPrinter(new FileWriter(mriSentencesCSVFile), CSVFormat.DEFAULT);
         ) {

        // print out the column headers for the websites csv file
        // https://commons.apache.org/proper/commons-csv/apidocs/org/apache/commons/csv/CSVPrinter.html
        websitesCSVPrinter.printRecord("ID" /*websiteID*/, "siteID"/* site folder name*/,
           "domainURL","totalPagesInSite", "numPagesInMRI", "numOtherPagesContainingMRI",
           "nutchCrawlTimestamp", "crawlUnfinished", "redoCrawl");
        webpagesCSVPrinter.printRecord("webpageID", "websiteID", "URL", "isMRI",
                       "numSentences", "numSentencesInMRI");
        mriSentencesCSVPrinter.printRecord("sentenceID", "webpageID", "sentence");
        
        MaoriTextDetector mriTxtDetector = new MaoriTextDetector(true); // true: run silent
        File[] sites = sitesDir.listFiles();

        // sort site folders in alphabetical order
        // https://stackoverflow.com/questions/7199911/how-to-file-listfiles-in-alphabetical-order
        Arrays.sort(sites);
        
        for(File siteDir : sites) { // e.g. 00001
        if(siteDir.isDirectory()) {
            // look for dump.txt
            File txtDumpFile = new File(siteDir, "dump.txt");
            if(!txtDumpFile.exists()) {
            logger.error("Text dump file " + txtDumpFile + " did not exist");
            continue;
            }
            
            else {
            File UNFINISHED_FILE = new File(siteDir, "UNFINISHED");         
            
            String siteID = siteDir.getName();
            long lastModified = siteDir.lastModified();
            logger.debug("Found siteID: " + siteID);            
            NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(
                 webpagesCSVPrinter, mriSentencesCSVPrinter, mriTxtDetector,
                 siteID, txtDumpFile, lastModified, UNFINISHED_FILE.exists());
            // now it's parsed all the web pages in the site's text dump

            // Let's print stats on each web page's detected language being MRI or not
            // and how many pages there were in the site in total.
            
            nutchTxtDump.printSiteStats();
            nutchTxtDump.writeSiteRecordToCSV(websitesCSVPrinter);
            }
        }
        
        }
        
    } catch(Exception e) {
        // can get an exception when instantiating CCWETProcessor instance
        // or with CSV file
        logger.error(e.getMessage(), e);
    }
    }
}