source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java@ 33587

package org.greenstone.atea;

import java.io.*;
import java.lang.ArrayIndexOutOfBoundsException;
import java.util.ArrayList;
import java.util.Arrays;

import org.apache.log4j.Logger;

/**
 * Class to process the dump text files produced FOR EACH SITE (e.g. site "00001") that
 * Nutch has finished crawling and whose text has been dumped out to a file called dump.txt.
 * This reads in the dump.txt file contained in each site folder within the input folder.
 * (e.g. input folder "crawled" could contain folders 00001 to 01465. Each contains a dump.txt)
 * Each dump.txt could contain the text contents for an entire site, or for individual pages.
 * This class then uses class TextDumpPage to parse each webpage within a dump.txt,
 * which parses out the actual text body content of each webpage's section within a dump.txt.
 * Finally, MaoriTextDetector is run over that to determine whether the full body text is
 * likely to be in Maori or not.
 *
 * Potential issues: since a web page's text is dumped out by nutch with neither paragraph
 * nor even newline separator, it's hard to be sure that the entire page is in language.
 * If it's in multiple languages, there's no way to be sure there aren't promising Maori language
 * paragraphs contained in a page, if the majority/the remainder happen to be in English.
 * 
 * So if we're looking for any paragraphs in Maori to store in a DB, perhaps it's better to run
 * the MaoriTextDetector.isTextInMaori(BufferedReader reader) over two "lines" at a time,
 * instead of running it over the entire html body's text.
 *
 * TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled
 *
 * or:
 *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled > ../crawled/bla.txt 2>&1
 *
*/
public class NutchTextDumpProcessor {
    private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());

    static boolean DEBUG_MODE = true;
    
    private final MaoriTextDetector maoriTxtDetector;
    
    public final String siteID; // is this necessary?
    private String domainOfSite;
    
    /** keep a list to store the text of each page */
    private ArrayList<TextDumpPage> pages;

    /** list of pages in this site which were detected as being in MRI */
    private ArrayList<MRIWebPageStats> pagesInMRI;
    /** list of pages in this site which were NOT detected as being in MRI but nevertheless
     * contain one or more sentences in MRI 
     */
    private ArrayList<MRIWebPageStats> pagesContainingMRI;
    
    private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
    // The start of a new web page's record in nutch's text dump of an entire site
    // is denoted by a newline followed by a URL (protocol)
    // or the very start of the file with a URL (protocol)
    return ((prevLine == null || prevLine.equals(""))
        && (line.startsWith("http://") || line.startsWith("https://")));
    }

    public void debugPageDump(StringBuilder pageDump) {
    if(DEBUG_MODE) {
        // START DEBUG
        debug("__________________________________________");
        debug("@@@ Found page entry: ");
        debug("__________________________________________");
        debug(pageDump.toString());
        debug("------------------------------------------");
        // END DEBUG
    }
    }
    
    public NutchTextDumpProcessor(MaoriTextDetector maoriTxtDetector, String siteID, File txtDumpFile) {
    // siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
    this.siteID = siteID;
    this.maoriTxtDetector = maoriTxtDetector;
    
    pages = new ArrayList<TextDumpPage>();

    String line = null;
    StringBuilder pageDump = null;
    try (
         BufferedReader reader = new BufferedReader(new FileReader(txtDumpFile));
         ) {

        boolean readingText = false;
        String prevLine = null;
        
        while((line = reader.readLine()) != null) { // readLine removes newline separator
        line = line.trim();
        // iff outside of a page's body text, then an empty line marks the end of a page
        // in nutch's text dump of a site.
        // But note, there can be an empty line (or more?) between the start and end
        // markers of a page's text, though.

        if(isStartOfNewWebPageRecord(prevLine, line)) {

            if(pageDump != null) { // should also be the case then: if(prevLine != null)
            // finish old pageDump and begin new one            

            //debugPageDump(pageDump);
                
            TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
            // parses the fields and body text of a webpage in nutch's txt dump of entire site
            //page.parseFields();
            //page.getText();
            pages.add(page);
            pageDump = null;

            }           

            // begin new webpage dump
            pageDump = new StringBuilder();
            pageDump.append(line);
            pageDump.append("\n");
            
        }
        else if(!line.equals("")) {
            pageDump.append(line);
            pageDump.append("\n");

        }
        // can throw away any newlines between text start and end markers.
        
        prevLine = line;        
        }

        // process final webpage record:
        //debugPageDump(pageDump);
        
        TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
        pages.add(page);
        pageDump = null;
        
    } catch (IOException ioe) {
        error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
    }

    // Just do this once: get and store domain of site.
    // Passing true to get domain with protocol prefix
    if(pages.size() > 0) {
        TextDumpPage firstPage = pages.get(0);
        String url = firstPage.getPageURL();
        this.domainOfSite = CCWETProcessor.getDomainForURL(url, true);
    }
    else {
        this.domainOfSite = "UNKNOWN";
    }
    
    prepareSiteStats();
    }
    
    /** pageID: id into pages array */
    public boolean isPageInMaori(int pageID) throws ArrayIndexOutOfBoundsException {    

    String text = getTextForPage(pageID);
    
    // QTODO: what to do when page body text is empty?
    if(text.equals("")) return false;
    return maoriTxtDetector.isTextInMaori(text);
    }

    private TextDumpPage getPage(int pageID) throws ArrayIndexOutOfBoundsException {
    if(pageID < 0 || pageID >= pages.size()) {
        throw new ArrayIndexOutOfBoundsException();
    }

    TextDumpPage page = pages.get(pageID);
    return page;
    }

    public String getTextForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageText();  
    }
    public String getURLForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageURL();
    }   
       
    public int totalNumPages() {
    return pages.size();
    } 

    private void prepareSiteStats() {
    pagesInMRI = new ArrayList<MRIWebPageStats>();
    pagesContainingMRI = new ArrayList<MRIWebPageStats>();

    TextDumpPage page = null;
    for(int i = 0; i < pages.size(); i++) {
        
        page = pages.get(i);
        
        String text = page.getPageText();
        if(text.equals("")) {
        page.addMRILanguageStatus(false);
        continue;
        }
        else {
        boolean isMRI = maoriTxtDetector.isTextInMaori(text);
        
        page.addMRILanguageStatus(isMRI);

        // Even if the entire page is not found to be overall in Māori,
        // let's sitll inspect the sentences of the page and count how many (if any)
        // are in te reo.
        ArrayList<String> mriSentences = maoriTxtDetector.getAllSentencesInMaori(text);
        // first element is always total num sentences
        // remaining elements are the actual sentences that were detected as being Māori
        int totalSentences = Integer.parseInt(mriSentences.get(0));
        int numSentencesInMRI = mriSentences.size() - 1;            
        

        // Add page to list of MRI pages if the page's body text overall was detected
        // as Māori
        // Add page to list of pages containing MRI if >= 1 sentences in the page
        // were detected as being in MRI
        if(isMRI || numSentencesInMRI >= 1) {
            String url = page.getPageURL();
            MRIWebPageStats MRIpageStats = new MRIWebPageStats(this.siteID, url, i, isMRI,
                           totalSentences, numSentencesInMRI);
            if(isMRI) {   
            pagesInMRI.add(MRIpageStats);
            } else if(numSentencesInMRI >= 1) {
            pagesContainingMRI.add(MRIpageStats);
            }

        }       
        }
    }
    }
    
    public void printSiteStats() {
    
    
    info("------------- " + this.siteID + " SITE STATS -----------");

    info("SITE DOMAIN: " + this.domainOfSite);
    info("Total number of web pages in site: " + pages.size());
    info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
    
    if(pagesInMRI.size() > 0) {
        info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
        for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
        info(mriWebPageInfo.toString());
        }
    }

    info("                      -----------                   ");
    if(pagesContainingMRI.size() > 0) {     
        info("The following pages weren't detected as primarily being in Māori");
        info("But still contained sentences detected as Māori");
        for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
        info(mriWebPageInfo.toString());
        }
        
    } else {
        info("No further pages detected as containing any sentences in MRI");      
    }
    info("                      -----------                   ");
    }

    
    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
    public static void info(String msg) {
    System.err.println(msg);
    logger.info(msg);
    }
    public static void debug(String msg) {
    System.err.println(msg);
    logger.debug(msg);
    }
    public static void warn(String msg) {
    System.err.println(msg);
    logger.warn(msg);
    }
    public static void error(String msg) {
    System.err.println(msg);
    logger.error(msg);
    }
    public static void error(String msg, Exception e) {
    logger.error(msg, e);
    System.err.println("\n"+msg);
    e.printStackTrace();
    }
    
    public static void printUsage() {
    info("Run this program as:");
    info("\tNutchTextDumpProcessor <path to 'sites' folder>");  
    }
    
    public static void main(String[] args) {
    if(args.length != 1) {
        printUsage();
        return;
    }

    File sitesDir = new File(args[0]);
    if(!sitesDir.exists() || !sitesDir.isDirectory()) {
        error("Error: " + args[0] + " does not exist or is not a directory");
        return;
    }

    NutchTextDumpProcessor.DEBUG_MODE = false;
    
    try {
        MaoriTextDetector mriTxtDetector = new MaoriTextDetector(true); // true: run silent
        File[] sites = sitesDir.listFiles();

        // sort site folders in alphabetical order
        // https://stackoverflow.com/questions/7199911/how-to-file-listfiles-in-alphabetical-order
        Arrays.sort(sites);
        
        for(File siteDir : sites) { // e.g. 00001
        if(siteDir.isDirectory()) {
            // look for dump.txt
            File txtDumpFile = new File(siteDir, "dump.txt");
            if(!txtDumpFile.exists()) {
            error("Text dump file " + txtDumpFile + " did not exist");
            continue;
            }
            
            else {
            String siteID = siteDir.getName();
            debug("Found siteID: " + siteID);           
            NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);
            // now it's parsed all the web pages in the site's text dump

            // Let's print stats on each web page's detected language being MRI or not
            // and how many pages there were in the site in total.
            
            nutchTxtDump.printSiteStats();
            }
        }
        
        }
        
    } catch(Exception e) {
        // can get an exception when instantiating CCWETProcessor instance
        error(e.getMessage(), e);
    }
    }
}