source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java@ 33579

package org.greenstone.atea;

import java.io.*;
import java.util.ArrayList;
//import java.util.HashMap;
//import java.util.Map;
import java.lang.ArrayIndexOutOfBoundsException;

import org.apache.log4j.Logger;

/**
 * Class to process the dump text files produced for each site (e.g. site "00001") that
 * Nutch has finished crawling and whose text has been dumped out to a file called dump.txt.
 * This reads in the dump.txt file contained in each site folder within the input folder.
 * (e.g. input folder "crawled" could contain folders 00001 to 01465. Each contains a dump.txt)
 * Each dump.txt could contain the text contents for an entire site, or for individual pages.
 * This class then uses class TextDumpPage to parse each webpage within a dump.txt,
 * which parses out the actual text body content of each webpage's section within a dump.txt.
 * Finally, MaoriTextDetector is run over that to determine whether the full body text is
 * likely to be in Maori or not.
 *
 * Potential issues: since a web page's text is dumped out by nutch with neither paragraph
 * nor even newline separator, it's hard to be sure that the entire page is in language.
 * If it's in multiple languages, there's no way to be sure there aren't promising Maori language
 * paragraphs contained in a page, if the majority/the remainder happen to be in English.
 * 
 * So if we're looking for any paragraphs in Maori to store in a DB, perhaps it's better to run
 * the MaoriTextDetector.isTextInMaori(BufferedReader reader) over two "lines" at a time,
 * instead of running it over the entire html body's text.
 *
 * TO COMPILE OR RUN, FIRST DO:
 *    cd maori-lang-detection/apache-opennlp-1.9.1
 *    export OPENNLP_HOME=`pwd`
 *    cd maori-lang-detection/src
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor.java
 *
 * TO RUN:
 *    maori-lang-detection/src$
 *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled
 *
*/
public class NutchTextDumpProcessor {
    private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());

    private final MaoriTextDetector maoriTxtDetector;
    
    public final String siteID; // is this necessary?

    /** keep a list to store the text of each page */
    private ArrayList<TextDumpPage> pages;
    
    
    public NutchTextDumpProcessor(MaoriTextDetector maoriTxtDetector, String siteID, File txtDumpFile) {
    // siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
    this.siteID = siteID;
    this.maoriTxtDetector = maoriTxtDetector;
    
    pages = new ArrayList<TextDumpPage>();

    String line = null;
    StringBuilder pageDump = new StringBuilder();
    try (
         BufferedReader reader = new BufferedReader(new FileReader(txtDumpFile));
         ) {

        boolean readingText = false;
        
        while((line = reader.readLine()) != null) { // readLine removes newline separator
        line = line.trim();
        // iff outside of a page's body text, then an empty line marks the end of a page
        // in nutch's text dump of a site.
        // But note, there can be an empty line (or more?) between the start and end
        // markers of a page's text, though.

        if(!readingText && line.equals("")) { 
            pageDump.append(line);
            pageDump.append("\n");

            
            // START DEBUG
            debug("__________________________________________");
            debug("@@@ Found page entry: ");
            debug("__________________________________________");
            debug(pageDump.toString());
            debug("------------------------------------------");
            // END DEBUG
            
            
            TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
            // parses the fields and body text of a webpage in nutch's txt dump of entire site
            //page.parseFields();
            //page.getText();
            pages.add(page);
            pageDump = null;
            
            pageDump = new StringBuilder();
        }
        else if (!line.equals("")) { // empty line
            if(line.equals(TextDumpPage.TEXT_START_MARKER)) {
            readingText = true;
            }
            if(line.equals(TextDumpPage.TEXT_END_MARKER)) {
            readingText = false;
            }
            pageDump.append(line);
            pageDump.append("\n");
        }
        // can throw away any newlines between text start and end markers.
        }
        
    } catch (IOException ioe) {
        error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
    }
    
    }
    
    /** pageID: id into pages array */
    public boolean isPageInMaori(int pageID) throws ArrayIndexOutOfBoundsException {    

    String text = getTextForPage(pageID);
    // QTODO: what to do when page body text is empty?
    if(text.equals("")) return false;
    return maoriTxtDetector.isTextInMaori(text);
    }

    private TextDumpPage getPage(int pageID) throws ArrayIndexOutOfBoundsException {
    if(pageID < 0 || pageID >= pages.size()) {
        throw new ArrayIndexOutOfBoundsException();
    }

    TextDumpPage page = pages.get(pageID);
    return page;
    }

    public String getTextForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageText();  
    }
    public String getURLForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageURL();
    }
    
    
    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
    public static void info(String msg) {
    System.err.println(msg);
    logger.info(msg);
    }
    public static void debug(String msg) {
    System.err.println(msg);
    logger.debug(msg);
    }
    public static void warn(String msg) {
    System.err.println(msg);
    logger.warn(msg);
    }
    public static void error(String msg) {
    System.err.println(msg);
    logger.error(msg);
    }
    public static void error(String msg, Exception e) {
    logger.error(msg, e);
    System.err.println("\n"+msg);
    e.printStackTrace();
    }
    
    public static void printUsage() {
    info("Run this program as:");
    info("\tNutchTextDumpProcessor <path to 'sites' folder>");  
    }
    
    public static void main(String[] args) {
    if(args.length != 1) {
        printUsage();
        return;
    }

    File sitesDir = new File(args[0]);
    if(!sitesDir.exists() || !sitesDir.isDirectory()) {
        error("Error: " + args[0] + " does not exist or is not a directory");
        return;
    }

    try {
        MaoriTextDetector mriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
        File[] sites = sitesDir.listFiles();
        for(File siteDir : sites) { // e.g. 00001
        // look for dump.txt
        File txtDumpFile = new File(siteDir, "dump.txt");
        if(!txtDumpFile.exists()) {
            error("Text dump file " + txtDumpFile + " did not exist");
            continue;
        }

        else {
            String siteID = siteDir.getName();
            debug("Found siteID: " + siteID);           
            NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);          
        }
        
        }
        
    } catch(Exception e) {
        // can get an exception when instantiating CCWETProcessor instance
        error(e.getMessage(), e);
    }
    }
}