source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java@ 33576

package org.greenstone.atea;

import java.io.*;
import java.util.HashMap;
import java.util.Map;
import java.lang.ArrayIndexOutOfBoundsException;
    
public class NutchTextDumpProcessor {
    private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());

    private static MaoriTextDetector maoriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
    
    public final String siteID; // is this necessary?

    /** keep a list to store the text of each page */
    private ArrayList<TextDumpPage> pages;
    
    
    public NutchTextDumpProcessor(String siteID, File txtDumpFile) {
    // siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
    this.siteID = siteID;


    pages = new ArrayList<TextDumpPage>();

    String line = null;
    StringBuilder pageDump = new StringBuilder();
    try (
         BufferedReader reader = new BufferedReader(new FileReader(txtDumpFile));
         ) {

        while((line = reader.readLine()) != null) { // readLine removes newline separator
        line = line.trim();
        // an empty line marks the end of a page in nutch's text dump of a site
        if(!line.equals("")) { 
            pageDump.append(line);
            pageDump.append("\n");
        } else {
            TextDumpPage page = new TextDumpPage(pageDump.toString());
            // parses the fields and body text of a webpage in nutch's txt dump of entire site
            //page.parseFields();
            //page.getText();
            pages.add(page);
            pageDump = null;
            pageDump = new StringBuilder();
        }
        }
        
    } catch (IOException ioe) {
        error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
    }
    
    }
    
    /** pageID: id into pages array */
    public boolean isPageInMaori(int pageID) throws ArrayIndexOutOfBoundsException {    

    String text = getTextForPage(pageID);
    return maoriTxtDetector.isTextInMaori(text);
    }

    private TextDumpPage getPage(int pageID) throws ArrayIndexOutOfBoundsException {
    if(pageID < 0 || pageID >= pages.size()) {
        throw new ArrayIndexOutOfBoundsException();
    }

    TextDumpPage page = pages.get(pageID);
    return page;
    }

    public String getTextForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageText();  
    }
    public String getURLForPage(int pageID) throws ArrayIndexOutOfBoundsException {
    TextDumpPage page = getPage(pageID);
    return page.getPageURL();
    }
    
    
    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
    public static void info(String msg) {
    System.err.println(msg);
    logger.info(msg);
    }
    public static void debug(String msg) {
    System.err.println(msg);
    logger.debug(msg);
    }
    public static void warn(String msg) {
    System.err.println(msg);
    logger.warn(msg);
    }
    public static void error(String msg) {
    System.err.println(msg);
    logger.error(msg);
    }
    public static void error(String msg, Exception e) {
    logger.error(msg, e);
    System.err.println("\n"+msg);
    e.printStackTrace();
    }
    
    public static void printUsage() {
    info("Run this program as:");
    info("\tNutchTextDumpProcessor <path to 'sites' folder>");  
    }
    
    public static void main(String[] args) {
    if(args.length != 1) {
        printUsage();
        return;
    }

    File sitesDir = new File(args[0]);
    if(!sitesDir.exists() || !sitesDir.isDirectory()) {
        error("Error: " + args[0] + " does not exist or is not a directory");
        return;
    }

    try {
        File[] sites = sitesDir.listFiles();
        for(File siteDir : sites) { // e.g. 00001
        // look for dump.txt
        File txtDumpFile = new File(siteDir, dump.txt);
        if(!txtDumpFile.exists()) {
            error("Text dump file " + txtDumpFile + " did not exist");
            continue;
        }

        else {
            String siteID = siteDir.getName();
            NutchTextDumpProcessor nutchTxtDump = NutchTextDumpProcessor(siteID, txtDumpFile);
            
        }
        
        }
        
    } catch(Exception e) {
        // can get an exception when instantiating CCWETProcessor instance
        error(e.getMessage(), e);
    }
    }
}