source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java@ 33579

package org.greenstone.atea;

import java.io.*;
import java.util.HashMap;
import java.util.Map;

import org.apache.log4j.Logger;


public class TextDumpPage {
    private static Logger logger = Logger.getLogger(org.greenstone.atea.TextDumpPage.class.getName());
    
    public static final String TEXT_START_MARKER="text:start:";
    public static final String TEXT_END_MARKER="text:end:";
    
    private Map<String, String> tuples;
    
    public TextDumpPage(String siteID, String unparsedPageDump) {
    tuples = new HashMap<String, String>();
    
    try (
         BufferedReader reader = new BufferedReader(new StringReader(unparsedPageDump));
         ) {

        String line = reader.readLine(); // should have at least first line

        // first line always has a "key:" somewhere after the pageURL
        int endIndex = line.indexOf("key:");

        //String pageURL = line.substring(0, endIndex);
        String pageURL = (endIndex == -1) ? line : line.substring(0, endIndex);     
        
        
        tuples.put("pageURL", pageURL.trim());

        if(endIndex != -1) {
        String key = line.substring(endIndex);
        tuples.put("key", key.trim());
        } else {
        debug("@@@@ no key for pageURL: " + pageURL);
        }

        if(pageURL.contains(TEXT_END_MARKER)) {
        debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
        debug("+++++++++");
        debug(unparsedPageDump);
        debug("+++++++++");
        }

        boolean readingPageText = false;
        StringBuilder pageText = null;

        // continue reading all other tuples for this page, if any
        while((line = reader.readLine()) != null) {
        
        if(!readingPageText) {
            // check if we're dealing with metadata or start/end of pagetext
            endIndex = line.indexOf(":");
            if(endIndex != -1) { // dealing with the rest of the page dump's metadata
            String k = line.substring(0, endIndex);
            String v = line.substring(endIndex+1);
            tuples.put(k.trim(), v.trim());
            }
            
            else if(line.equals(TEXT_START_MARKER)) { // dealing with the page body text
            pageText = new StringBuilder();
            readingPageText = true;
            }                   
        }

        else { // we're reading in the page text
            
            if(line.equals(TEXT_END_MARKER)) {
            // finished with a page body            
            // remove any FINAL artificial newline we introduced
            tuples.put("pageText", pageText.toString().trim());
            readingPageText = false;
            pageText = null;
            }
            else {
            pageText.append(line);
            pageText.append("\n"); // there are no newlines within pageText
            // but if there were newlines, add them back here as readLine() removes them
            }           

        }
        }

        // If the page had no pageText, add a "pageText" -> "" mapping
        if(!tuples.containsKey("pageText")) {
        tuples.put("pageText", "");
        }

        
    } catch (IOException ioe) {
        error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
    }

    /*
    // START DEBUG
    debug("__________________________________________");
    for(Map.Entry<String, String> entry : tuples.entrySet()) {
        String key = entry.getKey();
        String value = entry.getValue();        
        debug(key + " - " + value);
    }
    debug("__________________________________________");
    // END DEBUG
    */
    }

    
    public String getPageURL() {
    return tuples.get("url");
    }

    public String getPageText() {
    return tuples.get("pageText");
    }

    public String get(String key) {
    return tuples.get(key);
    }

    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
    public static void info(String msg) {
    System.err.println(msg);
    logger.info(msg);
    }
    public static void debug(String msg) {
    System.err.println(msg);
    logger.debug(msg);
    }
    public static void warn(String msg) {
    System.err.println(msg);
    logger.warn(msg);
    }
    public static void error(String msg) {
    System.err.println(msg);
    logger.error(msg);
    }
    public static void error(String msg, Exception e) {
    logger.error(msg, e);
    System.err.println("\n"+msg);
    e.printStackTrace();
    }
    
}