source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java@ 33519

package org.greenstone.atea;


import java.io.*;
import java.util.Properties;
import java.util.zip.GZIPInputStream;
import java.util.Iterator;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.log4j.Logger;

/**
 * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
 * the WET records in each, putting each WET record into a file. Each file is put into a
 * keep or discard or greyListed folder, and its url listed written into a keep, discard
 * or greylisted text file, based on based on
 * 
 * 1. whether it's whitelisted, else greylisted else blacklisted
 * 2. and if explicitly whitelisted or else not greylisted or blacklisted and there's
 * enough content. Formerly, content-length and number of lines were used to determine if
 * the content was sufficient. Now it's just word count and number of MAX characters
 * (not MINIMUM characters) that determine a string is a word. These settings can be adjusted
 * in conf/config.properties.
 *
 * Put a url-blacklist-filter.txt and/or url-greylist-filter.txt and/or url-whitelist-filter.txt
 * into the conf folder to control any url patterns that are explicitly included or excluded or
 * set aside for inspecting later. These filter text files don't use regexes, instead their
 * format is:
 * - precede URL by ^ to blacklist urls that match the given prefix
 * - succeed URL by $ to blacklist urls that match the given suffix
 * - ^url$ will blacklist urls that match the given url completely
 * - Without either ^ or $ symbol, urls containing the given url will get blacklisted
 * 
 * WETProcessor.java's current implementation is that explicit whitelisting has precedence
 * over greylisting and which takes precedence over blacklisting in turn. However, even
 * explicitly whitelisted urls still need to have sufficient content to end up in keepURLs.txt
 * and in the seedURLs.txt file used for nutch, along with its domain in regex-urlfilter.txt
 * also for nutch.
 *
 * A CCWETProcessor instance can be configured to process all the .warc.wet(.gz) files
 * in the given input folder. Then use a single instance of the WETProcessor class to process
 * each single unzipped warc.wet file.
 *
 * To compile, including the jars in lib/ for compiling.
 *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/CCWETProcessor.java 
 *
 * To run, passing the log4j and other properties files in conf/ folder:
 *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor <folder containing warc.wet(.gz) files> <outputFolder>
 *
 * e.g.
 *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
 *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.CCWETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
 *
*/

public class CCWETProcessor {
    private static Logger logger = Logger.getLogger(org.greenstone.atea.CCWETProcessor.class.getName());

    // Properties shared across WETProcessor instances
    public final int MAX_WORD_LENGTH;
    public final int MIN_NUM_WORDS;
    public final int MAX_WORDS_CAMELCASE;
    
    private Properties configProperties = new Properties();

    // File paths shared across WETProcessor instances
    public final File WETFilesDir;
    public final File outputFolder;
    public final File discardFolder;
    public final File keepFolder;
    public final File greyListedFolder;
    public final File keepURLsFile;
    public final File discardURLsFile;
    public final File greyListedFile;

    /** Possible values stored in the blackList/whiteList/greyList Maps */
    private final Integer LIST_ENTRY_CONTAINS = new Integer(0);
    private final Integer LIST_ENTRY_STARTSWITH = new Integer(1);
    private final Integer LIST_ENTRY_ENDSWITH = new Integer(2);
    private final Integer LIST_ENTRY_MATCHES = new Integer(3);

    /**
     * Store url patterns as keys and values indicated whether a url should
     * match it exactly, start/end with it, or contain it
     */
    private HashMap<String, Integer> blackList;
    private HashMap<String, Integer> greyList;
    private HashMap<String, Integer> whiteList;

    /** Map of domains we keep and the full urls we're keeping that are of that domain. 
     * No need to use a TreeMap which preserves natural (alphabetical) ordering of keys,
     * while a HashMap has no notion of ordering, because we just need to store urls with
     * their domains. Whether the domains are sorted or the urls per domain are sorted becomes
     * irrelevant. (Does it really? What if we have urls followed vs preceded by urls with the
     * same prefix, e.g. pinky.com/toto/index.html and pinky.com/toto/nono/file.html
     * Is there any benefit to nutch when crawling if these seedURLs are ordered or not?)
     */
    private Map<String, Set<String>> domainsToURLsMap;
    
    // Keep a count of all the records that all WETProcessors instantiated
    // by our main method combined have processed
    private int totalRecordCount = 0;

    private int wetFileCount = 0;
    
    public CCWETProcessor(File inFolder, File outFolder) throws Exception {
    this.WETFilesDir = inFolder;
    this.outputFolder = outFolder;

    // load up the properties from the config file
    try (InputStream infile = org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
        configProperties = new Properties();
        configProperties.load(infile);      
        //infile.close(); // not explicitly called in examples of try-with-resources
        
    } catch(Exception e) {
        System.err.println("Exception attempting to read properties from config.properties.");
        logger.error("Exception attempting to read properties from config.properties.");
        e.printStackTrace();
    }

    if(configProperties.size() == 0) {
        System.err.println("*** Warning: no values read into config properties. Using defaults.");
    }
    
    MAX_WORD_LENGTH = Integer.parseInt(configProperties.getProperty("WETprocessor.max.word.length", "15"));
    MIN_NUM_WORDS = Integer.parseInt(configProperties.getProperty("WETprocessor.min.num.words", "20"));
    MAX_WORDS_CAMELCASE = Integer.parseInt(configProperties.getProperty("WETprocessor.max.words.camelcase", "10"));


    this.discardFolder = new File(outFolder, "discard");
    if(!discardFolder.exists()) {
        discardFolder.mkdir();
    }   
    this.keepFolder = new File(outFolder, "keep");
    if(!keepFolder.exists()) {
        keepFolder.mkdir();
    }
    
    this.greyListedFolder = new File(outFolder, "greylisted");
    if(!greyListedFolder.exists()) {
        greyListedFolder.mkdir();
    }
    
    this.keepURLsFile = new File(outFolder, "keepURLs.txt");    
    if(keepURLsFile.exists() && !keepURLsFile.delete()) {
        throw new Exception("Warning: Unable to delete " + this.keepURLsFile + ". Unable to proceed.");
    }
    this.discardURLsFile = new File(outFolder, "discardURLs.txt");
    if(discardURLsFile.exists() && !discardURLsFile.delete()) {
        throw new Exception ("Warning Unable to delete " + discardURLsFile + ". Unable to proceed.");
    }
    this.greyListedFile = new File(outFolder, "greyListed.txt");
    if(greyListedFile.exists() && !greyListedFile.delete()) {
        throw new Exception ("Warning Unable to delete " + greyListedFile + ". Unable to proceed.");
    }

    // prepare our blacklist, greylist (for inspection) and whitelist
    System.err.println("Loading blacklist.");
    blackList = new HashMap<String, Integer>();
    initURLFilterList(blackList, "url-blacklist-filter.txt");
    
    System.err.println("Loading greylist.");
    greyList = new HashMap<String, Integer>();
    initURLFilterList(greyList, "url-greylist-filter.txt");
    
    System.err.println("Loading whitelist.");
    whiteList = new HashMap<String, Integer>();
    initURLFilterList(whiteList, "url-whitelist-filter.txt");

    //System.err.println("Prematurely terminating for testing purposes.");
    //System.exit(-1);
    }
    
    /**
     * Using the keepURLs.txt file generated by running WETProcessor instances, produces
     * as output the URL seed list and regex-urlfilter text files required by nutch, see
     * https://cwiki.apache.org/confluence/display/nutch/NutchTutorial
     */
    public void createSeedURLsFiles(File seedURLsFile, File urlFilterFile) {
    // Maintain Sets of unique domains and urls
    // TreeSet: by default, "the elements are ordered using their natural ordering"
    // (or by a Comparator provided at set creation time).
    // Whereas HashSet doesn't guarantee ordering.
    // So we get alphabetic sorting for free. And guaranteed log(n) for basic operations.

    //Set<String> domainsSet = new TreeSet<String>();
    //Set<String> urlsSet = new TreeSet<String>();
    domainsToURLsMap = new TreeMap<String, Set<String>>();
    
    final String FILTER_REGEX_PREFIX = "+https?://([a-z0-9-]+\\.)*"; // https?://([a-z0-9-]+\.)*
    
    try (
         BufferedReader reader = new BufferedReader(new FileReader(this.keepURLsFile));
         ) {
        
        // read a URL at a time from urlsFile
        String url = null;
        String domain = null;
        while((url = reader.readLine()) != null) { // readLine removes newline separator
        
        // work out domain. This retains any www. or subdomain prefix:
        int startIndex = url.indexOf("//"); // http:// or https:// prefix
        startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
        domain = url.substring(startIndex);
        int endIndex = domain.indexOf("/");
        if(endIndex == -1) endIndex = domain.length();
        domain = domain.substring(0, endIndex);

        //urlsSet.add(url);
        //domainsSet.add(domain);
        Set<String> urlsSet;
        if(!domainsToURLsMap.containsKey(domain)) {
            urlsSet = new TreeSet<String>();
            urlsSet.add(url);
            domainsToURLsMap.put(domain, urlsSet);
        } else {
            urlsSet = domainsToURLsMap.get(domain);
            urlsSet.add(url);
        }

        }
    } catch (IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile);
    }

    /*
    try (BufferedWriter seedURLsWriter = new BufferedWriter(new FileWriter(seedURLsFile))) {
        Iterator<String> i = urlsSet.iterator();
        while(i.hasNext()) {
        String url = i.next();
        seedURLsWriter.write(url + "\n");
        }
        
    } catch (IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error writing to " + seedURLsFile);
    }
    */

    int domainCount = 0;
    File sitesFolder = new File(outputFolder, "sites");
    if(!sitesFolder.exists()) {
        sitesFolder.mkdir();
    }
    final String FORMATSTR = "%05d";
    
    // write out each domain followed in sequence by all urls we found in that domain
    // (urls with tab up front)
    try (
         BufferedWriter seedURLsWriter = new BufferedWriter(new FileWriter(seedURLsFile));
         BufferedWriter urlFilterWriter = new BufferedWriter(new FileWriter(urlFilterFile))
         ) {
        //Set<Map.Entry<String, Set<String>>> domainsSet = domainsToURLsMap.keySet();
        Set<String> domainsSet = domainsToURLsMap.keySet();
        Iterator<String> domainIterator = domainsSet.iterator();
        
        while(domainIterator.hasNext()) {
        domainCount++;
        String siteID = String.format(FORMATSTR, domainCount);
        File domainFolder = new File(sitesFolder, siteID);
        domainFolder.mkdir();
        
        // write out the domain
        String domain = domainIterator.next();
        //seedURLsWriter.write(domain + "\n");
        // nutch.apache.org => +^https?://([a-z0-9-]+\.)*nutch\.apache\.org/
        String regexed_domain = FILTER_REGEX_PREFIX + domain.replace(".", "\\.") + "/";
        urlFilterWriter.write(regexed_domain + "\n");

        // for every domain, we need sites/0000x/ folder containing its own
        // INDIVIDUAL seedURLs.txt and regex-urlfilter.txt
        // We still have a global seedURLs.txt and regex-urlfilter.txt too.
        File siteSeedsFile = new File(domainFolder, "seedURLs.txt"); // e.g. sites/00001/seedURLs.txt
        File siteRegexFile = new File(domainFolder, "regex-urlfilter.txt"); // e.g. sites/00001/regex-urlfilter.txt
        try (
             BufferedWriter siteURLsWriter = new BufferedWriter(new FileWriter(siteSeedsFile));
             BufferedWriter siteRegexWriter = new BufferedWriter(new FileWriter(siteRegexFile));
             ) {
            // only write urls and no domain into single global seedurls file
            // But write domain and tabbed urls into individual sites/0000x.txt files
            // and write regexed domain into it too
            siteURLsWriter.write(domain + "\n");        
            siteRegexWriter.write(regexed_domain + "\n");
            
            // next write out the urls for the domain with a tab prefixed to each
            // into the sites/0000x/seedURLs.txt file - also write into the global seeds file
            Set<String> urlsForDomainSet = domainsToURLsMap.get(domain);
            Iterator<String> urlIterator = urlsForDomainSet.iterator();
            while(urlIterator.hasNext()) {
            String url = urlIterator.next();
            seedURLsWriter.write(url + "\n"); // global seedURLs file
            siteURLsWriter.write("\t" + url + "\n"); 
            }
        } catch (IOException ioe) {
            ioe.printStackTrace();
            System.err.println("\n@@@@@@@@@ Error writing to " + siteSeedsFile + " or " + siteRegexFile);
        }
        }
        
    } catch (IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error writing to " + seedURLsFile + " or " + urlFilterFile);
    }
    
    // write out domains as regular expressions into "regex-urlfilter.txt" file
    try (BufferedWriter urlFilterWriter = new BufferedWriter(new FileWriter(urlFilterFile))) {
        Set<String> domainsSet = domainsToURLsMap.keySet();     
        Iterator<String> i = domainsSet.iterator();
        // nutch.apache.org => +^https?://([a-z0-9-]+\.)*nutch\.apache\.org/
        while(i.hasNext()) {
        String domain = i.next();
        domain = FILTER_REGEX_PREFIX + domain.replace(".", "\\.") + "/";        
        urlFilterWriter.write(domain + "\n");
        }
        
    } catch (IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error writing to " + urlFilterFile);
    }
    }

    private boolean isListedInFilterList(Map<String, Integer> filterListMap, String url) {
    Set<Map.Entry<String,Integer>> entries = filterListMap.entrySet();
    Iterator<Map.Entry<String, Integer>> i = entries.iterator();
    while(i.hasNext()) {
        Map.Entry<String, Integer> entry = i.next();
        String urlPattern = entry.getKey();
        Integer matchRule = entry.getValue();

        if(matchRule == LIST_ENTRY_CONTAINS && url.contains(urlPattern)) {
        return true;
        }
        else if(matchRule == LIST_ENTRY_STARTSWITH && url.startsWith(urlPattern)) {
        return true;
        }
        else if(matchRule == LIST_ENTRY_ENDSWITH && url.endsWith(urlPattern)) {
        return true;
        }
        else if(matchRule == LIST_ENTRY_MATCHES && url.equals(urlPattern)) {
        return true;
        }
        // else check the rest of the filter list against this url
        // before returning false to be certain it's not been listed in the filter list
    }
    
    return false;
    }

    /**
     * Returns true if the url or pattern is found in the blacklist file.
     * Note that if eventually the same url pattern is found in the greylist or whitelist too,
     * it won't get blacklisted after all. But that's not implemented here.
     */
    public boolean isBlacklisted(String url) {
    return isListedInFilterList(blackList, url);
    }
    
    /**
     * Returns true if the url or pattern is explicitly mentioned in the greylist file.
     * Will eventually take precedence over if the same URL pattern was mentioned in the blacklist.
     * Will eventually be pre-empted into the whitelist if mentioned in the whitelist.
     */
    public boolean isGreylisted(String url) {
    // TODO: alexa top sites and auto-translated product sites
    return isListedInFilterList(greyList, url);
    }

    /**
     * Returns true if the url or pattern is explicitly mentioned in the whitelist file
     * Its mention in a whitelist moreover overrides any mention in the blacklist and greylist.
     */
    public boolean isWhitelisted(String url) {
    return isListedInFilterList(whiteList, url);
    }
    
    /**
     * Checks URL parameter against each line ("filter") of conf/url-discard-filter.txt to decide
     * whether it is in the discard list.
     * Filters don't represent actual regex, just ^ and $ as start and end terminators.
     * By not having this method deal with actual regex for filters, this has the advantage that
     * we don't have to remember to escape or double escape each filter to turn it into a regex.
     */
    public void initURLFilterList(Map<String, Integer> list, String filterListFilename) {
    
    // if filterListFilename does not exist in the conf folder, just return
    if(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResource(filterListFilename) == null) {
        System.err.println(filterListFilename + " does not exist");
        return;     
    }

    try (
         BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream(filterListFilename), "UTF-8"));
         ) {
        String filter = null;
        while((filter = reader.readLine()) != null) {
        // skip comments and empty lines
        filter = filter.trim();
        if(filter.equals("") || filter.startsWith("#")) {
            continue;
        }
        
        if(filter.startsWith("^") && filter.endsWith("$")) {
            filter = filter.substring(1, filter.length()-1);
            list.put(filter, LIST_ENTRY_MATCHES);
        }
        else if(filter.startsWith("^")) {
            filter = filter.substring(1);
            list.put(filter, LIST_ENTRY_STARTSWITH);
            System.err.println("Match filter startswith: " + filter);
        }
        else if(filter.endsWith("$")) {
            filter = filter.substring(0, filter.length()-1);
            list.put(filter, LIST_ENTRY_ENDSWITH);
        }
        else {
            list.put(filter, LIST_ENTRY_CONTAINS);
        }
        //System.err.println("Got filter: " + filter);
        }
        
    } catch (IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error reading into map from file " + filterListFilename);
    }
    
    }

    /** Maintain a count of all WET files processed. */
    public void setWETFileCount(int count) { this.wetFileCount = count; }
    
    /** Maintain a count of all WET records processed. */
    //public int getRecordCount() { return this.totalRecordCount; }
    //public void addToRecordCount(int count) { this.totalRecordCount += count; }
    public void setRecordCount(int count) { this.totalRecordCount = count; }
    
    public static void printUsage() {
    System.err.println("Run this program as:");
    System.err.println("\tWetProcessor <folder containing wet(.gz) files> <output folder path>");   
    }

    /** Filename filter to only list warc.wet files or else warc.wet.gz files
     * for which unzipped warc.wet equivalents don't yet exist.
     */
    private static class WETFilenameFilter implements FilenameFilter {
    
    public boolean accept(File dir, String name) {
        if(name.endsWith(".warc.wet")) {
        logger.debug("Will include " + name + " for processing.");
        return true;
        }

        if(name.endsWith(".warc.wet.gz")) {
        String nameWithoutGZext = name.substring(0, name.lastIndexOf(".gz"));
        File unzippedVersion = new File(dir, nameWithoutGZext);
        if(unzippedVersion.exists()) {
            logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
            logger.debug("Skipping " + name);
            return false; // don't count gzipped version if unzipped version exists.
        }
        else {
            logger.debug("Only zipped version " + name + " exists.");
            return true; // No unzipped version, so have to work with gzipped version
        }
        }

        // we're not even interested in any other file extensions
        logger.debug("Not a WET file. Skipping " + name);
        return false;
    }
    }
    

    public static void main(String[] args) {
    if(args.length != 2) {
        printUsage();
        return;
    }
    
    
    File WETFileDir = new File(args[0]);
    if(!WETFileDir.exists() || !WETFileDir.isDirectory()) {
        System.out.println("Error: " + args[0] + " does not exist or is not a directory");
        return;
    }
    
    File outFolder = new File(args[1]);
    if(!outFolder.exists() || !outFolder.isDirectory()) {
        System.out.println("Error: " + args[1] + " does not exist or is not a directory.");
        return;
    }   

    try {
    CCWETProcessor ccWETFilesProcessor = new CCWETProcessor(WETFileDir, outFolder);

    //ccWETFilesProcessor.processAllWETFiles();
    
    // Will list all the warc.wet files in the input directory or else their gzipped versions
    File[] WETFiles = WETFileDir.listFiles(new WETFilenameFilter());

    int wetRecordCount = 0;
    int wetFileCount = 0;
    
    for(int i = 0; i < WETFiles.length; i++) {
        File WETFile = WETFiles[i];     
        logger.debug("Processing WETfile: " + WETFile);

        // Any .gz files listed means they haven't been unzipped yet. So unzip.
        String WETFilename = WETFile.toString();
        if(WETFilename.endsWith(".gz")) {
        File GZippedWETFile = WETFile;
        String WETGZippedFilename = WETFilename;
        WETFilename = WETFilename.substring(0, WETFilename.lastIndexOf(".gz"));     

        WETFile = new File(WETFilename);
        Utility.unzipFile(GZippedWETFile, WETFile);
        }
        // hereafter all WETFiles should refer to the unzipped version
        // Check the unzipped WETFile exists        

        if(!WETFile.exists() || !WETFile.isFile()) {
        System.err.println("Error: " + WETFile + " does not exist (failure to unzip?)");
        logger.error("Error: " + WETFile + " does not exist (failure to unzip?)");
        return;
        }

        // Finally, we can process this WETFile's records into the keep and discard pile
        wetFileCount++;
        logger.debug("Off to process " + WETFile);
        WETProcessor wetFileProcessor = new WETProcessor(WETFile, ccWETFilesProcessor);
        wetFileProcessor.processWETFile();
        wetRecordCount += wetFileProcessor.getRecordCount();
    }

    // for information purposes
    ccWETFilesProcessor.setWETFileCount(wetFileCount);
    ccWETFilesProcessor.setRecordCount(wetRecordCount);
    
    File seedURLsFile = new File(outFolder, "seedURLs.txt");
    File urlFilterFile = new File(outFolder, "regex-urlfilter.txt");
    ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile);

    System.out.println("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
    
    } catch(Exception e) {
    // can get an exception when instantiating CCWETProcessor instance
    e.printStackTrace();
    System.err.println(e.getMessage());
    }
    
    return; 
    
    }
}