Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33502)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33503)
@@ -6,4 +6,6 @@
 import java.util.zip.GZIPInputStream;
 import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
@@ -14,6 +16,32 @@
  * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
  * the WET records in each, putting each WET record into a file. Each file is put into a
- * keep or discard folder, based on content-length and number of lines.
- * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
+ * keep or discard or greyListed folder, and its url listed written into a keep, discard
+ * or greylisted text file, based on based on
+ * 
+ * 1. whether it's whitelisted, else greylisted else blacklisted
+ * 2. and if explicitly whitelisted or else not greylisted or blacklisted and there's
+ * enough content. Formerly, content-length and number of lines were used to determine if
+ * the content was sufficient. Now it's just word count and number of MAX characters
+ * (not MINIMUM characters) that determine a string is a word. These settings can be adjusted
+ * in conf/config.properties.
+ *
+ * Put a url-blacklist-filter.txt and/or url-greylist-filter.txt and/or url-whitelist-filter.txt
+ * into the conf folder to control any url patterns that are explicitly included or excluded or
+ * set aside for inspecting later. These filter text files don't use regexes, instead their
+ * format is:
+ * - precede URL by ^ to blacklist urls that match the given prefix
+ * - succeed URL by $ to blacklist urls that match the given suffix
+ * - ^url$ will blacklist urls that match the given url completely
+ * - Without either ^ or $ symbol, urls containing the given url will get blacklisted
+ * 
+ * WETProcessor.java's current implementation is that explicit whitelisting has precedence
+ * over greylisting and which takes precedence over blacklisting in turn. However, even
+ * explicitly whitelisted urls still need to have sufficient content to end up in keepURLs.txt
+ * and in the seedURLs.txt file used for nutch, along with its domain in regex-urlfilter.txt
+ * also for nutch.
+ *
+ * A CCWETProcessor instance can be configured to process all the .warc.wet(.gz) files
+ * in the given input folder. Then use a single instance of the WETProcessor class to process
+ * each single unzipped warc.wet file.
  *
  * To compile, including the jars in lib/ for compiling.
@@ -44,6 +72,17 @@
     public final File discardFolder;
     public final File keepFolder;
+    public final File greyListedFolder;
     public final File keepURLsFile;
     public final File discardURLsFile;
+    public final File greyListedFile;
+
+    private final Integer LIST_ENTRY_CONTAINS = new Integer(0);
+    private final Integer LIST_ENTRY_STARTSWITH = new Integer(1);
+    private final Integer LIST_ENTRY_ENDSWITH = new Integer(2);
+    private final Integer LIST_ENTRY_MATCHES = new Integer(3);
+    
+    private HashMap<String, Integer> blackList;
+    private HashMap<String, Integer> greyList;
+    private HashMap<String, Integer> whiteList;
 
     // Keep a count of all the records that all WETProcessors instantiated
@@ -53,13 +92,13 @@
     private int wetFileCount = 0;
     
-    public CCWETProcessor(File inFolder, File outFolder) {
+    public CCWETProcessor(File inFolder, File outFolder) throws Exception {
 	this.WETFilesDir = inFolder;
 	this.outputFolder = outFolder;
 
 	// load up the properties from the config file
-	try (InputStream infile = org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
+	try (InputStream infile = org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
 	    configProperties = new Properties();
 	    configProperties.load(infile);	    
-	    //infile.close();
+	    //infile.close(); // not explicitly called in examples of try-with-resources
 	    
 	} catch(Exception e) {
@@ -86,16 +125,35 @@
 	    keepFolder.mkdir();
 	}
-
+	
+	this.greyListedFolder = new File(outFolder, "greylisted");
+	if(!greyListedFolder.exists()) {
+	    greyListedFolder.mkdir();
+	}
+	
 	this.keepURLsFile = new File(outFolder, "keepURLs.txt");	
 	if(keepURLsFile.exists() && !keepURLsFile.delete()) {
-	    System.err.println("Warning: Unable to delete " + this.keepURLsFile + ". Unable to proceed.");
-	    //return;
+	    throw new Exception("Warning: Unable to delete " + this.keepURLsFile + ". Unable to proceed.");
 	}
 	this.discardURLsFile = new File(outFolder, "discardURLs.txt");
 	if(discardURLsFile.exists() && !discardURLsFile.delete()) {
-	    System.err.println("Warning Unable to delete " + discardURLsFile + ". Unable to proceed.");
-	    //return;
-	}
-	
+	    throw new Exception ("Warning Unable to delete " + discardURLsFile + ". Unable to proceed.");
+	}
+	this.greyListedFile = new File(outFolder, "greyListed.txt");
+	if(greyListedFile.exists() && !greyListedFile.delete()) {
+	    throw new Exception ("Warning Unable to delete " + greyListedFile + ". Unable to proceed.");
+	}
+
+	System.err.println("Loading blacklist.");
+	blackList = new HashMap<String, Integer>();
+	initURLFilterList(blackList, "url-blacklist-filter.txt");
+	System.err.println("Loading greylist.");
+	greyList = new HashMap<String, Integer>();
+	initURLFilterList(greyList, "url-greylist-filter.txt");
+	System.err.println("Loading whitelist.");
+	whiteList = new HashMap<String, Integer>();
+	initURLFilterList(whiteList, "url-whitelist-filter.txt");
+
+	//System.err.println("Prematurely terminating for testing purposes.");
+	//System.exit(-1);
     }
     
@@ -171,13 +229,56 @@
     }
 
-    /*
+    private boolean isListedInFilterList(Map<String, Integer> filterListMap, String url) {
+	Set<Map.Entry<String,Integer>> entries = filterListMap.entrySet();
+	Iterator<Map.Entry<String, Integer>> i = entries.iterator();
+	while(i.hasNext()) {
+	    Map.Entry<String, Integer> entry = i.next();
+	    String urlPattern = entry.getKey();
+	    Integer matchRule = entry.getValue();
+
+	    if(matchRule == LIST_ENTRY_CONTAINS && url.contains(urlPattern)) {
+		return true;
+	    }
+	    else if(matchRule == LIST_ENTRY_STARTSWITH && url.startsWith(urlPattern)) {
+		return true;
+	    }
+	    else if(matchRule == LIST_ENTRY_ENDSWITH && url.endsWith(urlPattern)) {
+		return true;
+	    }
+	    else if(matchRule == LIST_ENTRY_MATCHES && url.equals(urlPattern)) {
+		return true;
+	    }
+	    // else check the rest of the filter list against this url
+	    // before returning false to be certain it's not been listed in the filter list
+	}
+	
+	return false;
+    }
+
+    /**
+     * Returns true if the url or pattern is found in the blacklist file.
+     * Note that if eventually the same url pattern is found in the greylist or whitelist too,
+     * it won't get blacklisted after all. But that's not implemented here.
+     */
     public boolean isBlacklisted(String url) {
-	return false;
-    }
-    */
-
+	return isListedInFilterList(blackList, url);
+    }
+    
+    /**
+     * Returns true if the url or pattern is explicitly mentioned in the greylist file.
+     * Will eventually take precedence over if the same URL pattern was mentioned in the blacklist.
+     * Will eventually be pre-empted into the whitelist if mentioned in the whitelist.
+     */
     public boolean isGreylisted(String url) {
-	// alexa top sites and auto-translated product sites
-	return false;
+	// TODO: alexa top sites and auto-translated product sites
+	return isListedInFilterList(greyList, url);
+    }
+
+    /**
+     * Returns true if the url or pattern is explicitly mentioned in the whitelist file
+     * Its mention in a whitelist moreover overrides any mention in the blacklist and greylist.
+     */
+    public boolean isWhitelisted(String url) {
+	return isListedInFilterList(whiteList, url);
     }
     
@@ -189,11 +290,54 @@
      * we don't have to remember to escape or double escape each filter to turn it into a regex.
      */
-    //public boolean isInDiscardFilter(String url) {
-
-    public boolean isBlacklisted(String url) {	
+    public void initURLFilterList(Map<String, Integer> list, String filterListFilename) {
+	
+	// if filterListFilename does not exist in the conf folder, just return
+	if(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResource(filterListFilename) == null) {
+	    System.err.println(filterListFilename + " does not exist");
+	    return;	    
+	}
+
+	try (
+	     BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream(filterListFilename), "UTF-8"));
+	     ) {
+	    String filter = null;
+	    while((filter = reader.readLine()) != null) {
+		// skip comments and empty lines
+		filter = filter.trim();
+		if(filter.equals("") || filter.startsWith("#")) {
+		    continue;
+		}
+		
+		if(filter.startsWith("^") && filter.endsWith("$")) {
+		    filter = filter.substring(1, filter.length()-1);
+		    list.put(filter, LIST_ENTRY_MATCHES);
+		}
+		else if(filter.startsWith("^")) {
+		    filter = filter.substring(1);
+		    list.put(filter, LIST_ENTRY_STARTSWITH);
+		    System.err.println("Match filter startswith: " + filter);
+		}
+		else if(filter.endsWith("$")) {
+		    filter = filter.substring(0, filter.length()-1);
+		    list.put(filter, LIST_ENTRY_ENDSWITH);
+		}
+		else {
+		    list.put(filter, LIST_ENTRY_CONTAINS);
+		}
+		//System.err.println("Got filter: " + filter);
+	    }
+		
+	} catch (IOException ioe) {
+	    ioe.printStackTrace();
+	    System.err.println("\n@@@@@@@@@ Error reading into map from file " + filterListFilename);
+	}
+	
+    }
+    /*
+    public boolean isInDiscardFilter(String url) {
 	String discardFilterFile = "url-discard-filter.txt"; // in conf folder
 
 	try (
-	     BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream(discardFilterFile), "UTF-8"));
+	     BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream(discardFilterFile), "UTF-8"));
 	     ) {
 	    String filter = null;
@@ -227,5 +371,5 @@
 
 	return false;
-    }
+    }*/
 
     /** Maintain a count of all WET files processed. */
@@ -293,4 +437,5 @@
 	}	
 
+    try {
 	CCWETProcessor ccWETFilesProcessor = new CCWETProcessor(WETFileDir, outFolder);
 
@@ -341,4 +486,9 @@
 	File urlFilterFile = new File(outFolder, "regex-urlfilter.txt");
 	ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile);
+    } catch(Exception e) {
+	// can get an exception when instantiating CCWETProcessor instance
+	e.printStackTrace();
+	System.err.println(e.getMessage());
+    }
 	
 	return;	
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33502)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33503)
@@ -12,18 +12,20 @@
 
 /**
- * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
- * the WET records in each, putting each WET record into a file. Each file is put into a
- * keep or discard folder, based on content-length and number of lines.
- * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
+ * A single instance of the WETProcessor class can process a single unzipped warc.wet file.
+ * A WETProcessor take a warc.wet file and goes through all its WET records,
+ * putting each WET record into a file. Each file is put into a keep, discard or greylisted folder
+ * and its url listed written into a keep, discard or greylisted text file, based on:
  *
- * To compile, including the jars in lib/ for compiling.
- *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/WETProcessor.java 
+ * 1. whether it's whitelisted, else greylisted else blacklisted
+ * 2. and if explicitly whitelisted or else not greylisted or blacklisted and there's
+ * enough content. Formerly, content-length and number of lines were used to determine if
+ * the content was sufficient. Now it's just word count and number of MAX characters
+ * (not MINIMUM characters) that determine a string is a word.
+ * Explicit whitelisting has precedence over greylisting and which takes precedence
+ * over blacklisting in turn.
+ * However, even explicitly whitelisted urls still need to have sufficient content to end 
+ * up in keepURLs.txt.
  *
- * To run, passing the log4j and other properties files in conf/ folder:
- *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor &lt;folder containing warc.wet(.gz) files&gt; &lt;outputFolder&gt;
- *
- * e.g.
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2&gt;&amp;1 | less
+ * See CCWETProcessor.java for compile instructions and how to run.
  *
 */
@@ -72,5 +74,6 @@
     public int processWETFile() {
 	File keepURLsFile = this.batchProcessor.keepURLsFile;
-	File discardURLsFile = this.batchProcessor.discardURLsFile;	
+	File discardURLsFile = this.batchProcessor.discardURLsFile;
+	File greyListedFile = this.batchProcessor.greyListedFile;
 	
 	StringBuilder record = null;
@@ -90,4 +93,5 @@
 	     BufferedWriter keepURLsWriter = new BufferedWriter(new FileWriter(keepURLsFile, true));
 	     BufferedWriter discardURLsWriter = new BufferedWriter(new FileWriter(discardURLsFile, true)); // true to append
+	     BufferedWriter greyListedURLsWriter = new BufferedWriter(new FileWriter(greyListedFile, true)); // true to append
 	     ) {
 	    
@@ -104,5 +108,5 @@
 		    // process any previous record
 		    if(record != null) {
-			processWETrecord(keepURLsWriter, discardURLsWriter,
+			processWETrecord(keepURLsWriter, discardURLsWriter, greyListedURLsWriter,
 					 recordCount, contentLength, lineCount,
 					 WARCtargetURI, record.toString());
@@ -147,5 +151,5 @@
 	    // flush the last record. If it was a warcinfo record, record would be null here
 	    if(record != null) {
-		processWETrecord(keepURLsWriter, discardURLsWriter,
+		processWETrecord(keepURLsWriter, discardURLsWriter, greyListedURLsWriter,
 				 recordCount, contentLength, lineCount,
 				 WARCtargetURI, record.toString());
@@ -169,4 +173,5 @@
      */
     private void processWETrecord(BufferedWriter keepURLsWriter, BufferedWriter discardURLsWriter,
+				  BufferedWriter greyListedURLsWriter,
 				  int recordID, int contentLength, int lineCount,
 				  String recordURI, String record)
@@ -210,11 +215,38 @@
 	}
 	*/
-
+	
 	if(batchProcessor.isBlacklisted(recordURI)) {
-	    parentFolder = batchProcessor.discardFolder;
+
+	    
+	    // explicit whitelisting overrides blacklisting
+	    if(batchProcessor.isWhitelisted(recordURI)) {
+		parentFolder = batchProcessor.keepFolder; //tentative
+	    }
+	    // if not whitelisted, then greylisting overrides blacklisting
+	    else if(batchProcessor.isGreylisted(recordURI)) {
+		parentFolder = batchProcessor.greyListedFolder;
+		System.err.println("@@@GREYLISTED");
+	    }
+	    else { // only blacklisted
+		parentFolder = batchProcessor.discardFolder;
+		System.err.println("@@@DISCARDING - blacklisted");
+	    }
 	}
 	else if(batchProcessor.isGreylisted(recordURI)) { // e.g. products sites
-	    parentFolder = batchProcessor.discardFolder; // TODO: checkfolder
-	} else {
+	    // explicit whitelisting overrides greylisting
+	    if(batchProcessor.isWhitelisted(recordURI)) {
+		parentFolder = batchProcessor.keepFolder; // tentative
+	    }
+	    else {
+		parentFolder = batchProcessor.greyListedFolder;
+		System.err.println("@@@GREYLISTED");
+	    }
+	}
+
+	// If URL was not blacklisted/greylisted, or was even explicitly whitelisted,
+	// it still can't be in the keep list as it needs further inspection:
+	// it needs sufficient content for language analysis.
+	if(parentFolder != batchProcessor.greyListedFolder && parentFolder != batchProcessor.discardFolder) { // i.e. parentFolder == keepFolder if whiteListed || parentFolder == null
+	    
 	    // If a web page's WET record contains a certain minimum number of words,
 	    // we will think it's a meaningful web page and has sufficient content for text analysis
@@ -254,5 +286,6 @@
 	    }
 	}
-	// if parentFolder still not set, set to discard pile folder
+	// if parentFolder still not set, it means that the content length/num words or lines
+	// were insufficient, so meant to be discarded
 	if(parentFolder == null) {
 	    parentFolder = batchProcessor.discardFolder;
@@ -263,4 +296,6 @@
 	    if (parentFolder == batchProcessor.keepFolder) {
 		keepURLsWriter.write(recordURI + "\n");
+	    } else if (parentFolder == batchProcessor.greyListedFolder) {
+		greyListedURLsWriter.write(recordURI + "\n");
 	    } else {
 		discardURLsWriter.write(recordURI + "\n");
