Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33516)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33517)
@@ -77,13 +77,24 @@
     public final File greyListedFile;
 
+    /** Possible values stored in the blackList/whiteList/greyList Maps */
     private final Integer LIST_ENTRY_CONTAINS = new Integer(0);
     private final Integer LIST_ENTRY_STARTSWITH = new Integer(1);
     private final Integer LIST_ENTRY_ENDSWITH = new Integer(2);
     private final Integer LIST_ENTRY_MATCHES = new Integer(3);
-    
+
+    /**
+     * Store url patterns as keys and values indicated whether a url should
+     * match it exactly, start/end with it, or contain it
+     */
     private HashMap<String, Integer> blackList;
     private HashMap<String, Integer> greyList;
     private HashMap<String, Integer> whiteList;
 
+    /** Map of domains we keep and the full urls we're keeping that are of that domain. 
+     * Choosing a TreeMap to preserve natural (alphabetical) ordering of keys,
+     * since a HashMap has no notion of ordering.
+     */
+    private TreeMap<String, TreeSet<String>> domainsToURLsMap;
+    
     // Keep a count of all the records that all WETProcessors instantiated
     // by our main method combined have processed
@@ -144,10 +155,13 @@
 	}
 
+	// prepare our blacklist, greylist (for inspection) and whitelist
 	System.err.println("Loading blacklist.");
 	blackList = new HashMap<String, Integer>();
 	initURLFilterList(blackList, "url-blacklist-filter.txt");
+	
 	System.err.println("Loading greylist.");
 	greyList = new HashMap<String, Integer>();
 	initURLFilterList(greyList, "url-greylist-filter.txt");
+	
 	System.err.println("Loading whitelist.");
 	whiteList = new HashMap<String, Integer>();
@@ -159,6 +173,6 @@
     
     /**
-     * Takes as input the keepURLs.txt file generated by running WETProcessor instances.
-     * As output produces the URL seed list and regex-urlfilter text files required by nutch,
+     * Using the keepURLs.txt file generated by running WETProcessor instances, produces
+     * as output the URL seed list and regex-urlfilter text files required by nutch, see
      * https://cwiki.apache.org/confluence/display/nutch/NutchTutorial
      */
@@ -448,4 +462,7 @@
 	File urlFilterFile = new File(outFolder, "regex-urlfilter.txt");
 	ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile);
+
+	System.out.println("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
+	
     } catch(Exception e) {
 	// can get an exception when instantiating CCWETProcessor instance
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33516)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33517)
@@ -188,34 +188,6 @@
 	File parentFolder = null;
 
-	// want to match "product(s)" but not "production"
-	//if(recordURI.matches(".*/?product[^a-rt-z].*")) {//if(recordURI.matches(".*/?products?/?.*")) {
-	
-
-	/*
-	if(recordURI.contains("product") && !recordURI.contains("production")) {
-
-	    // don't want a "translated" product site/online store
-	    // These curiously often tend to have "product(s)" in the URL
-	    parentFolder = batchProcessor.discardFolder;
-	}
-
-	else if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
-	    parentFolder = batchProcessor.keepFolder;
-	    System.err.println("@@@KEEPING");
-	} else if(contentLength >= MIN_CONTENT_LENGTH_WRAPPED_LINE) {
-	    int countSpaces = 0;
-	    for(int i = 0; i < record.length(); i++) {
-		if(record.charAt(i) == ' ') countSpaces++;
-	    }
-	    if(countSpaces >= MIN_SPACES_IN_A_WRAPPED_LINE) {
-		// So we have at least 500 chars (possibly on a single wrapped line)
-		// containing at least 10 spaces. Such a record is also worth keeping.
-		parentFolder = batchProcessor.keepFolder;
-	    }
-	}
-	*/
-	
+
 	if(batchProcessor.isBlacklisted(recordURI)) {
-
 	    
 	    // explicit whitelisting overrides blacklisting
@@ -223,10 +195,10 @@
 		parentFolder = batchProcessor.keepFolder; //tentative
 	    }
-	    // if not whitelisted, then greylisting overrides blacklisting
+	    // if not whitelisted, then greylisting still overrides blacklisting
 	    else if(batchProcessor.isGreylisted(recordURI)) {
 		parentFolder = batchProcessor.greyListedFolder;
 		System.err.println("@@@GREYLISTED");
 	    }
-	    else { // only blacklisted
+	    else { // url was only blacklisted
 		parentFolder = batchProcessor.discardFolder;
 		System.err.println("@@@DISCARDING - blacklisted");
@@ -247,4 +219,6 @@
 	// it still can't be in the keep list as it needs further inspection:
 	// it needs sufficient content for language analysis.
+	// We don't care about the combination of number of lines and content-length,
+	// we just care about the number of "valid words" as defined by us.
 	if(parentFolder != batchProcessor.greyListedFolder && parentFolder != batchProcessor.discardFolder) { // i.e. parentFolder == keepFolder if whiteListed || parentFolder == null
 	    
@@ -267,19 +241,27 @@
 
 		// throw away if n words contain camelcase, which is another case of words glued together
+		// For now, we'll only skip camelcased words in our count of valid words
 		if(word.matches(".*[a-z][A-Z].*") && word.length() >= 5) {
 		    numCamelCaseWords++;		    
-		}
-		
+		}		
 		// In Maori, word length of 1 is not uncommon
 		// but let's skip camelcased words when counting valid words
-		else if(word.length() >= 1 && word.length() <= batchProcessor.MAX_WORD_LENGTH) validWordCount++;
-	    }
-
-	    // dump if too many camelcase words (ideally keep none of that kind?)
+		else if(word.length() >= 1 && word.length() <= batchProcessor.MAX_WORD_LENGTH) {
+		    validWordCount++;
+		}
+	    }
+
+	    
+	    /*
+	    // dump if too many camelcase words (ideally keep no WET record of that kind?)
 	    if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
 		parentFolder = batchProcessor.discardFolder;
 		System.err.println("@@@DISCARDING - CAMELCASED CONTENTS");
 	    }
-	    else if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
+	    else*/
+	    // For now, don't discount content with too many camelcased words
+	    // Just focus on whether there are a sufficient number of valid words
+	    // (camelcased words are however still ignored in our count of valid words)
+	    if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
 		parentFolder = batchProcessor.keepFolder;
 		System.err.println("@@@KEEPING");
