Index: /gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt	(revision 33615)
@@ -332,2 +332,21 @@
 https://ipinfo.info/html/ip_checker.php
 
+
+
+----------
+MongoDB
+Installation:
+	https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
+		https://docs.mongodb.com/manual/administration/install-on-linux/
+	https://hevodata.com/blog/install-mongodb-on-ubuntu/
+	https://www.digitalocean.com/community/tutorials/how-to-install-mongodb-on-ubuntu-16-04
+	CENTOS (Analytics): https://tecadmin.net/install-mongodb-on-centos/
+	FROM SOURCE: https://github.com/mongodb/mongo/wiki/Build-Mongodb-From-Source
+GUI:
+	https://robomongo.org/
+	Robomongo is Robo 3T now
+
+https://www.tutorialspoint.com/mongodb/mongodb_java.htm
+JAR FILE:
+	http://central.maven.org/maven2/org/mongodb/mongo-java-driver/
+	https://mongodb.github.io/mongo-java-driver/
Index: /gs3-extensions/maori-lang-detection/conf/config.properties
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/config.properties	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/conf/config.properties	(revision 33615)
@@ -21,2 +21,8 @@
 WETprocessor.min.num.words=20
 WETprocessor.max.words.camelcase=10
+
+
+mongodb.user=admin
+mongodb.pwd=pinky
+#CHANGEME
+
Index: /gs3-extensions/maori-lang-detection/conf/log4j.properties
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/log4j.properties	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/conf/log4j.properties	(revision 33615)
@@ -12,11 +12,14 @@
 
 # Let's log everything from the most basic DEBUG msgs on (so INFO, WARN, ERROR too)
-log4j.rootCategory=DEBUG, mldlog
+# Also want to log to the console, see
+# https://stackoverflow.com/questions/3382985/how-to-make-log4j-to-write-to-the-console-as-well
 
-# copied from GS3
-log4j.category.web.servlets.DirectInput=INFO, mldlog
-# Prevent logging duplicating its messages into rootLogger:
-# https://stackoverflow.com/questions/9208710/java-log4j-log-to-both-category-and-root-logger
-log4j.additivity.web.servlets.DirectInput=false
+log4j.rootCategory=DEBUG, console, mldlog
+
+# Define Console Appender
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+# Define the layout for console appender. If you do not define it, you will get an error
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.Target=System.err
 
 # We're saying our log "mldlog" is some file that gets appended to,
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33615)
@@ -150,9 +150,9 @@
 	    
 	} catch(Exception e) {
-	    error("Exception attempting to read properties from config.properties.", e);
+	    logger.error("Exception attempting to read properties from config.properties.", e);
 	}
 
 	if(configProperties.size() == 0) {
-	    warn("*** Warning: no values read into config properties. Using defaults.");
+	    logger.warn("*** Warning: no values read into config properties. Using defaults.");
 	}
 	
@@ -190,18 +190,18 @@
 
 	// prepare our blacklist, greylist (for inspection) and whitelist
-	info("Loading blacklist.");
+	logger.info("Loading blacklist.");
 	blackList = new HashMap<String, Integer>();
 	initURLFilterList(blackList, "url-blacklist-filter.txt");
 	
-	info("Loading greylist.");
+	logger.info("Loading greylist.");
 	greyList = new HashMap<String, Integer>();
 	initURLFilterList(greyList, "url-greylist-filter.txt");
 	
-	info("Loading whitelist.");
+	logger.info("Loading whitelist.");
 	whiteList = new HashMap<String, Integer>();
 	initURLFilterList(whiteList, "url-whitelist-filter.txt");
 
 	// Create the map of topSites
-	info("Loading map of topsites with regex of allowable url patterns for each topsite.");
+	logger.info("Loading map of topsites with regex of allowable url patterns for each topsite.");
 	topSitesMap = new HashMap<String, String>();
 	
@@ -226,12 +226,12 @@
 		topSitesMap.put(topsite, allowed_url_pattern);
 
-		//debug("@@@@ topsite: " + topsite + " - " + allowed_url_pattern);
+		//logger.debug("@@@@ topsite: " + topsite + " - " + allowed_url_pattern);
 		
 	    }
 	} catch(Exception e) {
-	    error("@@@@@@@@@ Error attempting to parse CSV format of text file " + topSitesCSVData, e);
+	    logger.error("@@@@@@@@@ Error attempting to parse CSV format of text file " + topSitesCSVData, e);
 	}	
  
-	//debug("Prematurely terminating for testing purposes.");
+	//logger.debug("Prematurely terminating for testing purposes.");
 	//System.exit(-1);
     }
@@ -323,4 +323,5 @@
 		// for later manual inspection
 		if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
+		    /*
 		    if(!possibleProductDomains.contains(domainWithProtocol)) {
 
@@ -333,5 +334,5 @@
 			    countryCode = ""; // forces domain to be included for inspection
 			    
-			    error("Could not check if domain " + domainWithProtocol
+			    logger.error("Could not check if domain " + domainWithProtocol
 				  + " was in country: " + countryCode,
 				  exceptObj);
@@ -348,5 +349,5 @@
 			    possibleProductSitesWriter.write("\t" + url + "\n");
 			}
-		    } /*else {
+			}*/ /*else {
 			// already wrote out domain to file at some point, write just the URL out to file
 			possibleProductSitesWriter.write("\t" + url + "\n");
@@ -355,5 +356,5 @@
 	    }
 	} catch (IOException ioe) {
-	    error("@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile, ioe);
+	    logger.error("@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile, ioe);
 	}
 
@@ -393,7 +394,7 @@
 	String value = topSitesMap.get("wikipedia.org");
 	if(value == null) {
-	    debug("### wikipedia.org had null value");
+	    logger.debug("### wikipedia.org had null value");
 	} else {
-	    debug("### wikipedia.org had value: " + value);
+	    logger.debug("### wikipedia.org had value: " + value);
 	} // DEBUG
 	    */
@@ -407,6 +408,6 @@
 
 		/*if(domain.contains("docs.google.com")) {
-		    debug("domain with protocol: " + domainWithProtocol);
-		    debug("domain: " + domain);
+		    logger.debug("domain with protocol: " + domainWithProtocol);
+		    logger.debug("domain: " + domain);
 		    }*/
 		
@@ -531,5 +532,5 @@
 		    
 		} catch (IOException ioe) {
-		    error("@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile, ioe);
+		    logger.error("@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile, ioe);
 		}
 		
@@ -537,5 +538,5 @@
 
 	} catch (IOException ioe) {
-	    error("\n@@@@@@@@@ Error writing to one of:\n\t" + seedURLsFile
+	    logger.error("\n@@@@@@@@@ Error writing to one of:\n\t" + seedURLsFile
 			       + "\n\t" + urlFilterFile
 			       + "\n\t" + domainURLsFile
@@ -545,9 +546,9 @@
 	/*
 	// BEGIN DEBUG
-	debug("@@@@ TopSitesMap contains: ");
+	logger.debug("@@@@ TopSitesMap contains: ");
 	for(Map.Entry<String, String> entry : topSitesMap.entrySet()) {
 	    String topSite = entry.getKey();
 	    String urlPattern = entry.getValue();	    
-	    debug(topSite + " - " + urlPattern);
+	    logger.debug(topSite + " - " + urlPattern);
 	} // END DEBUG
 	*/
@@ -620,5 +621,5 @@
 	    if(allowed_url_pattern != null) { // if topSitesMap.containsKey(domain);
 		// there's an entry for the URL in the topSitesMap
-		debug("##### A top site matches URL domain " + domain);
+		logger.debug("##### A top site matches URL domain " + domain);
 
 		// if we're dealing with SUBDOMAIN-COPY, then the fullSeedDomain, with or without
@@ -687,5 +688,5 @@
 	String domainWithoutProtocol = getDomainForURL(url, false); // remove protocol
 	if(!isBlackListed && url.contains("jasmin")) {
-	    warn("### Blacklisting additional domain (likely an adult site): " + domainWithoutProtocol);
+	    logger.warn("### Blacklisting additional domain (likely an adult site): " + domainWithoutProtocol);
 	    blackList.put(domainWithoutProtocol, LIST_ENTRY_CONTAINS);
 	}
@@ -722,5 +723,5 @@
 	// if filterListFilename does not exist in the conf folder, just return
 	if(MY_CLASSLOADER.getResource(filterListFilename) == null) {
-	    warn("Filter list filename: " + filterListFilename + " does not exist");
+	    logger.warn("Filter list filename: " + filterListFilename + " does not exist");
 	    return;	    
 	}
@@ -744,19 +745,19 @@
 		    filter = filter.substring(1);
 		    list.put(filter, LIST_ENTRY_STARTSWITH);
-		    //debug("Match filter startswith: " + filter);
+		    //logger.debug("Match filter startswith: " + filter);
 		}
 		else if(filter.endsWith("$")) {
 		    filter = filter.substring(0, filter.length()-1);
 		    list.put(filter, LIST_ENTRY_ENDSWITH);
-		    //debug("@@@ Match filter endswith: " + filter);
+		    //logger.debug("@@@ Match filter endswith: " + filter);
 		}
 		else {
 		    list.put(filter, LIST_ENTRY_CONTAINS);
 		}
-		//debug("Got filter: " + filter);
+		//logger.debug("Got filter: " + filter);
 	    }
 		
 	} catch (IOException ioe) {
-	    error("@@@@@@@@@ Error reading into map from file " + filterListFilename, ioe);
+	    logger.error("@@@@@@@@@ Error reading into map from file " + filterListFilename, ioe);
 	}
 	
@@ -781,5 +782,5 @@
 	for(int i = 0; i < WETFiles.length; i++) {
 	    File WETFile = WETFiles[i];	    
-	    debug("Processing WETfile: " + WETFile);
+	    logger.debug("Processing WETfile: " + WETFile);
 
 	    // Any .gz files listed means they haven't been unzipped yet. So unzip.
@@ -797,5 +798,5 @@
 
 	    if(!WETFile.exists() || !WETFile.isFile()) {		
-		error("Error: " + WETFile + " does not exist (failure to unzip?)");
+		logger.error("Error: " + WETFile + " does not exist (failure to unzip?)");
 		return;
 	    }
@@ -803,5 +804,5 @@
 	    // Finally, we can process this WETFile's records into the keep and discard pile
 	    wetFileCount++;
-	    debug("Off to process " + WETFile);
+	    logger.debug("Off to process " + WETFile);
 	    String crawlID = ccrawlWETFileDir.getName(); // something like CC-MAIN-YYYY-##-wet-files
 	    crawlID = crawlID.substring("CC-MAIN-".length(), crawlID.indexOf("-wet-files")); // YYYY-##	    
@@ -817,30 +818,8 @@
 
 
-    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-	System.err.println(msg);
-	logger.info(msg);
-    }
-    public static void debug(String msg) {
-	System.err.println(msg);
-	logger.debug(msg);
-    }
-    public static void warn(String msg) {
-	System.err.println(msg);
-	logger.warn(msg);
-    }
-    public static void error(String msg) {
-	System.err.println(msg);
-	logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-	logger.error(msg, e);
-	System.err.println("\n"+msg);
-	e.printStackTrace();
-    }
-    
+    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //    
     public static void printUsage() {
-	info("Run this program as:");
-	info("\tCCWetProcessor <path to 'ccrawl-data' folder> <output folder path>");	
+	System.err.println("Run this program as:");
+	System.err.println("\tCCWetProcessor <path to 'ccrawl-data' folder> <output folder path>");	
     }
 
@@ -852,5 +831,5 @@
 	public boolean accept(File dir, String name) {
 	    if(name.endsWith(".warc.wet")) {
-		debug("Will include " + name + " for processing.");
+		logger.debug("Will include " + name + " for processing.");
 		return true;
 	    }
@@ -860,10 +839,10 @@
 		File unzippedVersion = new File(dir, nameWithoutGZext);
 		if(unzippedVersion.exists()) {
-		    debug("--- Unzipped version " + unzippedVersion + " exists.");
-		    debug("Skipping " + name);
+		    logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
+		    logger.debug("Skipping " + name);
 		    return false; // don't count gzipped version if unzipped version exists.
 		}
 		else {
-		    debug("Only zipped version " + name + " exists.");
+		    logger.debug("Only zipped version " + name + " exists.");
 		    return true; // No unzipped version, so have to work with gzipped version
 		}
@@ -871,5 +850,5 @@
 
 	    // we're not even interested in any other file extensions
-	    debug("Not a WET file. Skipping " + name);
+	    logger.debug("Not a WET file. Skipping " + name);
 	    return false;
 	}
@@ -887,5 +866,5 @@
 	    }
 	    else {
-		info("File " + f + " is not a directory");
+		logger.info("File " + f + " is not a directory");
 	    }
 	    return false;				   
@@ -901,5 +880,5 @@
 	File commoncrawlDir = new File(args[0]);
 	if(!commoncrawlDir.exists() || !commoncrawlDir.isDirectory()) {
-	    error("Error: " + args[0] + " does not exist or is not a directory");
+	    logger.error("Error: " + args[0] + " does not exist or is not a directory");
 	    return;
 	}
@@ -907,5 +886,5 @@
 	File outFolder = new File(args[1]);
 	if(!outFolder.exists() || !outFolder.isDirectory()) {
-	    error("Error: " + args[1] + " does not exist or is not a directory.");
+	    logger.error("Error: " + args[1] + " does not exist or is not a directory.");
 	    return;
 	}	
@@ -918,5 +897,5 @@
 	for(int i = 0; i < ccrawlFolders.length; i++) {
 	    File ccrawlFolder = ccrawlFolders[i];
-	    info("About to process commoncrawl WET files folder: " + ccrawlFolder);
+	    logger.info("About to process commoncrawl WET files folder: " + ccrawlFolder);
 	    ccWETFilesProcessor.processAllWETFilesOfCrawl(ccrawlFolder);	    
 	}
@@ -933,12 +912,12 @@
 	ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile, domainURLsFile, topSitesMatchedFile, possibleProductSitesFile);
 
-	info("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
-
-	info("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
+	logger.info("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
+
+	logger.info("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
 	
 	
     } catch(Exception e) {
 	// can get an exception when instantiating CCWETProcessor instance
-	error(e.getMessage(), e);
+	logger.error(e.getMessage(), e);
     }
 	
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java	(revision 33615)
@@ -12,10 +12,14 @@
 package org.greenstone.atea;
 
-import java.io.*;
+
 import opennlp.tools.langdetect.*;
 import opennlp.tools.sentdetect.*;
 import opennlp.tools.util.*;
 
+import java.io.*;
 import java.util.ArrayList;
+
+//import org.apache.log4j.Logger;
+
 
 /**
@@ -27,13 +31,13 @@
  *
  * Then, to compile this program, do the following from the "src" folder (the folder containing this java file):
- *    maori-lang-detection/src$ javac -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/MaoriTextDetector.java
+ *    maori-lang-detection/src$ javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/MaoriTextDetector.java
  *
  * To run this program, issue one of the following commands from the "src" folder (the folder containing this java file):
  *
- *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --help
- *
- *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --file &lt;full/path/to/textfile&gt;
- *
- *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector -
+ *    maori-lang-detection/src$ java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --help
+ *
+ *    maori-lang-detection/src$ java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --file &lt;full/path/to/textfile&gt;
+ *
+ *    maori-lang-detection/src$ java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector -
  *       Press enter. This variant of the program expects text to stream in from standard input.
  *       If entering text manually, then remember to press Ctrl-D to indicate the usual end of StdIn.
@@ -43,4 +47,6 @@
  */
 public class MaoriTextDetector extends TextLanguageDetector {
+    //static Logger logger = Logger.getLogger(org.greenstone.atea.MaoriTextDetector.class.getName());
+    
     /** The 3 letter language code for Maori in ISO 639-2 or ISO 639-3 */ 
     public static final String MAORI_3LETTER_CODE = "mri";
@@ -135,5 +141,5 @@
     }
     public static void doPrintErr(boolean runSilent, String msg) {
-	if(!runSilent) System.err.println(msg);	
+	if(!runSilent) System.err.println(msg);
     }
     
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33615)
@@ -49,5 +49,5 @@
 */
 public class NutchTextDumpProcessor {
-    private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
+    static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
 
     static boolean DEBUG_MODE = true;
@@ -91,9 +91,9 @@
 	if(DEBUG_MODE) {
 	    // START DEBUG
-	    debug("__________________________________________");
-	    debug("@@@ Found page entry: ");
-	    debug("__________________________________________");
-	    debug(pageDump.toString());
-	    debug("------------------------------------------");
+	    logger.debug("__________________________________________");
+	    logger.debug("@@@ Found page entry: ");
+	    logger.debug("__________________________________________");
+	    logger.debug(pageDump.toString());
+	    logger.debug("------------------------------------------");
 	    // END DEBUG
 	}
@@ -174,5 +174,5 @@
 		
 	} catch (IOException ioe) {
-	    error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
+	    logger.error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
 	}
 
@@ -302,29 +302,29 @@
 	
 	
-	info("------------- " + this.siteID + " SITE STATS -----------");
-
-	info("SITE DOMAIN: " + this.domainOfSite);
-	info("Total number of web pages in site: " + pages.size());
-	info("Of these, the number of pages in MÄori (mri) were: " + this.pagesInMRI.size());
+	logger.info("------------- " + this.siteID + " SITE STATS -----------");
+
+	logger.info("SITE DOMAIN: " + this.domainOfSite);
+	logger.info("Total number of web pages in site: " + pages.size());
+	logger.info("Of these, the number of pages in MÄori (mri) were: " + this.pagesInMRI.size());
 	
 	if(pagesInMRI.size() > 0) {
-	    info("The following were the pages detected by OpenNLP as being in MÄori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
+	    logger.info("The following were the pages detected by OpenNLP as being in MÄori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
 	    for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
-		info(mriWebPageInfo.toString());
-	    }
-	}
-
-	info("                      -----------                   ");
+		logger.info(mriWebPageInfo.toString());
+	    }
+	}
+
+	logger.info("                      -----------                   ");
 	if(pagesContainingMRI.size() > 0) {	    
-	    info("The following pages weren't detected as primarily being in MÄori");
-	    info("But still contained sentences detected as MÄori");
+	    logger.info("The following pages weren't detected as primarily being in MÄori");
+	    logger.info("But still contained sentences detected as MÄori");
 	    for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
-		info(mriWebPageInfo.toString());
+		logger.info(mriWebPageInfo.toString());
 	    }
 	    
 	} else {
-	    info("No further pages detected as containing any sentences in MRI");	   
-	}
-	info("                      -----------                   ");
+	    logger.info("No further pages detected as containing any sentences in MRI");	   
+	}
+	logger.info("                      -----------                   ");
     }
 
@@ -358,29 +358,8 @@
     
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-	System.err.println(msg);
-	logger.info(msg);
-    }
-    public static void debug(String msg) {
-	System.err.println(msg);
-	logger.debug(msg);
-    }
-    public static void warn(String msg) {
-	System.err.println(msg);
-	logger.warn(msg);
-    }
-    public static void error(String msg) {
-	System.err.println(msg);
-	logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-	logger.error(msg, e);
-	System.err.println("\n"+msg);
-	e.printStackTrace();
-    }
-    
+   
     public static void printUsage() {
-	info("Run this program as:");
-	info("\tNutchTextDumpProcessor <path to 'crawled' folder>");
+	System.err.println("Run this program as:");
+	System.err.println("\tNutchTextDumpProcessor <path to 'crawled' folder>");
     }
     
@@ -393,5 +372,5 @@
 	File sitesDir = new File(args[0]);
 	if(!sitesDir.exists() || !sitesDir.isDirectory()) {
-	    error("Error: " + args[0] + " does not exist or is not a directory");
+	    logger.error("Error: " + args[0] + " does not exist or is not a directory");
 	    return;
 	}
@@ -430,5 +409,5 @@
 		    File txtDumpFile = new File(siteDir, "dump.txt");
 		    if(!txtDumpFile.exists()) {
-			error("Text dump file " + txtDumpFile + " did not exist");
+			logger.error("Text dump file " + txtDumpFile + " did not exist");
 			continue;
 		    }
@@ -439,5 +418,5 @@
 			String siteID = siteDir.getName();
 			long lastModified = siteDir.lastModified();
-			debug("Found siteID: " + siteID);		    
+			logger.debug("Found siteID: " + siteID);		    
 			NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(
 				 webpagesCSVPrinter, mriSentencesCSVPrinter, mriTxtDetector,
@@ -458,5 +437,5 @@
 	    // can get an exception when instantiating CCWETProcessor instance
 	    // or with CSV file
-	    error(e.getMessage(), e);
+	    logger.error(e.getMessage(), e);
 	}
     }
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33615)
@@ -38,12 +38,12 @@
 	    tuples.put("key", key.trim());
 	    //} else {
-	    //debug("@@@@ no key for pageURL: " + pageURL);
+	    //logger.debug("@@@@ no key for pageURL: " + pageURL);
 	    //}
 	    /*
 	    if(pageURL.contains(TEXT_END_MARKER)) {
-		debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
-		debug("+++++++++");
-		debug(unparsedPageDump);
-		debug("+++++++++");
+		logger.debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
+		logger.debug("+++++++++");
+		logger.debug(unparsedPageDump);
+		logger.debug("+++++++++");
 	    }
 	    */
@@ -87,6 +87,6 @@
 			} else {
 			    if(NutchTextDumpProcessor.DEBUG_MODE) {
-				error("No meta key for meta: " + line);
-				error(unparsedPageDump);
+				logger.error("No meta key for meta: " + line);
+				logger.error(unparsedPageDump);
 			    }
 			}
@@ -103,5 +103,5 @@
 	    
 	} catch (IOException ioe) {
-	    error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
+	    logger.error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
 	}
 
@@ -115,11 +115,11 @@
     public void debugTuples() {
 	if(NutchTextDumpProcessor.DEBUG_MODE) {
-	    debug("__________________________________________");
+	    logger.debug("__________________________________________");
 	    for(Map.Entry<String, String> entry : tuples.entrySet()) {
 		String key = entry.getKey();
 		String value = entry.getValue();	    
-		debug(key + " - " + value);
+		logger.debug(key + " - " + value);
 	    }
-	    debug("__________________________________________");
+	    logger.debug("__________________________________________");
 	}
     }
@@ -164,26 +164,3 @@
     }
     
-    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-	System.err.println(msg);
-	logger.info(msg);
-    }
-    public static void debug(String msg) {
-	System.err.println(msg);
-	logger.debug(msg);
-    }
-    public static void warn(String msg) {
-	System.err.println(msg);
-	logger.warn(msg);
-    }
-    public static void error(String msg) {
-	System.err.println(msg);
-	logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-	logger.error(msg, e);
-	System.err.println("\n"+msg);
-	e.printStackTrace();
-    }
-    
 }
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33614)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33615)
@@ -77,5 +77,5 @@
 
 	String fileID = inFile.getName();
-	//debug("*** Processing wetfile: " + fileID);		    
+	//logger.debug("*** Processing wetfile: " + fileID);		    
 	fileID = fileID.substring(fileID.lastIndexOf("0")+1);
 	if(fileID.startsWith(".")) { // took off too many zeroes, as happens with *000000.warc.wet
@@ -195,12 +195,12 @@
 				  String recordURI, String record)
     {
-	info("CrawlID: CC-MAIN-" + this.crawlID
+	logger.info("CrawlID: CC-MAIN-" + this.crawlID
 			   + " WET #" + this.WETFileID
 			   + " record #" + recordID
 			   + " - contentLength: " + contentLength
 			   + " - lineCount: " + lineCount);
-	info("URI: " + recordURI);
-	//debug(record);
-	//info("--------------------------");
+	logger.info("URI: " + recordURI);
+	//logger.debug(record);
+	//logger.info("--------------------------");
 
 	File parentFolder = null;
@@ -215,9 +215,9 @@
 	    else if(batchProcessor.isGreylisted(recordURI)) {
 		parentFolder = batchProcessor.greyListedFolder;
-		debug("@@@GREYLISTED");
+		logger.debug("@@@GREYLISTED");
 	    }
 	    else { // url was only blacklisted
 		parentFolder = batchProcessor.discardFolder;
-		debug("@@@DISCARDING - blacklisted");
+		logger.debug("@@@DISCARDING - blacklisted");
 	    }
 	}
@@ -229,5 +229,5 @@
 	    else {
 		parentFolder = batchProcessor.greyListedFolder;
-		debug("@@@GREYLISTED");
+		logger.debug("@@@GREYLISTED");
 	    }
 	}
@@ -274,5 +274,5 @@
 	    if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
 		parentFolder = batchProcessor.discardFolder;
-		debug("@@@DISCARDING - CAMELCASED CONTENTS");
+		logger.debug("@@@DISCARDING - CAMELCASED CONTENTS");
 	    }
 	    else*/
@@ -282,5 +282,5 @@
 	    if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
 		parentFolder = batchProcessor.keepFolder;
-		debug("@@@KEEPING");
+		logger.debug("@@@KEEPING");
 	    }
 	}
@@ -289,5 +289,5 @@
 	if(parentFolder == null) {
 	    parentFolder = batchProcessor.discardFolder;
-	    debug("@@@DISCARDING");
+	    logger.debug("@@@DISCARDING");
 	}
 
@@ -301,9 +301,9 @@
 	    }
 	} catch(Exception e) {
-	    debug("Unable to write URL");
+	    logger.debug("Unable to write URL");
 	    e.printStackTrace();
 	}
    
-	debug("--------------------------");
+	logger.debug("--------------------------");
 
 	// outFilename will look something like YYYY-##-####
@@ -319,29 +319,8 @@
 	} catch(IOException ioe) {
 	    ioe.printStackTrace();
-	    error("@@@@@@@@@ Error writing to file " + outFile, ioe);
+	    logger.error("@@@@@@@@@ Error writing to file " + outFile, ioe);
 	}
     }
 
 
-    public void info(String msg) {
-	System.err.println(msg);
-	logger.info(msg);
-    }
-    public void debug(String msg) {
-	System.err.println(msg);
-	logger.debug(msg);
-    }
-    public void warn(String msg) {
-	System.err.println(msg);
-	logger.warn(msg);
-    }
-    public void error(String msg) {
-	System.err.println(msg);
-	logger.error(msg);
-    }
-    public void error(String msg, Exception e) {
-	logger.error(msg, e);
-	System.err.println("\n"+msg);
-	e.printStackTrace();
-    }
 }
