Index: /gs3-extensions/maori-lang-detection/MoreReading/CommonCrawl.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/MoreReading/CommonCrawl.txt	(revision 33466)
+++ /gs3-extensions/maori-lang-detection/MoreReading/CommonCrawl.txt	(revision 33467)
@@ -47,4 +47,14 @@
 Sebastian
 
+====================
+wharariki:[239]/Scratch/ak19/gs3-extensions/maori-lang-detection/src>java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
+
+wharariki:[188]/Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET>ls keep | wc
+   4090    4090   65440
+wharariki:[189]/Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET>ls discard | wc
+   1515    1515   24240
+
+We keep 4090 WET records and are discarding 1515.
+
 =======================
 Latest version of the index's schema:
Index: /gs3-extensions/maori-lang-detection/MoreReading/Vagrant-Spark-Hadoop.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/MoreReading/Vagrant-Spark-Hadoop.txt	(revision 33466)
+++ /gs3-extensions/maori-lang-detection/MoreReading/Vagrant-Spark-Hadoop.txt	(revision 33467)
@@ -221,5 +221,4 @@
  
 
-
 vagrant@node1:~$ locate guava.jar
 /usr/share/java/guava.jar
@@ -243,5 +242,5 @@
 vagrant@node1:~/ia-hadoop-tools$ hdfs dfs -put /usr/share/java/guava.jar /usr/local/hadoop/share/hadoop/common/.
 put: `/usr/local/hadoop/share/hadoop/common/.': No such file or directory
-# hadoop classpath locations are not hdfs filesystem
+# hadoop classpath locations are not on the hdfs filesystem, but on the regular fs
 
 vagrant@node1:~/ia-hadoop-tools$ sudo cp /usr/share/java/guava.jar /usr/local/hadoop/share/hadoop/common/.
Index: /gs3-extensions/maori-lang-detection/conf/config.properties
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/config.properties	(revision 33466)
+++ /gs3-extensions/maori-lang-detection/conf/config.properties	(revision 33467)
@@ -9,2 +9,8 @@
 # for downloading a single file
 wget.file.cmd=wget %%FILE_URL%%
+
+# Arbitrary cutoff values for WETProcessor.java
+WETprocessor.min.content.length=100
+WETprocessor.min.line.count=2
+WETprocessor.min.content.length.wrapped.line=500
+WETprocessor.min.spaces.per.wrapped.line=10
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33466)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33467)
@@ -4,5 +4,8 @@
 import java.util.zip.GZIPInputStream;
 
+import org.apache.log4j.Logger;
+
 public class Utility {
+    private static Logger logger = Logger.getLogger(org.greenstone.atea.Utility.class.getName());
     
     // Run gunzip
@@ -26,5 +29,5 @@
 	    //out.close();
 	    
-	    //log("Unzipped " + inZipFile + " to " + outFile);
+	    logger.debug("Unzipped " + inZipFile + " to " + outFile);
 	    
 	} catch(IOException ex) {
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33466)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33467)
@@ -5,4 +5,5 @@
 import java.util.Properties;
 import java.util.zip.GZIPInputStream;
+
 import org.apache.log4j.Logger;
 
@@ -26,17 +27,36 @@
 */
 public class WETProcessor {
-    //private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());
-    private Properties configProperties = null;
-
+    private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());    
+    private static Properties configProperties = new Properties();
+
+    // In Java, can initialize static final variables inside a static block
+    // But the unavoidable try/catch in this static block prevents initialization of
+    // the static final int variables further below that therefore need to be declared
+    // and initialized thereafter.
+    static {	
+	
+	// load up the properties from the config file
+	try (InputStream infile = org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
+	    configProperties = new Properties();
+	    configProperties.load(infile);	    
+	    //infile.close();
+	    
+	} catch(Exception e) {
+	    System.err.println("Exception attempting to read properties from config.properties.");
+	    logger.error("Exception attempting to read properties from config.properties.");
+	    e.printStackTrace();
+	}
+    }
     
-    // arbitrary cut-off values, TODO: put in .properties file
-    private static final int MIN_CONTENT_LENGTH = 100;
-    private static final int MIN_LINE_COUNT = 2;
-    private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = 500;
-    private static final int MIN_SPACES_IN_A_WRAPPED_LINE = 10;    
-
-    // keep a count of all the records that all WETProcessors instantiated
-    // by our main method have processed
-    private static int recordCount = 0;
+    // Providing fall-back cuttoff values if config.properties doesn't load
+    // or doesn't have the named props. But what happens when Integer.parseInt throws an exception?
+    private static final int MIN_CONTENT_LENGTH = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length", "100"));
+    private static final int MIN_LINE_COUNT= Integer.parseInt(configProperties.getProperty("WETprocessor.min.line.count", "2"));
+    private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length.wrapped.line", "500"));
+    private static final int MIN_SPACES_IN_A_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.spaces.per.wrapped.line", "10"));
+    
+    // Keep a count of all the records that all WETProcessors instantiated
+    // by our main method combined have processed
+    //private static int recordCount = 0;
 
     private final File outputFolder;
@@ -44,15 +64,18 @@
 
 
-    public static int getRecordCount() { return recordCount; }
-
-    public WETProcessor(File inFile, File outFolder, Properties configProps) {
+    /**
+     * WET processor processes a single warc.wet file containing multiple WET records
+     * containing text identified as primary langcode=mri. Each individual WET record is written
+     * out to a uniquely named file in either the keep or discard folder depending on the WET
+     * record's content length and number of lines of actual content (excluding WARC headers).
+     */
+    public WETProcessor(File inFile, File outFolder) {
 	this.outputFolder = outFolder;
-	this.configProperties = configProps;
 	
 	StringBuilder record = null;
 	String line = null;	
 	boolean readingRecord = false;
-
-	//this.recordCount = 0;
+	
+	int recordCount = 0;
 	
 	int contentLength = -1; // of record
@@ -62,5 +85,6 @@
 	// inFile name looks something like MAORI-CC-2019-30-20190902100139-000000.warc.wet
 	// the prefix will be everything after the last hyphen and without file extension,
-	// so "000000" in our example.
+	// so "000000" in our example. Then suffix the recordCount (keeping track of the current
+	// WET record) to get a unique filename to store each WET record into.
 
 	String fileID = inFile.getName();
@@ -129,5 +153,9 @@
     }
 
-    
+    /**
+     * Determines if a WET record belongs in the keep or discard pile depending on if it
+     * contains enough text, based on contentLength and line count of the record body.
+     * Then writes out the WET record to a uniquely named file in the keep or discard folder.
+     */
     private void processWETrecord(int recordID, int contentLength, int lineCount, String record)
     {
@@ -171,5 +199,7 @@
 	try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
 	    writer.write(record);
-	    writer.close();
+	    // Try-with-resources examples don't call close() explicitly:
+	    // https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
+	    //writer.close();
 	} catch(IOException ioe) {
 	    ioe.printStackTrace();
@@ -178,4 +208,6 @@
     }
     
+
+    //public static int getRecordCount() { return recordCount; }
     
     public static void printUsage() {
@@ -184,5 +216,5 @@
     }
 
-    /** Filename filter that only lists warc.wet files or else warc.wet.gz files
+    /** Filename filter to only list warc.wet files or else warc.wet.gz files
      * for which unzipped warc.wet equivalents don't yet exist.
      */
@@ -191,5 +223,5 @@
 	public boolean accept(File dir, String name) {
 	    if(name.endsWith(".warc.wet")) {
-		System.err.println("Will include " + name + " for processing.");
+		logger.debug("Will include " + name + " for processing.");
 		return true;
 	    }
@@ -199,17 +231,16 @@
 		File unzippedVersion = new File(dir, nameWithoutGZext);
 		if(unzippedVersion.exists()) {
-		    System.err.println("--- Unzipped version " + unzippedVersion + " exists.");
-		    System.err.println("Skipping " + name);
+		    logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
+		    logger.debug("Skipping " + name);
 		    return false; // don't count gzipped version if unzipped version exists.
 		}
 		else {
-		    System.err.println("Only zipped version " + name + " exists.");
+		    logger.debug("Only zipped version " + name + " exists.");
 		    return true; // No unzipped version, so have to work with gzipped version
 		}
 	    }
-	    
-	    System.err.println("Skipping " + name);
 
 	    // we're not even interested in any other file extensions
+	    logger.debug("Not a WET file. Skipping " + name);
 	    return false;
 	}
@@ -225,5 +256,5 @@
 	File WETFileDir = new File(args[0]);
 	if(!WETFileDir.exists() || !WETFileDir.isDirectory()) {
-	    System.err.println("Error: " + args[0] + " does not exist or is not a directory");
+	    System.out.println("Error: " + args[0] + " does not exist or is not a directory");
 	    return;
 	}
@@ -231,5 +262,5 @@
 	File outFolder = new File(args[1]);
 	if(!outFolder.exists() || !outFolder.isDirectory()) {
-	    System.err.println("Error: " + args[1] + " does not exist or is not a directory.");
+	    System.out.println("Error: " + args[1] + " does not exist or is not a directory.");
 	    return;
 	}	
@@ -244,17 +275,4 @@
 	}
 
-	// load up the properties from the config file
-	Properties configProps;
-	try (InputStream infile = Class.forName("org.greenstone.atea.WETProcessor").getClassLoader().getResourceAsStream("config.properties")) {
-	    configProps = new Properties();
-	    configProps.load(infile);
-	    infile.close();
-	    
-	} catch(Exception e) {
-	    System.err.println("Exception attempting to read properties from config.properties.");
-	    e.printStackTrace();
-	    return;
-	}
-
 	// Will list all the warc.wet files in the input directory or else their gzipped versions
 	File[] WETFiles = WETFileDir.listFiles(new WETFilenameFilter());
@@ -262,5 +280,5 @@
 	for(int i = 0; i < WETFiles.length; i++) {
 	    File WETFile = WETFiles[i];	    
-	    System.err.println("Processing WETfile: " + WETFile);
+	    logger.debug("Processing WETfile: " + WETFile);
 
 	    // Any .gz files listed means they haven't been unzipped yet. So unzip.
@@ -279,10 +297,11 @@
 	    if(!WETFile.exists() || !WETFile.isFile()) {
 		System.err.println("Error: " + WETFile + " does not exist (failure to unzip?)");
+		logger.error("Error: " + WETFile + " does not exist (failure to unzip?)");
 		return;
 	    }
 
 	    // Finally, we can process this WETFile's records into the keep and discard pile
-	    System.err.println("Off to process " + WETFile);
-	    WETProcessor processor = new WETProcessor(WETFile, outFolder, configProps);	    
+	    logger.debug("Off to process " + WETFile);
+	    WETProcessor processor = new WETProcessor(WETFile, outFolder);	    
 	    
 	}
