Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33465)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java	(revision 33465)
@@ -0,0 +1,304 @@
+
+
+package org.greenstone.atea;
+
+
+import java.io.*;
+
+/**
+ * To compile:
+ *      maori-lang-detection/src>javac -cp "." org/greenstone/atea/WETProcessor.java
+ * To run:
+ *      java org.greenstone.atea.WETProcessor <wetfile>.wet.warc <outputFolder>
+ * e.g.
+ *    - java org.greenstone.atea.WETProcessor ../wetprocessor/MAORI-CC-2019-30-20190902100139-000000.warc.wet /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
+ *    - java org.greenstone.atea.WETProcessor ../wetprocessor/MAORI-CC-2019-30-20190902100139-000000.warc.wet /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
+*/
+public class WETProcessor {
+    // arbitrary cut-off values, TODO: put in .properties file
+    private static final int MIN_CONTENT_LENGTH = 100;
+    private static final int MIN_LINE_COUNT = 2;
+    private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = 500;
+    private static final int MIN_SPACES_IN_A_WRAPPED_LINE = 10;    
+    
+    private final File outputFolder;
+    private final String WETFileID;    
+    private int recordCount = -1;
+    
+    /*
+    public WETProcessor(File inFile) {
+
+	StringBuilder record = null;
+	String line = null;	
+	boolean readingRecord = false;
+	
+	// read from WETfile
+	try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
+	    while((line = reader.readLine()) != null) { // readLine removes newline separator
+
+		if(line.startsWith("Content-Length:")) {
+		    readingRecord = true;
+		    record = new StringBuilder();
+		    continue;
+		}
+
+		if(readingRecord) {
+		    if(line.startsWith("WARC/1.0")) { // finished previous WET record
+			readingRecord = false;
+			
+			System.err.println("WET record:");
+			System.err.println(record);
+			System.err.println("--------------------------");
+			record = null;
+
+			processWETrecord(record);
+		    }
+		    else {
+			record.append(line + "\n"); // add back (unix style) line ending
+		    }
+		}
+	    }
+	} catch(IOException ioe) {
+	    ioe.printStackTrace();
+	}
+    }
+
+    public WETProcessor(File inFile, File outFolder) {
+	this.outputFolder = outFolder;
+    
+	StringBuilder record = null;
+	String line = null;	
+	boolean readingRecord = false;
+
+	int recordCount = 0;
+	
+	// read from WETfile
+	try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
+	    
+	    while((line = reader.readLine()) != null) { // readLine removes newline separator
+
+		if(line.startsWith("WARC-Type: warcinfo")) {
+		    readingRecord = false;
+		    record = null; // drop this record, which is just an info record not actual web page's text
+		    recordCount--;
+		    continue;
+		}
+		
+		if(line.startsWith("WARC/1.0")) { // finished previous WET record
+		    // process any previous record
+		    if(record != null) {
+			processWETrecord(record, recordCount);
+			record = null;
+		    }
+		    
+		    recordCount++;		    
+		    // get ready to start a new record
+		    readingRecord = true;
+		    record = new StringBuilder();
+		}
+
+		if(readingRecord) { // append current line to current record
+		    record.append(line + "\n"); // add back (unix style) line ending
+		}
+		
+	    }
+
+	    // flush the last record. If it was a warcinfo record, record would be null here
+	    if(record != null) {
+		processWETrecord(record, recordCount);
+		record = null;
+	    }
+	    
+	} catch(IOException ioe) {
+	    ioe.printStackTrace();
+	}
+    }
+    */
+
+    public WETProcessor(File inFile, File outFolder) {
+	this.outputFolder = outFolder;
+    
+	StringBuilder record = null;
+	String line = null;	
+	boolean readingRecord = false;
+
+	this.recordCount = 0;
+	int contentLength = -1; // of record
+	int lineCount = -1; // actual number of non-empty lines in record body (i.e. excludes WET/WARC headers)
+
+	// We just want a unique recordID prefix, which we get from the wet file name suffix:
+	// inFile name looks something like MAORI-CC-2019-30-20190902100139-000000.warc.wet
+	// the prefix will be everything after the last hyphen and without file extension,
+	// so "000000" in our example.
+
+	String fileID = inFile.getName();
+	fileID = fileID.substring(fileID.lastIndexOf("-")+1);
+	fileID = fileID.substring(0, fileID.indexOf("."));
+	this.WETFileID = fileID;
+	
+	// read from WETfile
+	try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
+	    
+	    while((line = reader.readLine()) != null) { // readLine removes newline separator
+
+		if(line.startsWith("WARC-Type: warcinfo")) {
+		    readingRecord = false;
+		    record = null; // drop this record, which is just an info record not actual web page's text
+		    recordCount--;
+		    continue;
+		}
+		
+		if(line.startsWith("WARC/1.0")) { // finished previous WET record
+		    // process any previous record
+		    if(record != null) {
+			processWETrecord(recordCount, contentLength, lineCount, record.toString());
+			record = null;
+			contentLength = -1;
+			lineCount = -1;
+		    }
+		    
+		    recordCount++;		    
+		    // get ready to start a new record
+		    readingRecord = true;
+		    record = new StringBuilder();
+		}
+
+		if(readingRecord) { // append current line to current record
+
+		    record.append(line + "\n"); // add back (unix style) line ending
+
+		    // if the line is non-empty
+		    // AND if we've started counting lines, which happens only when the current
+		    // line is past WARC/WET headers and we're into the actual body portion
+		    // of the WET record,
+		    // start incrementing the line counter.
+		    if(lineCount >= 0 && !line.trim().equals("")) {
+			lineCount++;
+		    }
+		    else if(line.startsWith("Content-Length:")) {
+			String contentLengthStr = line.substring("Content-Length:".length()).trim();
+			contentLength = Integer.parseInt(contentLengthStr);
+			lineCount = 0;
+		    }
+		    
+		}
+		
+	    }
+
+	    // flush the last record. If it was a warcinfo record, record would be null here
+	    if(record != null) {
+		processWETrecord(recordCount, contentLength, lineCount, record.toString());
+		record = null;
+	    }
+	    
+	} catch(IOException ioe) {
+	    ioe.printStackTrace();
+	}
+    }
+
+    /*
+    private void processWETrecord(StringBuilder str, int recordID) {
+	String record = str.toString();
+	//System.err.println("WET record:");
+	//System.err.println(record);
+	//System.err.println("--------------------------");
+
+	String paddedFileName = String.format("%04d.txt", recordID);
+	File outFile = new File(this.outputFolder, paddedFileName);
+	try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
+	    writer.write(record);
+	    writer.close();
+	} catch(IOException ioe) {
+	    ioe.printStackTrace();
+	    System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
+	}
+    }
+    */
+
+    public int getRecordCount() { return this.recordCount; }
+    
+    private void processWETrecord(int recordID, int contentLength, int lineCount, String record) {
+	System.err.println("WET record - contentLength: " + contentLength + " - lineCount: " + lineCount);
+	//System.err.println(record);
+	//System.err.println("--------------------------");
+
+	String paddedFileName = String.format("%04d.txt", recordID);
+	
+	File discardFolder = new File(this.outputFolder, "discard");		
+	File keepFolder = new File(this.outputFolder, "keep");
+	File parentFolder = null;
+	
+	if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
+	    parentFolder = keepFolder;
+	    System.err.println("@@@KEEPING");
+	} else if(contentLength >= MIN_CONTENT_LENGTH_WRAPPED_LINE) {
+	    int countSpaces = 0;
+	    for(int i = 0; i < record.length(); i++) {
+		if(record.charAt(i) == ' ') countSpaces++;
+	    }
+	    if(countSpaces >= MIN_SPACES_IN_A_WRAPPED_LINE) {
+		// So we have at least 500 chars (possibly on a single wrapped line)
+		// containing at least 10 spaces. Such a record is also worth keeping.
+		parentFolder = keepFolder;
+	    }
+	}
+
+	// if parentFolder still not set, set to discard pile folder
+	if(parentFolder == null) {
+	    parentFolder = discardFolder;
+	    System.err.println("@@@DISCARDING");
+	}
+
+	System.err.println("--------------------------");
+	
+	File outFile = new File(parentFolder, this.WETFileID + "-" + paddedFileName);
+	
+	try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
+	    writer.write(record);
+	    writer.close();
+	} catch(IOException ioe) {
+	    ioe.printStackTrace();
+	    System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
+	}
+    }
+    
+    
+    public static void printUsage() {
+	System.err.println("Run this program as:");
+	System.err.println("\tWetProcessor <infile>.warc.wet <output folder path>");	
+    }
+	
+    public static void main(String[] args) {
+	if(args.length != 2) {
+	    printUsage();
+	    return;
+	}
+
+	File WETfile = new File(args[0]);
+	if(!WETfile.exists() || !WETfile.isFile()) {
+	    System.err.println("Error: " + args[0] + " does not exist or is not a file");
+	    return;
+	}
+	
+	File outFolder = new File(args[1]);
+	if(!outFolder.exists() || !outFolder.isDirectory()) {
+	    System.err.println("Error: " + args[1] + " does not exist or is not a directory.");
+	    return;
+	}
+
+	
+	
+	File discardFolder = new File(outFolder, "discard");
+	if(!discardFolder.exists()) {
+	    discardFolder.mkdir();
+	}	
+	File keepFolder = new File(outFolder, "keep");
+	if(!keepFolder.exists()) {
+	    keepFolder.mkdir();
+	}	
+	
+	WETProcessor processor = new WETProcessor(WETfile, outFolder);
+	return;
+	
+    }
+}
