source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java@ 33501

package org.greenstone.atea;


import java.io.*;
import java.util.Properties;
import java.util.zip.GZIPInputStream;
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;

import org.apache.log4j.Logger;

/**
 * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
 * the WET records in each, putting each WET record into a file. Each file is put into a
 * keep or discard folder, based on content-length and number of lines.
 * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
 *
 * To compile, including the jars in lib/ for compiling.
 *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/WETProcessor.java 
 *
 * To run, passing the log4j and other properties files in conf/ folder:
 *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor <folder containing warc.wet(.gz) files> <outputFolder>
 *
 * e.g.
 *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
 *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
 *
*/
public class WETProcessor {
    private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());

    // WARC WET header lines and header line prefixes of interest
    static final String WARC_RECORD_START = "WARC/1.0";
    static final String WARC_INFO_HEADER = "WARC-Type: warcinfo";
    static final String WARC_TARGET_URI_HEADER_PREFIX = "WARC-Target-URI:";
    static final String WARC_CONTENT_LENGTH_HEADER_PREFIX = "Content-Length:";    
  
    private final String WETFileID;
    private final File inFile;

    private int recordCount = 0;

    /** Handle to a CCWETProcessor that processes a set of WET files
     * Whereas a WETProcessor instance only processes a single WET file
     * containing multiple WET records.
     */
    private CCWETProcessor batchProcessor;
    
    /**
     * WET processor processes a single warc.wet file containing multiple WET records
     * containing text identified as primary langcode=mri. Each individual WET record is written
     * out to a uniquely named file in either the keep or discard folder depending on the WET
     * record's content length and number of lines of actual content (excluding WARC headers).
     */
    public WETProcessor(File inFile, CCWETProcessor batchProcessor) {
    this.batchProcessor = batchProcessor;
    
    this.inFile = inFile;
    // We just want a unique recordID prefix, which we get from the wet file name suffix:
    // inFile name looks something like MAORI-CC-2019-30-20190902100139-000000.warc.wet
    // the prefix will be everything after the last hyphen and without file extension,
    // so "000000" in our example. Then suffix the recordCount (keeping track of the current
    // WET record) to get a unique filename to store each WET record into.

    String fileID = inFile.getName();
    fileID = fileID.substring(fileID.lastIndexOf("-")+1);
    fileID = fileID.substring(0, fileID.indexOf("."));
    this.WETFileID = fileID;
    }

    public int processWETFile() {
    File keepURLsFile = this.batchProcessor.keepURLsFile;
    File discardURLsFile = this.batchProcessor.discardURLsFile; 
    
    StringBuilder record = null;
    String line = null; 
    boolean readingRecord = false;

    String WARCtargetURI = "";
    
    //int recordCount = 0;
    
    int contentLength = -1; // of record
    int lineCount = -1; // actual number of non-empty lines in record body (i.e. excludes WET/WARC headers)
        
    // read from WETfile
    try (
         BufferedReader reader = new BufferedReader(new FileReader(this.inFile));
         BufferedWriter keepURLsWriter = new BufferedWriter(new FileWriter(keepURLsFile, true));
         BufferedWriter discardURLsWriter = new BufferedWriter(new FileWriter(discardURLsFile, true)); // true to append
         ) {
        
        while((line = reader.readLine()) != null) { // readLine removes newline separator

        if(line.startsWith(WARC_INFO_HEADER)) { // "WARC-Type: warcinfo"
            readingRecord = false;
            record = null; // drop this record, which is just an info record not actual web page's text
            recordCount--;
            continue;
        }
        
        if(line.startsWith(WARC_RECORD_START)) { // "WARC/1.0" means finished previous WET record
            // process any previous record
            if(record != null) {
            processWETrecord(keepURLsWriter, discardURLsWriter,
                     recordCount, contentLength, lineCount,
                     WARCtargetURI, record.toString());
            record = null;
            contentLength = -1;
            lineCount = -1;
            }
            
            recordCount++;          
            // get ready to start a new record
            readingRecord = true;
            record = new StringBuilder();
        }

        if(readingRecord) { // append current line to current record

            if(line.startsWith(WARC_TARGET_URI_HEADER_PREFIX)) { // "WARC-Target-URI:"
            // get and store the value
            WARCtargetURI = line.substring(WARC_TARGET_URI_HEADER_PREFIX.length()).trim();
            }
            
            record.append(line + "\n"); // add back (unix style) line ending

            // if the line is non-empty
            // AND if we've started counting lines, which happens only when the current
            // line is past WARC/WET headers and we're into the actual body portion
            // of the WET record,
            // start incrementing the line counter.
            if(lineCount >= 0 && !line.trim().equals("")) {
            lineCount++;
            }
            else if(line.startsWith(WARC_CONTENT_LENGTH_HEADER_PREFIX)) { // "Content-Length:"
            String contentLengthStr = line.substring(WARC_CONTENT_LENGTH_HEADER_PREFIX.length()).trim();
            contentLength = Integer.parseInt(contentLengthStr);
            lineCount = 0;
            }
            
        }
        
        }

        // flush the last record. If it was a warcinfo record, record would be null here
        if(record != null) {
        processWETrecord(keepURLsWriter, discardURLsWriter,
                 recordCount, contentLength, lineCount,
                 WARCtargetURI, record.toString());
        record = null;
        }
        
    } catch(IOException ioe) {
        ioe.printStackTrace();
    }

    return recordCount;
    }

    public int getRecordCount() { return this.recordCount; }

    /**
     * Determines if a WET record belongs in the keep or discard pile depending on if it
     * contains enough text, based on contentLength and line count of the record body.
     * Then writes out the WET record to a uniquely named file in the keep or discard folder,
     * and writes out the WET record's URL to the keepURLs.txt file or discardURLs.txt file.
     */
    private void processWETrecord(BufferedWriter keepURLsWriter, BufferedWriter discardURLsWriter,
                  int recordID, int contentLength, int lineCount,
                  String recordURI, String record)
    {
    System.err.println("WET #" + this.WETFileID + " record #" + recordID
               + " - contentLength: " + contentLength
               + " - lineCount: " + lineCount);
    System.err.println("URI: " + recordURI);
    //System.err.println(record);
    //System.err.println("--------------------------");

    String paddedFileName = String.format("%04d.txt", recordID);
    
    File parentFolder = null;

    // want to match "product(s)" but not "production"
    //if(recordURI.matches(".*/?product[^a-rt-z].*")) {//if(recordURI.matches(".*/?products?/?.*")) {
    

    /*
    if(recordURI.contains("product") && !recordURI.contains("production")) {

        // don't want a "translated" product site/online store
        // These curiously often tend to have "product(s)" in the URL
        parentFolder = batchProcessor.discardFolder;
    }

    else if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
        parentFolder = batchProcessor.keepFolder;
        System.err.println("@@@KEEPING");
    } else if(contentLength >= MIN_CONTENT_LENGTH_WRAPPED_LINE) {
        int countSpaces = 0;
        for(int i = 0; i < record.length(); i++) {
        if(record.charAt(i) == ' ') countSpaces++;
        }
        if(countSpaces >= MIN_SPACES_IN_A_WRAPPED_LINE) {
        // So we have at least 500 chars (possibly on a single wrapped line)
        // containing at least 10 spaces. Such a record is also worth keeping.
        parentFolder = batchProcessor.keepFolder;
        }
    }
    */

    if(batchProcessor.isBlacklisted(recordURI)) {
        parentFolder = batchProcessor.discardFolder;
    }
    else if(batchProcessor.isGreylisted(recordURI)) { // e.g. products sites
        parentFolder = batchProcessor.discardFolder; // TODO: checkfolder
    } else {
        // If a web page's WET record contains a certain minimum number of words,
        // we will think it's a meaningful web page and has sufficient content for text analysis
        // to have been successful. Cut off values at present are:
        // - a minimum of 20 words
        // - a word consists of 1 to 15 chars demarcated by spaces. Any more chars may point
        // to words having been glued together. This is used by irrelevant sites and moreover
        // can't be analysed for language, so may not be actually MRI.
        
        // Though StringTokenizer still in use, as seen in discussion at
        // https://stackoverflow.com/questions/6983856/why-is-stringtokenizer-deprecated
        // String.split(regex) seems better for splitting on general whitespace
        String[] allWords = record.split("\\s");
        int validWordCount = 0;
        int numCamelCaseWords = 0;
        for(int i = 0; i < allWords.length; i++) {
        String word = allWords[i];

        // throw away if n words contain camelcase, which is another case of words glued together
        if(word.matches(".*[a-z][A-Z].*") && word.length() >= 5) {
            numCamelCaseWords++;            
        }
        
        // In Maori, word length of 1 is not uncommon
        // but let's skip camelcased words when counting valid words
        else if(word.length() >= 1 && word.length() <= batchProcessor.MAX_WORD_LENGTH) validWordCount++;
        }

        // dump if too many camelcase words (ideally keep none of that kind?)
        if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
        parentFolder = batchProcessor.discardFolder;
        System.err.println("@@@DISCARDING - CAMELCASED CONTENTS");
        }
        else if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
        parentFolder = batchProcessor.keepFolder;
        System.err.println("@@@KEEPING");
        }
    }
    // if parentFolder still not set, set to discard pile folder
    if(parentFolder == null) {
        parentFolder = batchProcessor.discardFolder;
        System.err.println("@@@DISCARDING");
    }

    try {
        if (parentFolder == batchProcessor.keepFolder) {
        keepURLsWriter.write(recordURI + "\n");
        } else {
        discardURLsWriter.write(recordURI + "\n");
        }
    } catch(Exception e) {
        System.err.println("Unable to write URL");
        e.printStackTrace();
    }
   
    System.err.println("--------------------------");
    
    File outFile = new File(parentFolder, this.WETFileID + "-" + paddedFileName);
    
    try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
        writer.write(record);
        // Try-with-resources examples don't call close() explicitly:
        // https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
        //writer.close();
    } catch(IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
    }
    }
}