source: gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java@ 33466

package org.greenstone.atea;


import java.io.*;
import java.util.Properties;
import java.util.zip.GZIPInputStream;
import org.apache.log4j.Logger;

/**
 * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
 * the WET records in each, putting each WET record into a file. Each file is put into a
 * keep or discard folder, based on content-length and number of lines.
 * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
 *
 * To compile, including the jars in lib/ for compiling.
 *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/WETProcessor.java 
 *
 * To run, passing the log4j and other properties files in conf/ folder:
 *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor <folder containing warc.wet(.gz) files> <outputFolder>
 *
 * e.g.
 *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
 *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
 *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
 *
*/
public class WETProcessor {
    //private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());
    private Properties configProperties = null;

    
    // arbitrary cut-off values, TODO: put in .properties file
    private static final int MIN_CONTENT_LENGTH = 100;
    private static final int MIN_LINE_COUNT = 2;
    private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = 500;
    private static final int MIN_SPACES_IN_A_WRAPPED_LINE = 10;    

    // keep a count of all the records that all WETProcessors instantiated
    // by our main method have processed
    private static int recordCount = 0;

    private final File outputFolder;
    private final String WETFileID;


    public static int getRecordCount() { return recordCount; }

    public WETProcessor(File inFile, File outFolder, Properties configProps) {
    this.outputFolder = outFolder;
    this.configProperties = configProps;
    
    StringBuilder record = null;
    String line = null; 
    boolean readingRecord = false;

    //this.recordCount = 0;
    
    int contentLength = -1; // of record
    int lineCount = -1; // actual number of non-empty lines in record body (i.e. excludes WET/WARC headers)

    // We just want a unique recordID prefix, which we get from the wet file name suffix:
    // inFile name looks something like MAORI-CC-2019-30-20190902100139-000000.warc.wet
    // the prefix will be everything after the last hyphen and without file extension,
    // so "000000" in our example.

    String fileID = inFile.getName();
    fileID = fileID.substring(fileID.lastIndexOf("-")+1);
    fileID = fileID.substring(0, fileID.indexOf("."));
    this.WETFileID = fileID;
    
    // read from WETfile
    try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
        
        while((line = reader.readLine()) != null) { // readLine removes newline separator

        if(line.startsWith("WARC-Type: warcinfo")) {
            readingRecord = false;
            record = null; // drop this record, which is just an info record not actual web page's text
            recordCount--;
            continue;
        }
        
        if(line.startsWith("WARC/1.0")) { // finished previous WET record
            // process any previous record
            if(record != null) {
            processWETrecord(recordCount, contentLength, lineCount, record.toString());
            record = null;
            contentLength = -1;
            lineCount = -1;
            }
            
            recordCount++;          
            // get ready to start a new record
            readingRecord = true;
            record = new StringBuilder();
        }

        if(readingRecord) { // append current line to current record

            record.append(line + "\n"); // add back (unix style) line ending

            // if the line is non-empty
            // AND if we've started counting lines, which happens only when the current
            // line is past WARC/WET headers and we're into the actual body portion
            // of the WET record,
            // start incrementing the line counter.
            if(lineCount >= 0 && !line.trim().equals("")) {
            lineCount++;
            }
            else if(line.startsWith("Content-Length:")) {
            String contentLengthStr = line.substring("Content-Length:".length()).trim();
            contentLength = Integer.parseInt(contentLengthStr);
            lineCount = 0;
            }
            
        }
        
        }

        // flush the last record. If it was a warcinfo record, record would be null here
        if(record != null) {
        processWETrecord(recordCount, contentLength, lineCount, record.toString());
        record = null;
        }
        
    } catch(IOException ioe) {
        ioe.printStackTrace();
    }
    }

    
    private void processWETrecord(int recordID, int contentLength, int lineCount, String record)
    {
    System.err.println("WET #" + this.WETFileID + " record #" + recordID
               + " - contentLength: " + contentLength
               + " - lineCount: " + lineCount);
    //System.err.println(record);
    //System.err.println("--------------------------");

    String paddedFileName = String.format("%04d.txt", recordID);
    
    File discardFolder = new File(this.outputFolder, "discard");        
    File keepFolder = new File(this.outputFolder, "keep");
    File parentFolder = null;
    
    if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
        parentFolder = keepFolder;
        System.err.println("@@@KEEPING");
    } else if(contentLength >= MIN_CONTENT_LENGTH_WRAPPED_LINE) {
        int countSpaces = 0;
        for(int i = 0; i < record.length(); i++) {
        if(record.charAt(i) == ' ') countSpaces++;
        }
        if(countSpaces >= MIN_SPACES_IN_A_WRAPPED_LINE) {
        // So we have at least 500 chars (possibly on a single wrapped line)
        // containing at least 10 spaces. Such a record is also worth keeping.
        parentFolder = keepFolder;
        }
    }

    // if parentFolder still not set, set to discard pile folder
    if(parentFolder == null) {
        parentFolder = discardFolder;
        System.err.println("@@@DISCARDING");
    }

    System.err.println("--------------------------");
    
    File outFile = new File(parentFolder, this.WETFileID + "-" + paddedFileName);
    
    try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
        writer.write(record);
        writer.close();
    } catch(IOException ioe) {
        ioe.printStackTrace();
        System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
    }
    }
    
    
    public static void printUsage() {
    System.err.println("Run this program as:");
    System.err.println("\tWetProcessor <folder containing wet(.gz) files> <output folder path>");   
    }

    /** Filename filter that only lists warc.wet files or else warc.wet.gz files
     * for which unzipped warc.wet equivalents don't yet exist.
     */
    private static class WETFilenameFilter implements FilenameFilter {
    
    public boolean accept(File dir, String name) {
        if(name.endsWith(".warc.wet")) {
        System.err.println("Will include " + name + " for processing.");
        return true;
        }

        if(name.endsWith(".warc.wet.gz")) {
        String nameWithoutGZext = name.substring(0, name.lastIndexOf(".gz"));
        File unzippedVersion = new File(dir, nameWithoutGZext);
        if(unzippedVersion.exists()) {
            System.err.println("--- Unzipped version " + unzippedVersion + " exists.");
            System.err.println("Skipping " + name);
            return false; // don't count gzipped version if unzipped version exists.
        }
        else {
            System.err.println("Only zipped version " + name + " exists.");
            return true; // No unzipped version, so have to work with gzipped version
        }
        }
        
        System.err.println("Skipping " + name);

        // we're not even interested in any other file extensions
        return false;
    }
    }
    
    public static void main(String[] args) {
    if(args.length != 2) {
        printUsage();
        return;
    }

    
    File WETFileDir = new File(args[0]);
    if(!WETFileDir.exists() || !WETFileDir.isDirectory()) {
        System.err.println("Error: " + args[0] + " does not exist or is not a directory");
        return;
    }
    
    File outFolder = new File(args[1]);
    if(!outFolder.exists() || !outFolder.isDirectory()) {
        System.err.println("Error: " + args[1] + " does not exist or is not a directory.");
        return;
    }   
    
    File discardFolder = new File(outFolder, "discard");
    if(!discardFolder.exists()) {
        discardFolder.mkdir();
    }   
    File keepFolder = new File(outFolder, "keep");
    if(!keepFolder.exists()) {
        keepFolder.mkdir();
    }

    // load up the properties from the config file
    Properties configProps;
    try (InputStream infile = Class.forName("org.greenstone.atea.WETProcessor").getClassLoader().getResourceAsStream("config.properties")) {
        configProps = new Properties();
        configProps.load(infile);
        infile.close();
        
    } catch(Exception e) {
        System.err.println("Exception attempting to read properties from config.properties.");
        e.printStackTrace();
        return;
    }

    // Will list all the warc.wet files in the input directory or else their gzipped versions
    File[] WETFiles = WETFileDir.listFiles(new WETFilenameFilter());

    for(int i = 0; i < WETFiles.length; i++) {
        File WETFile = WETFiles[i];     
        System.err.println("Processing WETfile: " + WETFile);

        // Any .gz files listed means they haven't been unzipped yet. So unzip.
        String WETFilename = WETFile.toString();
        if(WETFilename.endsWith(".gz")) {
        File GZippedWETFile = WETFile;
        String WETGZippedFilename = WETFilename;
        WETFilename = WETFilename.substring(0, WETFilename.lastIndexOf(".gz"));     

        WETFile = new File(WETFilename);
        Utility.unzipFile(GZippedWETFile, WETFile);
        }
        // hereafter all WETFiles should refer to the unzipped version
        // Check the unzipped WETFile exists        

        if(!WETFile.exists() || !WETFile.isFile()) {
        System.err.println("Error: " + WETFile + " does not exist (failure to unzip?)");
        return;
        }

        // Finally, we can process this WETFile's records into the keep and discard pile
        System.err.println("Off to process " + WETFile);
        WETProcessor processor = new WETProcessor(WETFile, outFolder, configProps);     
        
    }
    
    return; 
    }
}