Changeset 33466 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

Timestamp:

2019-09-12T21:37:39+12:00 (5 years ago)

Author:

ak19

Message:

1. WETProcessor.main() now processes a folder of *.warc.wet(.gz) files. Each file's WET records is written out into an individual file and put into either the keep folder or discard folder, based on amount of content (number lines and/or content-length). 2. Moved unzipFile() from NZTLDProcessor.java into new Utility.java class as a static method.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java (modified) (7 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -1 @@
-
-
 package org.greenstone.atea;
 
 
 import java.io.*;
+import java.util.Properties;
+import java.util.zip.GZIPInputStream;
+import org.apache.log4j.Logger;
 
 /**
- * To compile:
- *      maori-lang-detection/src>javac -cp "." org/greenstone/atea/WETProcessor.java
- * To run:
- *      java org.greenstone.atea.WETProcessor <wetfile>.wet.warc <outputFolder>
+ * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
+ * the WET records in each, putting each WET record into a file. Each file is put into a
+ * keep or discard folder, based on content-length and number of lines.
+ * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
+ *
+ * To compile, including the jars in lib/ for compiling.
+ *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/WETProcessor.java 
+ *
+ * To run, passing the log4j and other properties files in conf/ folder:
+ *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor &lt;folder containing warc.wet(.gz) files&gt; &lt;outputFolder&gt;
+ *
  * e.g.
- *    - java org.greenstone.atea.WETProcessor ../wetprocessor/MAORI-CC-2019-30-20190902100139-000000.warc.wet /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
- *    - java org.greenstone.atea.WETProcessor ../wetprocessor/MAORI-CC-2019-30-20190902100139-000000.warc.wet /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
+ *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
+ *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
+ *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2&gt;&amp;1 | less
+ *
 */
 public class WETProcessor {
+    //private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());
+    private Properties configProperties = null;
+
+    
     // arbitrary cut-off values, TODO: put in .properties file
     private static final int MIN_CONTENT_LENGTH = 100;
@@ -21 +35 @@
     private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = 500;
     private static final int MIN_SPACES_IN_A_WRAPPED_LINE = 10;    
-    
+
+    // keep a count of all the records that all WETProcessors instantiated
+    // by our main method have processed
+    private static int recordCount = 0;
+
     private final File outputFolder;
-    private final String WETFileID;    
-    private int recordCount = -1;
-    
-    /*
-    public WETProcessor(File inFile) {
-
+    private final String WETFileID;
+
+
+    public static int getRecordCount() { return recordCount; }
+
+    public WETProcessor(File inFile, File outFolder, Properties configProps) {
+    this.outputFolder = outFolder;
+    this.configProperties = configProps;
+    
     StringBuilder record = null;
     String line = null; 
     boolean readingRecord = false;
-    
-    // read from WETfile
-    try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
-        while((line = reader.readLine()) != null) { // readLine removes newline separator
-
-        if(line.startsWith("Content-Length:")) {
-            readingRecord = true;
-            record = new StringBuilder();
-            continue;
-        }
-
-        if(readingRecord) {
-            if(line.startsWith("WARC/1.0")) { // finished previous WET record
-            readingRecord = false;
-            
-            System.err.println("WET record:");
-            System.err.println(record);
-            System.err.println("--------------------------");
-            record = null;
-
-            processWETrecord(record);
-            }
-            else {
-            record.append(line + "\n"); // add back (unix style) line ending
-            }
-        }
-        }
-    } catch(IOException ioe) {
-        ioe.printStackTrace();
-    }
-    }
-
-    public WETProcessor(File inFile, File outFolder) {
-    this.outputFolder = outFolder;
-    
-    StringBuilder record = null;
-    String line = null; 
-    boolean readingRecord = false;
-
-    int recordCount = 0;
-    
-    // read from WETfile
-    try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
-        
-        while((line = reader.readLine()) != null) { // readLine removes newline separator
-
-        if(line.startsWith("WARC-Type: warcinfo")) {
-            readingRecord = false;
-            record = null; // drop this record, which is just an info record not actual web page's text
-            recordCount--;
-            continue;
-        }
-        
-        if(line.startsWith("WARC/1.0")) { // finished previous WET record
-            // process any previous record
-            if(record != null) {
-            processWETrecord(record, recordCount);
-            record = null;
-            }
-            
-            recordCount++;          
-            // get ready to start a new record
-            readingRecord = true;
-            record = new StringBuilder();
-        }
-
-        if(readingRecord) { // append current line to current record
-            record.append(line + "\n"); // add back (unix style) line ending
-        }
-        
-        }
-
-        // flush the last record. If it was a warcinfo record, record would be null here
-        if(record != null) {
-        processWETrecord(record, recordCount);
-        record = null;
-        }
-        
-    } catch(IOException ioe) {
-        ioe.printStackTrace();
-    }
-    }
-    */
-
-    public WETProcessor(File inFile, File outFolder) {
-    this.outputFolder = outFolder;
-    
-    StringBuilder record = null;
-    String line = null; 
-    boolean readingRecord = false;
-
-    this.recordCount = 0;
+
+    //this.recordCount = 0;
+    
     int contentLength = -1; // of record
     int lineCount = -1; // actual number of non-empty lines in record body (i.e. excludes WET/WARC headers)
@@ -197 +129 @@
     }
 
-    /*
-    private void processWETrecord(StringBuilder str, int recordID) {
-    String record = str.toString();
-    //System.err.println("WET record:");
-    //System.err.println(record);
-    //System.err.println("--------------------------");
-
-    String paddedFileName = String.format("%04d.txt", recordID);
-    File outFile = new File(this.outputFolder, paddedFileName);
-    try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
-        writer.write(record);
-        writer.close();
-    } catch(IOException ioe) {
-        ioe.printStackTrace();
-        System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
-    }
-    }
-    */
-
-    public int getRecordCount() { return this.recordCount; }
-    
-    private void processWETrecord(int recordID, int contentLength, int lineCount, String record) {
-    System.err.println("WET record - contentLength: " + contentLength + " - lineCount: " + lineCount);
+    
+    private void processWETrecord(int recordID, int contentLength, int lineCount, String record)
+    {
+    System.err.println("WET #" + this.WETFileID + " record #" + recordID
+               + " - contentLength: " + contentLength
+               + " - lineCount: " + lineCount);
     //System.err.println(record);
     //System.err.println("--------------------------");
@@ -266 +181 @@
     public static void printUsage() {
     System.err.println("Run this program as:");
-    System.err.println("\tWetProcessor <infile>.warc.wet <output folder path>");    
+    System.err.println("\tWetProcessor <folder containing wet(.gz) files> <output folder path>");   
+    }
+
+    /** Filename filter that only lists warc.wet files or else warc.wet.gz files
+     * for which unzipped warc.wet equivalents don't yet exist.
+     */
+    private static class WETFilenameFilter implements FilenameFilter {
+    
+    public boolean accept(File dir, String name) {
+        if(name.endsWith(".warc.wet")) {
+        System.err.println("Will include " + name + " for processing.");
+        return true;
+        }
+
+        if(name.endsWith(".warc.wet.gz")) {
+        String nameWithoutGZext = name.substring(0, name.lastIndexOf(".gz"));
+        File unzippedVersion = new File(dir, nameWithoutGZext);
+        if(unzippedVersion.exists()) {
+            System.err.println("--- Unzipped version " + unzippedVersion + " exists.");
+            System.err.println("Skipping " + name);
+            return false; // don't count gzipped version if unzipped version exists.
+        }
+        else {
+            System.err.println("Only zipped version " + name + " exists.");
+            return true; // No unzipped version, so have to work with gzipped version
+        }
+        }
+        
+        System.err.println("Skipping " + name);
+
+        // we're not even interested in any other file extensions
+        return false;
+    }
     }
     
@@ -275 +222 @@
     }
 
-    File WETfile = new File(args[0]);
-    if(!WETfile.exists() || !WETfile.isFile()) {
-        System.err.println("Error: " + args[0] + " does not exist or is not a file");
+    
+    File WETFileDir = new File(args[0]);
+    if(!WETFileDir.exists() || !WETFileDir.isDirectory()) {
+        System.err.println("Error: " + args[0] + " does not exist or is not a directory");
         return;
     }
@@ -285 +233 @@
         System.err.println("Error: " + args[1] + " does not exist or is not a directory.");
         return;
-    }
-
-    
+    }   
     
     File discardFolder = new File(outFolder, "discard");
@@ -296 +242 @@
     if(!keepFolder.exists()) {
         keepFolder.mkdir();
-    }   
-    
-    WETProcessor processor = new WETProcessor(WETfile, outFolder);
-    return;
-    
+    }
+
+    // load up the properties from the config file
+    Properties configProps;
+    try (InputStream infile = Class.forName("org.greenstone.atea.WETProcessor").getClassLoader().getResourceAsStream("config.properties")) {
+        configProps = new Properties();
+        configProps.load(infile);
+        infile.close();
+        
+    } catch(Exception e) {
+        System.err.println("Exception attempting to read properties from config.properties.");
+        e.printStackTrace();
+        return;
+    }
+
+    // Will list all the warc.wet files in the input directory or else their gzipped versions
+    File[] WETFiles = WETFileDir.listFiles(new WETFilenameFilter());
+
+    for(int i = 0; i < WETFiles.length; i++) {
+        File WETFile = WETFiles[i];     
+        System.err.println("Processing WETfile: " + WETFile);
+
+        // Any .gz files listed means they haven't been unzipped yet. So unzip.
+        String WETFilename = WETFile.toString();
+        if(WETFilename.endsWith(".gz")) {
+        File GZippedWETFile = WETFile;
+        String WETGZippedFilename = WETFilename;
+        WETFilename = WETFilename.substring(0, WETFilename.lastIndexOf(".gz"));     
+
+        WETFile = new File(WETFilename);
+        Utility.unzipFile(GZippedWETFile, WETFile);
+        }
+        // hereafter all WETFiles should refer to the unzipped version
+        // Check the unzipped WETFile exists        
+
+        if(!WETFile.exists() || !WETFile.isFile()) {
+        System.err.println("Error: " + WETFile + " does not exist (failure to unzip?)");
+        return;
+        }
+
+        // Finally, we can process this WETFile's records into the keep and discard pile
+        System.err.println("Off to process " + WETFile);
+        WETProcessor processor = new WETProcessor(WETFile, outFolder, configProps);     
+        
+    }
+    
+    return; 
     }
 }