Changeset 33467

Timestamp:

2019-09-13T17:44:41+12:00 (5 years ago)

Author:

ak19

Message:

Improved the code to use a static block to load the needed properties from config.properties and initialise some static final ints from there. Code now uses the logger for debugging. New properties in config.properties. Returned code to use a counter, recordCount, re-zeroed for each WETProcessor since the count was used for unique filenames, and filename prefixes are unique for each warc.wet file. So these prefixes, in combination with keeping track of the recordcount per warc.wet file, each WET record written out to a file is assigned a unique filename. (No longer need a running total of all WET records across warc.wet files processed ensuring uniqueness of filenames.) All appears to still work similarly to previous commit in creating discard and keep subfolders.

Location:

gs3-extensions/maori-lang-detection

Files:

• MoreReading/CommonCrawl.txt (modified) (1 diff)
• MoreReading/Vagrant-Spark-Hadoop.txt (modified) (2 diffs)
• conf/config.properties (modified) (1 diff)
• src/org/greenstone/atea/Utility.java (modified) (2 diffs)
• src/org/greenstone/atea/WETProcessor.java (modified) (15 diffs)

gs3-extensions/maori-lang-detection/MoreReading/CommonCrawl.txt

@@ -47 +47 @@
 Sebastian
 
+====================
+wharariki:[239]/Scratch/ak19/gs3-extensions/maori-lang-detection/src>java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
+
+wharariki:[188]/Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET>ls keep | wc
+   4090    4090   65440
+wharariki:[189]/Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET>ls discard | wc
+   1515    1515   24240
+
+We keep 4090 WET records and are discarding 1515.
+
 =======================
 Latest version of the index's schema:

gs3-extensions/maori-lang-detection/MoreReading/Vagrant-Spark-Hadoop.txt

@@ -221 +221 @@
  
 
-
 vagrant@node1:~$ locate guava.jar
 /usr/share/java/guava.jar
@@ -243 +242 @@
 vagrant@node1:~/ia-hadoop-tools$ hdfs dfs -put /usr/share/java/guava.jar /usr/local/hadoop/share/hadoop/common/.
 put: `/usr/local/hadoop/share/hadoop/common/.': No such file or directory
-# hadoop classpath locations are not hdfs filesystem
+# hadoop classpath locations are not on the hdfs filesystem, but on the regular fs
 
 vagrant@node1:~/ia-hadoop-tools$ sudo cp /usr/share/java/guava.jar /usr/local/hadoop/share/hadoop/common/.

gs3-extensions/maori-lang-detection/conf/config.properties

@@ -9 +9 @@
 # for downloading a single file
 wget.file.cmd=wget %%FILE_URL%%
+
+# Arbitrary cutoff values for WETProcessor.java
+WETprocessor.min.content.length=100
+WETprocessor.min.line.count=2
+WETprocessor.min.content.length.wrapped.line=500
+WETprocessor.min.spaces.per.wrapped.line=10

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java

@@ -4 +4 @@
 import java.util.zip.GZIPInputStream;
 
+import org.apache.log4j.Logger;
+
 public class Utility {
+    private static Logger logger = Logger.getLogger(org.greenstone.atea.Utility.class.getName());
     
     // Run gunzip
@@ -26 +29 @@
         //out.close();
         
-        //log("Unzipped " + inZipFile + " to " + outFile);
+        logger.debug("Unzipped " + inZipFile + " to " + outFile);
         
     } catch(IOException ex) {

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -5 +5 @@
 import java.util.Properties;
 import java.util.zip.GZIPInputStream;
+
 import org.apache.log4j.Logger;
 
@@ -26 +27 @@
 */
 public class WETProcessor {
-    //private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());
-    private Properties configProperties = null;
-
+    private static Logger logger = Logger.getLogger(org.greenstone.atea.WETProcessor.class.getName());    
+    private static Properties configProperties = new Properties();
+
+    // In Java, can initialize static final variables inside a static block
+    // But the unavoidable try/catch in this static block prevents initialization of
+    // the static final int variables further below that therefore need to be declared
+    // and initialized thereafter.
+    static {    
+    
+    // load up the properties from the config file
+    try (InputStream infile = org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
+        configProperties = new Properties();
+        configProperties.load(infile);      
+        //infile.close();
+        
+    } catch(Exception e) {
+        System.err.println("Exception attempting to read properties from config.properties.");
+        logger.error("Exception attempting to read properties from config.properties.");
+        e.printStackTrace();
+    }
+    }
     
-    // arbitrary cut-off values, TODO: put in .properties file
-    private static final int MIN_CONTENT_LENGTH = 100;
-    private static final int MIN_LINE_COUNT = 2;
-    private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = 500;
-    private static final int MIN_SPACES_IN_A_WRAPPED_LINE = 10;    
-
-    // keep a count of all the records that all WETProcessors instantiated
-    // by our main method have processed
-    private static int recordCount = 0;
+    // Providing fall-back cuttoff values if config.properties doesn't load
+    // or doesn't have the named props. But what happens when Integer.parseInt throws an exception?
+    private static final int MIN_CONTENT_LENGTH = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length", "100"));
+    private static final int MIN_LINE_COUNT= Integer.parseInt(configProperties.getProperty("WETprocessor.min.line.count", "2"));
+    private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length.wrapped.line", "500"));
+    private static final int MIN_SPACES_IN_A_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.spaces.per.wrapped.line", "10"));
+    
+    // Keep a count of all the records that all WETProcessors instantiated
+    // by our main method combined have processed
+    //private static int recordCount = 0;
 
     private final File outputFolder;
@@ -44 +64 @@
 
 
-    public static int getRecordCount() { return recordCount; }
-
-    public WETProcessor(File inFile, File outFolder, Properties configProps) {
+    /**
+     * WET processor processes a single warc.wet file containing multiple WET records
+     * containing text identified as primary langcode=mri. Each individual WET record is written
+     * out to a uniquely named file in either the keep or discard folder depending on the WET
+     * record's content length and number of lines of actual content (excluding WARC headers).
+     */
+    public WETProcessor(File inFile, File outFolder) {
     this.outputFolder = outFolder;
-    this.configProperties = configProps;
     
     StringBuilder record = null;
     String line = null; 
     boolean readingRecord = false;
-
-    //this.recordCount = 0;
+    
+    int recordCount = 0;
     
     int contentLength = -1; // of record
@@ -62 +85 @@
     // inFile name looks something like MAORI-CC-2019-30-20190902100139-000000.warc.wet
     // the prefix will be everything after the last hyphen and without file extension,
-    // so "000000" in our example.
+    // so "000000" in our example. Then suffix the recordCount (keeping track of the current
+    // WET record) to get a unique filename to store each WET record into.
 
     String fileID = inFile.getName();
@@ -129 +153 @@
     }
 
-    
+    /**
+     * Determines if a WET record belongs in the keep or discard pile depending on if it
+     * contains enough text, based on contentLength and line count of the record body.
+     * Then writes out the WET record to a uniquely named file in the keep or discard folder.
+     */
     private void processWETrecord(int recordID, int contentLength, int lineCount, String record)
     {
@@ -171 +199 @@
     try (BufferedWriter writer = new BufferedWriter(new FileWriter(outFile))) {
         writer.write(record);
-        writer.close();
+        // Try-with-resources examples don't call close() explicitly:
+        // https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
+        //writer.close();
     } catch(IOException ioe) {
         ioe.printStackTrace();
@@ -178 +208 @@
     }
     
+
+    //public static int getRecordCount() { return recordCount; }
     
     public static void printUsage() {
@@ -184 +216 @@
     }
 
-    /** Filename filter that only lists warc.wet files or else warc.wet.gz files
+    /** Filename filter to only list warc.wet files or else warc.wet.gz files
      * for which unzipped warc.wet equivalents don't yet exist.
      */
@@ -191 +223 @@
     public boolean accept(File dir, String name) {
         if(name.endsWith(".warc.wet")) {
-        System.err.println("Will include " + name + " for processing.");
+        logger.debug("Will include " + name + " for processing.");
         return true;
         }
@@ -199 +231 @@
         File unzippedVersion = new File(dir, nameWithoutGZext);
         if(unzippedVersion.exists()) {
-            System.err.println("--- Unzipped version " + unzippedVersion + " exists.");
-            System.err.println("Skipping " + name);
+            logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
+            logger.debug("Skipping " + name);
             return false; // don't count gzipped version if unzipped version exists.
         }
         else {
-            System.err.println("Only zipped version " + name + " exists.");
+            logger.debug("Only zipped version " + name + " exists.");
             return true; // No unzipped version, so have to work with gzipped version
         }
         }
-        
-        System.err.println("Skipping " + name);
 
         // we're not even interested in any other file extensions
+        logger.debug("Not a WET file. Skipping " + name);
         return false;
     }
@@ -225 +256 @@
     File WETFileDir = new File(args[0]);
     if(!WETFileDir.exists() || !WETFileDir.isDirectory()) {
-        System.err.println("Error: " + args[0] + " does not exist or is not a directory");
+        System.out.println("Error: " + args[0] + " does not exist or is not a directory");
         return;
     }
@@ -231 +262 @@
     File outFolder = new File(args[1]);
     if(!outFolder.exists() || !outFolder.isDirectory()) {
-        System.err.println("Error: " + args[1] + " does not exist or is not a directory.");
+        System.out.println("Error: " + args[1] + " does not exist or is not a directory.");
         return;
     }   
@@ -244 +275 @@
     }
 
-    // load up the properties from the config file
-    Properties configProps;
-    try (InputStream infile = Class.forName("org.greenstone.atea.WETProcessor").getClassLoader().getResourceAsStream("config.properties")) {
-        configProps = new Properties();
-        configProps.load(infile);
-        infile.close();
-        
-    } catch(Exception e) {
-        System.err.println("Exception attempting to read properties from config.properties.");
-        e.printStackTrace();
-        return;
-    }
-
     // Will list all the warc.wet files in the input directory or else their gzipped versions
     File[] WETFiles = WETFileDir.listFiles(new WETFilenameFilter());
@@ -262 +280 @@
     for(int i = 0; i < WETFiles.length; i++) {
         File WETFile = WETFiles[i];     
-        System.err.println("Processing WETfile: " + WETFile);
+        logger.debug("Processing WETfile: " + WETFile);
 
         // Any .gz files listed means they haven't been unzipped yet. So unzip.
@@ -279 +297 @@
         if(!WETFile.exists() || !WETFile.isFile()) {
         System.err.println("Error: " + WETFile + " does not exist (failure to unzip?)");
+        logger.error("Error: " + WETFile + " does not exist (failure to unzip?)");
         return;
         }
 
         // Finally, we can process this WETFile's records into the keep and discard pile
-        System.err.println("Off to process " + WETFile);
-        WETProcessor processor = new WETProcessor(WETFile, outFolder, configProps);     
+        logger.debug("Off to process " + WETFile);
+        WETProcessor processor = new WETProcessor(WETFile, outFolder);      
         
     }