Changeset 33503 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

Timestamp:

2019-09-23T23:16:28+12:00 (5 years ago)

Author:

ak19

Message:

More efficient blacklisting/greylisting/whitelisting now by reading in the lists only once and then comparing each URL to each list. Explicit whitelisting has precedence over greylisting and which takes precedence over blacklisting. Then any remaining urls are checked for having sufficient content. The code that checks for sufficient content still needs some more adjusting.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -12 +12 @@
 
 /**
- * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
- * the WET records in each, putting each WET record into a file. Each file is put into a
- * keep or discard folder, based on content-length and number of lines.
- * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
+ * A single instance of the WETProcessor class can process a single unzipped warc.wet file.
+ * A WETProcessor take a warc.wet file and goes through all its WET records,
+ * putting each WET record into a file. Each file is put into a keep, discard or greylisted folder
+ * and its url listed written into a keep, discard or greylisted text file, based on:
  *
- * To compile, including the jars in lib/ for compiling.
- *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/WETProcessor.java 
+ * 1. whether it's whitelisted, else greylisted else blacklisted
+ * 2. and if explicitly whitelisted or else not greylisted or blacklisted and there's
+ * enough content. Formerly, content-length and number of lines were used to determine if
+ * the content was sufficient. Now it's just word count and number of MAX characters
+ * (not MINIMUM characters) that determine a string is a word.
+ * Explicit whitelisting has precedence over greylisting and which takes precedence
+ * over blacklisting in turn.
+ * However, even explicitly whitelisted urls still need to have sufficient content to end 
+ * up in keepURLs.txt.
  *
- * To run, passing the log4j and other properties files in conf/ folder:
- *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor <folder containing warc.wet(.gz) files> <outputFolder>
- *
- * e.g.
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
+ * See CCWETProcessor.java for compile instructions and how to run.
  *
 */
@@ -72 +74 @@
     public int processWETFile() {
     File keepURLsFile = this.batchProcessor.keepURLsFile;
-    File discardURLsFile = this.batchProcessor.discardURLsFile; 
+    File discardURLsFile = this.batchProcessor.discardURLsFile;
+    File greyListedFile = this.batchProcessor.greyListedFile;
     
     StringBuilder record = null;
@@ -90 +93 @@
          BufferedWriter keepURLsWriter = new BufferedWriter(new FileWriter(keepURLsFile, true));
          BufferedWriter discardURLsWriter = new BufferedWriter(new FileWriter(discardURLsFile, true)); // true to append
+         BufferedWriter greyListedURLsWriter = new BufferedWriter(new FileWriter(greyListedFile, true)); // true to append
          ) {
         
@@ -104 +108 @@
             // process any previous record
             if(record != null) {
-            processWETrecord(keepURLsWriter, discardURLsWriter,
+            processWETrecord(keepURLsWriter, discardURLsWriter, greyListedURLsWriter,
                      recordCount, contentLength, lineCount,
                      WARCtargetURI, record.toString());
@@ -147 +151 @@
         // flush the last record. If it was a warcinfo record, record would be null here
         if(record != null) {
-        processWETrecord(keepURLsWriter, discardURLsWriter,
+        processWETrecord(keepURLsWriter, discardURLsWriter, greyListedURLsWriter,
                  recordCount, contentLength, lineCount,
                  WARCtargetURI, record.toString());
@@ -169 +173 @@
      */
     private void processWETrecord(BufferedWriter keepURLsWriter, BufferedWriter discardURLsWriter,
+                  BufferedWriter greyListedURLsWriter,
                   int recordID, int contentLength, int lineCount,
                   String recordURI, String record)
@@ -210 +215 @@
     }
     */
-
+    
     if(batchProcessor.isBlacklisted(recordURI)) {
-        parentFolder = batchProcessor.discardFolder;
+
+        
+        // explicit whitelisting overrides blacklisting
+        if(batchProcessor.isWhitelisted(recordURI)) {
+        parentFolder = batchProcessor.keepFolder; //tentative
+        }
+        // if not whitelisted, then greylisting overrides blacklisting
+        else if(batchProcessor.isGreylisted(recordURI)) {
+        parentFolder = batchProcessor.greyListedFolder;
+        System.err.println("@@@GREYLISTED");
+        }
+        else { // only blacklisted
+        parentFolder = batchProcessor.discardFolder;
+        System.err.println("@@@DISCARDING - blacklisted");
+        }
     }
     else if(batchProcessor.isGreylisted(recordURI)) { // e.g. products sites
-        parentFolder = batchProcessor.discardFolder; // TODO: checkfolder
-    } else {
+        // explicit whitelisting overrides greylisting
+        if(batchProcessor.isWhitelisted(recordURI)) {
+        parentFolder = batchProcessor.keepFolder; // tentative
+        }
+        else {
+        parentFolder = batchProcessor.greyListedFolder;
+        System.err.println("@@@GREYLISTED");
+        }
+    }
+
+    // If URL was not blacklisted/greylisted, or was even explicitly whitelisted,
+    // it still can't be in the keep list as it needs further inspection:
+    // it needs sufficient content for language analysis.
+    if(parentFolder != batchProcessor.greyListedFolder && parentFolder != batchProcessor.discardFolder) { // i.e. parentFolder == keepFolder if whiteListed || parentFolder == null
+        
         // If a web page's WET record contains a certain minimum number of words,
         // we will think it's a meaningful web page and has sufficient content for text analysis
@@ -254 +286 @@
         }
     }
-    // if parentFolder still not set, set to discard pile folder
+    // if parentFolder still not set, it means that the content length/num words or lines
+    // were insufficient, so meant to be discarded
     if(parentFolder == null) {
         parentFolder = batchProcessor.discardFolder;
@@ -263 +296 @@
         if (parentFolder == batchProcessor.keepFolder) {
         keepURLsWriter.write(recordURI + "\n");
+        } else if (parentFolder == batchProcessor.greyListedFolder) {
+        greyListedURLsWriter.write(recordURI + "\n");
         } else {
         discardURLsWriter.write(recordURI + "\n");