Changeset 33503 for gs3-extensions

Timestamp:

2019-09-23T23:16:28+12:00 (5 years ago)

Author:

ak19

Message:

More efficient blacklisting/greylisting/whitelisting now by reading in the lists only once and then comparing each URL to each list. Explicit whitelisting has precedence over greylisting and which takes precedence over blacklisting. Then any remaining urls are checked for having sufficient content. The code that checks for sufficient content still needs some more adjusting.

Location:

gs3-extensions/maori-lang-detection/src/org/greenstone/atea

Files:

• CCWETProcessor.java (modified) (10 diffs)
• WETProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -6 +6 @@
 import java.util.zip.GZIPInputStream;
 import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
@@ -14 +16 @@
  * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
  * the WET records in each, putting each WET record into a file. Each file is put into a
- * keep or discard folder, based on content-length and number of lines.
- * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
+ * keep or discard or greyListed folder, and its url listed written into a keep, discard
+ * or greylisted text file, based on based on
+ * 
+ * 1. whether it's whitelisted, else greylisted else blacklisted
+ * 2. and if explicitly whitelisted or else not greylisted or blacklisted and there's
+ * enough content. Formerly, content-length and number of lines were used to determine if
+ * the content was sufficient. Now it's just word count and number of MAX characters
+ * (not MINIMUM characters) that determine a string is a word. These settings can be adjusted
+ * in conf/config.properties.
+ *
+ * Put a url-blacklist-filter.txt and/or url-greylist-filter.txt and/or url-whitelist-filter.txt
+ * into the conf folder to control any url patterns that are explicitly included or excluded or
+ * set aside for inspecting later. These filter text files don't use regexes, instead their
+ * format is:
+ * - precede URL by ^ to blacklist urls that match the given prefix
+ * - succeed URL by $ to blacklist urls that match the given suffix
+ * - ^url$ will blacklist urls that match the given url completely
+ * - Without either ^ or $ symbol, urls containing the given url will get blacklisted
+ * 
+ * WETProcessor.java's current implementation is that explicit whitelisting has precedence
+ * over greylisting and which takes precedence over blacklisting in turn. However, even
+ * explicitly whitelisted urls still need to have sufficient content to end up in keepURLs.txt
+ * and in the seedURLs.txt file used for nutch, along with its domain in regex-urlfilter.txt
+ * also for nutch.
+ *
+ * A CCWETProcessor instance can be configured to process all the .warc.wet(.gz) files
+ * in the given input folder. Then use a single instance of the WETProcessor class to process
+ * each single unzipped warc.wet file.
  *
  * To compile, including the jars in lib/ for compiling.
@@ -44 +72 @@
     public final File discardFolder;
     public final File keepFolder;
+    public final File greyListedFolder;
     public final File keepURLsFile;
     public final File discardURLsFile;
+    public final File greyListedFile;
+
+    private final Integer LIST_ENTRY_CONTAINS = new Integer(0);
+    private final Integer LIST_ENTRY_STARTSWITH = new Integer(1);
+    private final Integer LIST_ENTRY_ENDSWITH = new Integer(2);
+    private final Integer LIST_ENTRY_MATCHES = new Integer(3);
+    
+    private HashMap<String, Integer> blackList;
+    private HashMap<String, Integer> greyList;
+    private HashMap<String, Integer> whiteList;
 
     // Keep a count of all the records that all WETProcessors instantiated
@@ -53 +92 @@
     private int wetFileCount = 0;
     
-    public CCWETProcessor(File inFolder, File outFolder) {
+    public CCWETProcessor(File inFolder, File outFolder) throws Exception {
     this.WETFilesDir = inFolder;
     this.outputFolder = outFolder;
 
     // load up the properties from the config file
-    try (InputStream infile = org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
+    try (InputStream infile = org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
         configProperties = new Properties();
         configProperties.load(infile);      
-        //infile.close();
+        //infile.close(); // not explicitly called in examples of try-with-resources
         
     } catch(Exception e) {
@@ -86 +125 @@
         keepFolder.mkdir();
     }
-
+    
+    this.greyListedFolder = new File(outFolder, "greylisted");
+    if(!greyListedFolder.exists()) {
+        greyListedFolder.mkdir();
+    }
+    
     this.keepURLsFile = new File(outFolder, "keepURLs.txt");    
     if(keepURLsFile.exists() && !keepURLsFile.delete()) {
-        System.err.println("Warning: Unable to delete " + this.keepURLsFile + ". Unable to proceed.");
-        //return;
+        throw new Exception("Warning: Unable to delete " + this.keepURLsFile + ". Unable to proceed.");
     }
     this.discardURLsFile = new File(outFolder, "discardURLs.txt");
     if(discardURLsFile.exists() && !discardURLsFile.delete()) {
-        System.err.println("Warning Unable to delete " + discardURLsFile + ". Unable to proceed.");
-        //return;
-    }
-    
+        throw new Exception ("Warning Unable to delete " + discardURLsFile + ". Unable to proceed.");
+    }
+    this.greyListedFile = new File(outFolder, "greyListed.txt");
+    if(greyListedFile.exists() && !greyListedFile.delete()) {
+        throw new Exception ("Warning Unable to delete " + greyListedFile + ". Unable to proceed.");
+    }
+
+    System.err.println("Loading blacklist.");
+    blackList = new HashMap<String, Integer>();
+    initURLFilterList(blackList, "url-blacklist-filter.txt");
+    System.err.println("Loading greylist.");
+    greyList = new HashMap<String, Integer>();
+    initURLFilterList(greyList, "url-greylist-filter.txt");
+    System.err.println("Loading whitelist.");
+    whiteList = new HashMap<String, Integer>();
+    initURLFilterList(whiteList, "url-whitelist-filter.txt");
+
+    //System.err.println("Prematurely terminating for testing purposes.");
+    //System.exit(-1);
     }
     
@@ -171 +229 @@
     }
 
-    /*
+    private boolean isListedInFilterList(Map<String, Integer> filterListMap, String url) {
+    Set<Map.Entry<String,Integer>> entries = filterListMap.entrySet();
+    Iterator<Map.Entry<String, Integer>> i = entries.iterator();
+    while(i.hasNext()) {
+        Map.Entry<String, Integer> entry = i.next();
+        String urlPattern = entry.getKey();
+        Integer matchRule = entry.getValue();
+
+        if(matchRule == LIST_ENTRY_CONTAINS && url.contains(urlPattern)) {
+        return true;
+        }
+        else if(matchRule == LIST_ENTRY_STARTSWITH && url.startsWith(urlPattern)) {
+        return true;
+        }
+        else if(matchRule == LIST_ENTRY_ENDSWITH && url.endsWith(urlPattern)) {
+        return true;
+        }
+        else if(matchRule == LIST_ENTRY_MATCHES && url.equals(urlPattern)) {
+        return true;
+        }
+        // else check the rest of the filter list against this url
+        // before returning false to be certain it's not been listed in the filter list
+    }
+    
+    return false;
+    }
+
+    /**
+     * Returns true if the url or pattern is found in the blacklist file.
+     * Note that if eventually the same url pattern is found in the greylist or whitelist too,
+     * it won't get blacklisted after all. But that's not implemented here.
+     */
     public boolean isBlacklisted(String url) {
-    return false;
-    }
-    */
-
+    return isListedInFilterList(blackList, url);
+    }
+    
+    /**
+     * Returns true if the url or pattern is explicitly mentioned in the greylist file.
+     * Will eventually take precedence over if the same URL pattern was mentioned in the blacklist.
+     * Will eventually be pre-empted into the whitelist if mentioned in the whitelist.
+     */
     public boolean isGreylisted(String url) {
-    // alexa top sites and auto-translated product sites
-    return false;
+    // TODO: alexa top sites and auto-translated product sites
+    return isListedInFilterList(greyList, url);
+    }
+
+    /**
+     * Returns true if the url or pattern is explicitly mentioned in the whitelist file
+     * Its mention in a whitelist moreover overrides any mention in the blacklist and greylist.
+     */
+    public boolean isWhitelisted(String url) {
+    return isListedInFilterList(whiteList, url);
     }
     
@@ -189 +290 @@
      * we don't have to remember to escape or double escape each filter to turn it into a regex.
      */
-    //public boolean isInDiscardFilter(String url) {
-
-    public boolean isBlacklisted(String url) {  
+    public void initURLFilterList(Map<String, Integer> list, String filterListFilename) {
+    
+    // if filterListFilename does not exist in the conf folder, just return
+    if(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResource(filterListFilename) == null) {
+        System.err.println(filterListFilename + " does not exist");
+        return;     
+    }
+
+    try (
+         BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream(filterListFilename), "UTF-8"));
+         ) {
+        String filter = null;
+        while((filter = reader.readLine()) != null) {
+        // skip comments and empty lines
+        filter = filter.trim();
+        if(filter.equals("") || filter.startsWith("#")) {
+            continue;
+        }
+        
+        if(filter.startsWith("^") && filter.endsWith("$")) {
+            filter = filter.substring(1, filter.length()-1);
+            list.put(filter, LIST_ENTRY_MATCHES);
+        }
+        else if(filter.startsWith("^")) {
+            filter = filter.substring(1);
+            list.put(filter, LIST_ENTRY_STARTSWITH);
+            System.err.println("Match filter startswith: " + filter);
+        }
+        else if(filter.endsWith("$")) {
+            filter = filter.substring(0, filter.length()-1);
+            list.put(filter, LIST_ENTRY_ENDSWITH);
+        }
+        else {
+            list.put(filter, LIST_ENTRY_CONTAINS);
+        }
+        //System.err.println("Got filter: " + filter);
+        }
+        
+    } catch (IOException ioe) {
+        ioe.printStackTrace();
+        System.err.println("\n@@@@@@@@@ Error reading into map from file " + filterListFilename);
+    }
+    
+    }
+    /*
+    public boolean isInDiscardFilter(String url) {
     String discardFilterFile = "url-discard-filter.txt"; // in conf folder
 
     try (
-         BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream(discardFilterFile), "UTF-8"));
+         BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.CCWETProcessor.class.getClassLoader().getResourceAsStream(discardFilterFile), "UTF-8"));
          ) {
         String filter = null;
@@ -227 +371 @@
 
     return false;
-    }
+    }*/
 
     /** Maintain a count of all WET files processed. */
@@ -293 +437 @@
     }   
 
+    try {
     CCWETProcessor ccWETFilesProcessor = new CCWETProcessor(WETFileDir, outFolder);
 
@@ -341 +486 @@
     File urlFilterFile = new File(outFolder, "regex-urlfilter.txt");
     ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile);
+    } catch(Exception e) {
+    // can get an exception when instantiating CCWETProcessor instance
+    e.printStackTrace();
+    System.err.println(e.getMessage());
+    }
     
     return; 

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -12 +12 @@
 
 /**
- * The main() method of this class takes a folder of warc.wet(.gz) files and goes through
- * the WET records in each, putting each WET record into a file. Each file is put into a
- * keep or discard folder, based on content-length and number of lines.
- * A single instance of the WETProcessor class processes a single unzipped warc.wet file.
+ * A single instance of the WETProcessor class can process a single unzipped warc.wet file.
+ * A WETProcessor take a warc.wet file and goes through all its WET records,
+ * putting each WET record into a file. Each file is put into a keep, discard or greylisted folder
+ * and its url listed written into a keep, discard or greylisted text file, based on:
  *
- * To compile, including the jars in lib/ for compiling.
- *      maori-lang-detection/src$ javac -cp ".:../lib/*" org/greenstone/atea/WETProcessor.java 
+ * 1. whether it's whitelisted, else greylisted else blacklisted
+ * 2. and if explicitly whitelisted or else not greylisted or blacklisted and there's
+ * enough content. Formerly, content-length and number of lines were used to determine if
+ * the content was sufficient. Now it's just word count and number of MAX characters
+ * (not MINIMUM characters) that determine a string is a word.
+ * Explicit whitelisting has precedence over greylisting and which takes precedence
+ * over blacklisting in turn.
+ * However, even explicitly whitelisted urls still need to have sufficient content to end 
+ * up in keepURLs.txt.
  *
- * To run, passing the log4j and other properties files in conf/ folder:
- *      maori-lang-detection/src$ java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor <folder containing warc.wet(.gz) files> <outputFolder>
- *
- * e.g.
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
- *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
+ * See CCWETProcessor.java for compile instructions and how to run.
  *
 */
@@ -72 +74 @@
     public int processWETFile() {
     File keepURLsFile = this.batchProcessor.keepURLsFile;
-    File discardURLsFile = this.batchProcessor.discardURLsFile; 
+    File discardURLsFile = this.batchProcessor.discardURLsFile;
+    File greyListedFile = this.batchProcessor.greyListedFile;
     
     StringBuilder record = null;
@@ -90 +93 @@
          BufferedWriter keepURLsWriter = new BufferedWriter(new FileWriter(keepURLsFile, true));
          BufferedWriter discardURLsWriter = new BufferedWriter(new FileWriter(discardURLsFile, true)); // true to append
+         BufferedWriter greyListedURLsWriter = new BufferedWriter(new FileWriter(greyListedFile, true)); // true to append
          ) {
         
@@ -104 +108 @@
             // process any previous record
             if(record != null) {
-            processWETrecord(keepURLsWriter, discardURLsWriter,
+            processWETrecord(keepURLsWriter, discardURLsWriter, greyListedURLsWriter,
                      recordCount, contentLength, lineCount,
                      WARCtargetURI, record.toString());
@@ -147 +151 @@
         // flush the last record. If it was a warcinfo record, record would be null here
         if(record != null) {
-        processWETrecord(keepURLsWriter, discardURLsWriter,
+        processWETrecord(keepURLsWriter, discardURLsWriter, greyListedURLsWriter,
                  recordCount, contentLength, lineCount,
                  WARCtargetURI, record.toString());
@@ -169 +173 @@
      */
     private void processWETrecord(BufferedWriter keepURLsWriter, BufferedWriter discardURLsWriter,
+                  BufferedWriter greyListedURLsWriter,
                   int recordID, int contentLength, int lineCount,
                   String recordURI, String record)
@@ -210 +215 @@
     }
     */
-
+    
     if(batchProcessor.isBlacklisted(recordURI)) {
-        parentFolder = batchProcessor.discardFolder;
+
+        
+        // explicit whitelisting overrides blacklisting
+        if(batchProcessor.isWhitelisted(recordURI)) {
+        parentFolder = batchProcessor.keepFolder; //tentative
+        }
+        // if not whitelisted, then greylisting overrides blacklisting
+        else if(batchProcessor.isGreylisted(recordURI)) {
+        parentFolder = batchProcessor.greyListedFolder;
+        System.err.println("@@@GREYLISTED");
+        }
+        else { // only blacklisted
+        parentFolder = batchProcessor.discardFolder;
+        System.err.println("@@@DISCARDING - blacklisted");
+        }
     }
     else if(batchProcessor.isGreylisted(recordURI)) { // e.g. products sites
-        parentFolder = batchProcessor.discardFolder; // TODO: checkfolder
-    } else {
+        // explicit whitelisting overrides greylisting
+        if(batchProcessor.isWhitelisted(recordURI)) {
+        parentFolder = batchProcessor.keepFolder; // tentative
+        }
+        else {
+        parentFolder = batchProcessor.greyListedFolder;
+        System.err.println("@@@GREYLISTED");
+        }
+    }
+
+    // If URL was not blacklisted/greylisted, or was even explicitly whitelisted,
+    // it still can't be in the keep list as it needs further inspection:
+    // it needs sufficient content for language analysis.
+    if(parentFolder != batchProcessor.greyListedFolder && parentFolder != batchProcessor.discardFolder) { // i.e. parentFolder == keepFolder if whiteListed || parentFolder == null
+        
         // If a web page's WET record contains a certain minimum number of words,
         // we will think it's a meaningful web page and has sufficient content for text analysis
@@ -254 +286 @@
         }
     }
-    // if parentFolder still not set, set to discard pile folder
+    // if parentFolder still not set, it means that the content length/num words or lines
+    // were insufficient, so meant to be discarded
     if(parentFolder == null) {
         parentFolder = batchProcessor.discardFolder;
@@ -263 +296 @@
         if (parentFolder == batchProcessor.keepFolder) {
         keepURLsWriter.write(recordURI + "\n");
+        } else if (parentFolder == batchProcessor.greyListedFolder) {
+        greyListedURLsWriter.write(recordURI + "\n");
         } else {
         discardURLsWriter.write(recordURI + "\n");