Changeset 33517

Timestamp:

2019-09-24T20:30:40+12:00 (5 years ago)

Author:

ak19

Message:

1. Blacklists were introduced so that too many instances of camelcased words need no longer disqualify WET records from inclusion in the keep pile. Still check camelcasing of words as such words don't get counted as valid words, in the valid word count that determines if there's sufficient content in a WET record. 2. Some more commenting.

Location:

gs3-extensions/maori-lang-detection/src/org/greenstone/atea

Files:

• CCWETProcessor.java (modified) (4 diffs)
• WETProcessor.java (modified) (4 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -77 +77 @@
     public final File greyListedFile;
 
+    /** Possible values stored in the blackList/whiteList/greyList Maps */
     private final Integer LIST_ENTRY_CONTAINS = new Integer(0);
     private final Integer LIST_ENTRY_STARTSWITH = new Integer(1);
     private final Integer LIST_ENTRY_ENDSWITH = new Integer(2);
     private final Integer LIST_ENTRY_MATCHES = new Integer(3);
-    
+
+    /**
+     * Store url patterns as keys and values indicated whether a url should
+     * match it exactly, start/end with it, or contain it
+     */
     private HashMap<String, Integer> blackList;
     private HashMap<String, Integer> greyList;
     private HashMap<String, Integer> whiteList;
 
+    /** Map of domains we keep and the full urls we're keeping that are of that domain. 
+     * Choosing a TreeMap to preserve natural (alphabetical) ordering of keys,
+     * since a HashMap has no notion of ordering.
+     */
+    private TreeMap<String, TreeSet<String>> domainsToURLsMap;
+    
     // Keep a count of all the records that all WETProcessors instantiated
     // by our main method combined have processed
@@ -144 +155 @@
     }
 
+    // prepare our blacklist, greylist (for inspection) and whitelist
     System.err.println("Loading blacklist.");
     blackList = new HashMap<String, Integer>();
     initURLFilterList(blackList, "url-blacklist-filter.txt");
+    
     System.err.println("Loading greylist.");
     greyList = new HashMap<String, Integer>();
     initURLFilterList(greyList, "url-greylist-filter.txt");
+    
     System.err.println("Loading whitelist.");
     whiteList = new HashMap<String, Integer>();
@@ -159 +173 @@
     
     /**
-     * Takes as input the keepURLs.txt file generated by running WETProcessor instances.
-     * As output produces the URL seed list and regex-urlfilter text files required by nutch,
+     * Using the keepURLs.txt file generated by running WETProcessor instances, produces
+     * as output the URL seed list and regex-urlfilter text files required by nutch, see
      * https://cwiki.apache.org/confluence/display/nutch/NutchTutorial
      */
@@ -448 +462 @@
     File urlFilterFile = new File(outFolder, "regex-urlfilter.txt");
     ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile);
+
+    System.out.println("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
+    
     } catch(Exception e) {
     // can get an exception when instantiating CCWETProcessor instance

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -188 +188 @@
     File parentFolder = null;
 
-    // want to match "product(s)" but not "production"
-    //if(recordURI.matches(".*/?product[^a-rt-z].*")) {//if(recordURI.matches(".*/?products?/?.*")) {
-    
-
-    /*
-    if(recordURI.contains("product") && !recordURI.contains("production")) {
-
-        // don't want a "translated" product site/online store
-        // These curiously often tend to have "product(s)" in the URL
-        parentFolder = batchProcessor.discardFolder;
-    }
-
-    else if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
-        parentFolder = batchProcessor.keepFolder;
-        System.err.println("@@@KEEPING");
-    } else if(contentLength >= MIN_CONTENT_LENGTH_WRAPPED_LINE) {
-        int countSpaces = 0;
-        for(int i = 0; i < record.length(); i++) {
-        if(record.charAt(i) == ' ') countSpaces++;
-        }
-        if(countSpaces >= MIN_SPACES_IN_A_WRAPPED_LINE) {
-        // So we have at least 500 chars (possibly on a single wrapped line)
-        // containing at least 10 spaces. Such a record is also worth keeping.
-        parentFolder = batchProcessor.keepFolder;
-        }
-    }
-    */
-    
+
     if(batchProcessor.isBlacklisted(recordURI)) {
-
         
         // explicit whitelisting overrides blacklisting
@@ -223 +195 @@
         parentFolder = batchProcessor.keepFolder; //tentative
         }
-        // if not whitelisted, then greylisting overrides blacklisting
+        // if not whitelisted, then greylisting still overrides blacklisting
         else if(batchProcessor.isGreylisted(recordURI)) {
         parentFolder = batchProcessor.greyListedFolder;
         System.err.println("@@@GREYLISTED");
         }
-        else { // only blacklisted
+        else { // url was only blacklisted
         parentFolder = batchProcessor.discardFolder;
         System.err.println("@@@DISCARDING - blacklisted");
@@ -247 +219 @@
     // it still can't be in the keep list as it needs further inspection:
     // it needs sufficient content for language analysis.
+    // We don't care about the combination of number of lines and content-length,
+    // we just care about the number of "valid words" as defined by us.
     if(parentFolder != batchProcessor.greyListedFolder && parentFolder != batchProcessor.discardFolder) { // i.e. parentFolder == keepFolder if whiteListed || parentFolder == null
         
@@ -267 +241 @@
 
         // throw away if n words contain camelcase, which is another case of words glued together
+        // For now, we'll only skip camelcased words in our count of valid words
         if(word.matches(".*[a-z][A-Z].*") && word.length() >= 5) {
             numCamelCaseWords++;            
-        }
-        
+        }       
         // In Maori, word length of 1 is not uncommon
         // but let's skip camelcased words when counting valid words
-        else if(word.length() >= 1 && word.length() <= batchProcessor.MAX_WORD_LENGTH) validWordCount++;
-        }
-
-        // dump if too many camelcase words (ideally keep none of that kind?)
+        else if(word.length() >= 1 && word.length() <= batchProcessor.MAX_WORD_LENGTH) {
+            validWordCount++;
+        }
+        }
+
+        
+        /*
+        // dump if too many camelcase words (ideally keep no WET record of that kind?)
         if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
         parentFolder = batchProcessor.discardFolder;
         System.err.println("@@@DISCARDING - CAMELCASED CONTENTS");
         }
-        else if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
+        else*/
+        // For now, don't discount content with too many camelcased words
+        // Just focus on whether there are a sufficient number of valid words
+        // (camelcased words are however still ignored in our count of valid words)
+        if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
         parentFolder = batchProcessor.keepFolder;
         System.err.println("@@@KEEPING");