Changeset 33517 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

Timestamp:

2019-09-24T20:30:40+12:00 (5 years ago)

Author:

ak19

Message:

1. Blacklists were introduced so that too many instances of camelcased words need no longer disqualify WET records from inclusion in the keep pile. Still check camelcasing of words as such words don't get counted as valid words, in the valid word count that determines if there's sufficient content in a WET record. 2. Some more commenting.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java (modified) (4 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -77 +77 @@
     public final File greyListedFile;
 
+    /** Possible values stored in the blackList/whiteList/greyList Maps */
     private final Integer LIST_ENTRY_CONTAINS = new Integer(0);
     private final Integer LIST_ENTRY_STARTSWITH = new Integer(1);
     private final Integer LIST_ENTRY_ENDSWITH = new Integer(2);
     private final Integer LIST_ENTRY_MATCHES = new Integer(3);
-    
+
+    /**
+     * Store url patterns as keys and values indicated whether a url should
+     * match it exactly, start/end with it, or contain it
+     */
     private HashMap<String, Integer> blackList;
     private HashMap<String, Integer> greyList;
     private HashMap<String, Integer> whiteList;
 
+    /** Map of domains we keep and the full urls we're keeping that are of that domain. 
+     * Choosing a TreeMap to preserve natural (alphabetical) ordering of keys,
+     * since a HashMap has no notion of ordering.
+     */
+    private TreeMap<String, TreeSet<String>> domainsToURLsMap;
+    
     // Keep a count of all the records that all WETProcessors instantiated
     // by our main method combined have processed
@@ -144 +155 @@
     }
 
+    // prepare our blacklist, greylist (for inspection) and whitelist
     System.err.println("Loading blacklist.");
     blackList = new HashMap<String, Integer>();
     initURLFilterList(blackList, "url-blacklist-filter.txt");
+    
     System.err.println("Loading greylist.");
     greyList = new HashMap<String, Integer>();
     initURLFilterList(greyList, "url-greylist-filter.txt");
+    
     System.err.println("Loading whitelist.");
     whiteList = new HashMap<String, Integer>();
@@ -159 +173 @@
     
     /**
-     * Takes as input the keepURLs.txt file generated by running WETProcessor instances.
-     * As output produces the URL seed list and regex-urlfilter text files required by nutch,
+     * Using the keepURLs.txt file generated by running WETProcessor instances, produces
+     * as output the URL seed list and regex-urlfilter text files required by nutch, see
      * https://cwiki.apache.org/confluence/display/nutch/NutchTutorial
      */
@@ -448 +462 @@
     File urlFilterFile = new File(outFolder, "regex-urlfilter.txt");
     ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile);
+
+    System.out.println("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
+    
     } catch(Exception e) {
     // can get an exception when instantiating CCWETProcessor instance