Changeset 33497

Timestamp:

2019-09-22T21:17:48+12:00 (5 years ago)

Author:

ak19

Message:

First version of discard url filter file. Inefficient implementation. Better to read the file once, adjust the filters as required and keep in memory to do the comparisons with each URL.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java (modified) (5 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -212 +212 @@
     
 
+    /*
     if(recordURI.contains("product") && !recordURI.contains("production")) {
 
@@ -218 +219 @@
         parentFolder = WETProcessor.discardFolder;
     }
-    /*
+
     else if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
         parentFolder = WETProcessor.keepFolder;
@@ -235 +236 @@
     */
 
-    else {
+    if(isInDiscardFilter(recordURI)) {
+        parentFolder = WETProcessor.discardFolder;
+    }
+    else if(isInCheckFilter(recordURI)) { // products sites
+        parentFolder = WETProcessor.discardFolder; // TODO: checkfolder
+    } else {
         // If a web page's WET record contains a certain minimum number of words,
         // we will think it's a meaningful web page and has sufficient content for text analysis
@@ -359 +365 @@
     } catch (IOException ioe) {
         ioe.printStackTrace();
-        System.err.println("\n@@@@@@@@@ Error writing to either " + seedURLsFile + " or " + urlFilterFile);
+        System.err.println("\n@@@@@@@@@ Error writing to " + seedURLsFile);
     }
 
@@ -373 +379 @@
     } catch (IOException ioe) {
         ioe.printStackTrace();
-        System.err.println("\n@@@@@@@@@ Error writing to either " + seedURLsFile + " or " + urlFilterFile);
-    }
-    }
-
+        System.err.println("\n@@@@@@@@@ Error writing to " + urlFilterFile);
+    }
+    }
+
+    /**
+     * Checks URL parameter against each line ("filter") of conf/url-discard-filter.txt to decide
+     * whether it is in the discard list.
+     * Filters don't represent actual regex, just ^ and $ as start and end terminators.
+     * By not having this method deal with actual regex for filters, this has the advantage that
+     * we don't have to remember to escape or double escape each filter to turn it into a regex.
+     */
+    public boolean isInDiscardFilter(String url) {
+
+    String discardFilterFile = "url-discard-filter.txt"; // in conf folder
+
+    try (
+         BufferedReader reader = new BufferedReader(new InputStreamReader(org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream(discardFilterFile), "UTF-8"));
+         ) {
+        String filter = null;
+        while((filter = reader.readLine()) != null) {
+        if(filter.trim().equals("")) {
+            continue;
+        }
+        //System.err.println("Got filter: " + filter);
+        if(filter.startsWith("^") && filter.endsWith("$") && url.equals(filter.substring(1, filter.length()-1))) {
+            System.err.println("*** Discarding url " + url + "\n\tas it MATCHES filter " + filter);
+        }
+        else if(filter.startsWith("^") && url.startsWith(filter.substring(1))) {
+            System.err.println("*** Discarding url " + url + "\n\tas it STARTS WITH filter " + filter);
+            return true;
+        }
+        else if(filter.endsWith("$") && url.endsWith(filter.substring(0, filter.length()-1))) {
+            System.err.println("*** Discarding url " + url + "\n\tas it ENDS WITH filter " + filter);
+            return true;
+        }
+        else if(url.contains(filter)) {
+            System.err.println("*** Discarding url " + url + "\n\tas it CONTAINS filter " + filter);
+            return true;
+        }
+                
+        }
+        
+    } catch (IOException ioe) {
+        ioe.printStackTrace();
+        System.err.println("\n@@@@@@@@@ Error reading from " + discardFilterFile);
+    }
+
+    return false;
+    }
+
+    // TODO
+    public boolean isInCheckFilter(String url) {
+    //System.err.println("isInCheckFilter(url) is not yet implemented");    
+    return false;
+    }
     
     //public static int getRecordCount() { return recordCount; }