Changeset 33480

Timestamp:

2019-09-16T19:45:01+12:00 (5 years ago)

Author:

ak19

Message:

Much harder to remove pages where words are fused together as some are shorter than valid word-lengths of 15 chars, some are long, when the number of valid words still come to more than the required number of 20. The next solution was to ignore pages that had more than 2 instances of camelcase, but valid pages (actual Maori language pages) may end up with a few more camelcased words if navigation items get fused together. Not sure what to do.

Location:

gs3-extensions/maori-lang-detection

Files:

• conf/config.properties (modified) (1 diff)
• src/org/greenstone/atea/WETProcessor.java (modified) (5 diffs)

gs3-extensions/maori-lang-detection/conf/config.properties

@@ -15 +15 @@
 WETprocessor.min.content.length.wrapped.line=500
 WETprocessor.min.spaces.per.wrapped.line=10
+
+# Arbitrary cutoff values for WETProcessor.java
+# for determining whether a WET record has sufficient and sensible content
+WETprocessor.max.word.length=15
+WETprocessor.min.num.words=20
+WETprocessor.max.words.camelcase=10

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -31 +31 @@
     // In Java, can initialize static final variables inside a static block
     // But the unavoidable try/catch in this static block prevents initialization of
-    // the static final int variables further below that therefore need to be declared
-    // and initialized thereafter.
+    // the static final int variables (seen further below) inside the block itself,
+    // that therefore need to be declared and initialized thereafter.
     static {    
-    
     // load up the properties from the config file
     try (InputStream infile = org.greenstone.atea.WETProcessor.class.getClassLoader().getResourceAsStream("config.properties")) {
@@ -50 +49 @@
     // Providing fall-back cuttoff values if config.properties doesn't load
     // or doesn't have the named props. But what happens when Integer.parseInt throws an exception?
+    /*
     private static final int MIN_CONTENT_LENGTH = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length", "100"));
     private static final int MIN_LINE_COUNT= Integer.parseInt(configProperties.getProperty("WETprocessor.min.line.count", "2"));
     private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length.wrapped.line", "500"));
     private static final int MIN_SPACES_IN_A_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.spaces.per.wrapped.line", "10"));
-
+    */
+    private static final int MAX_WORD_LENGTH = Integer.parseInt(configProperties.getProperty("WETprocessor.max.word.length", "15")); // to identify and skip web pages where content consists of words glued together (with no spaces)
+    private static final int MIN_NUM_WORDS = Integer.parseInt(configProperties.getProperty("WETprocessor.min.num.words", "20"));
+    private static final int MAX_WORDS_CAMELCASE = Integer.parseInt(configProperties.getProperty("WETprocessor.max.words.camelcase", "10"));
+    
     // File paths shared across WETProcessor instances
     private static File discardFolder;
@@ -202 +206 @@
 
     // want to match "product(s)" but not "production"
-    
     //if(recordURI.matches(".*/?product[^a-rt-z].*")) {//if(recordURI.matches(".*/?products?/?.*")) {
+    
+
     if(recordURI.contains("product") && !recordURI.contains("production")) {
 
@@ -209 +214 @@
         // These curiously often tend to have "product(s)" in the URL
         parentFolder = WETProcessor.discardFolder;
-    }   
+    }
+    /*
     else if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
         parentFolder = WETProcessor.keepFolder;
@@ -224 +230 @@
         }
     }
-
+    */
+
+    else {
+        // If a web page's WET record contains a certain minimum number of words,
+        // we will think it's a meaningful web page and has sufficient content for text analysis
+        // to have been successful. Cut off values at present are:
+        // - a minimum of 20 words
+        // - a word consists of 1 to 15 chars demarcated by spaces. Any more chars may point
+        // to words having been glued together. This is used by irrelevant sites and moreover
+        // can't be analysed for language, so may not be actually MRI.
+        
+        // Though StringTokenizer still in use, as seen in discussion at
+        // https://stackoverflow.com/questions/6983856/why-is-stringtokenizer-deprecated
+        // String.split(regex) seems better for splitting on general whitespace
+        String[] allWords = record.split("\\s");
+        int validWordCount = 0;
+        int numCamelCaseWords = 0;
+        for(int i = 0; i < allWords.length; i++) {
+        String word = allWords[i];
+
+        // throw away if n words contain camelcase, which is another case of words glued together
+        if(word.matches(".*[a-z][A-Z].*") && word.length() >= 5) {
+            numCamelCaseWords++;            
+        }
+        
+        // In Maori, word length of 1 is not uncommon
+        // but let's skip camelcased words when counting valid words
+        else if(word.length() >= 1 && word.length() <= MAX_WORD_LENGTH) validWordCount++;
+        }
+
+        // dump if too many camelcase words (ideally keep none of that kind?)
+        if(numCamelCaseWords >= MAX_WORDS_CAMELCASE) {
+        parentFolder = WETProcessor.discardFolder;
+        System.err.println("@@@DISCARDING - CAMELCASED CONTENTS");
+        }
+        else if(validWordCount >= MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
+        parentFolder = WETProcessor.keepFolder;
+        System.err.println("@@@KEEPING");
+        }
+    }
     // if parentFolder still not set, set to discard pile folder
     if(parentFolder == null) {