Changeset 33582 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java

Timestamp:

2019-10-17T23:12:38+13:00 (5 years ago)

Author:

ak19

Message:

NutchTextDumpProcessor prints each crawled site's stats: number of webpages per crawled site and how many of those were detected by OpenNLP as being in Maori (mri). Needed to make a reusable method in CCWETProcessor as public and static.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java (modified) (4 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java

@@ -77 +77 @@
             }
             else { // dealing with the rest of the page dump's metadata
+            // QTODO: nutch's text dump output is problematic
+            // strange characters are in the stream and end up here
+            // and can't detect end of metadata or even end of line.
             endIndex = line.indexOf(":");
             if(endIndex != -1) { 
@@ -83 +86 @@
                 tuples.put(k.trim(), v.trim());
             } else {
-                error("No meta key for meta: " + line);
+                if(NutchTextDumpProcessor.DEBUG_MODE) {
+                error("No meta key for meta: " + line);
+                error(unparsedPageDump);
+                }
             }
             }
@@ -102 +108 @@
     
     // START DEBUG
-    debug("__________________________________________");
-    for(Map.Entry<String, String> entry : tuples.entrySet()) {
-        String key = entry.getKey();
-        String value = entry.getValue();        
-        debug(key + " - " + value);
-    }
-    debug("__________________________________________");
+    debugTuples();
     // END DEBUG
     
     }
 
+    public void debugTuples() {
+    if(NutchTextDumpProcessor.DEBUG_MODE) {
+        debug("__________________________________________");
+        for(Map.Entry<String, String> entry : tuples.entrySet()) {
+        String key = entry.getKey();
+        String value = entry.getValue();        
+        debug(key + " - " + value);
+        }
+        debug("__________________________________________");
+    }
+    }
+    
     
     public String getPageURL() {
-    return tuples.get("url");
+    return tuples.get("pageURL");
     }
 
@@ -126 +138 @@
     }
 
+    public void add(String key, String value) {
+    tuples.put(key, value);
+    }
+
+    public void addMRILanguageStatus(boolean status) {
+    if(status) {
+        tuples.put("isMRI", "true");
+    } else {
+        tuples.put("isMRI", "false");
+    }
+    }
+
+    public boolean getMRILanguageStatus() {
+    String value = tuples.get("isMRI");
+    if(value == null) {
+        return false;
+    }
+    if(value.equals("true")) {
+        return true;
+    }
+    else {
+        return false;
+    }
+
+    }
+    
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
     public static void info(String msg) {