Changeset 33578 for gs3-extensions

Timestamp:

2019-10-17T19:31:53+13:00 (5 years ago)

Author:

ak19

Message:

Corrections for compiling the 2 new classes.

Location:

gs3-extensions/maori-lang-detection/src/org/greenstone/atea

Files:

• NutchTextDumpProcessor.java (modified) (6 diffs)
• TextDumpPage.java (modified) (6 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java

@@ -2 +2 @@
 
 import java.io.*;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.ArrayList;
+//import java.util.HashMap;
+//import java.util.Map;
 import java.lang.ArrayIndexOutOfBoundsException;
-    
+
+import org.apache.log4j.Logger;
+
+/**
+ * Class to process the dump text files produced for each site (e.g. site "00001") that
+ * Nutch has finished crawling and whose text has been dumped out to a file called dump.txt.
+ * This reads in the dump.txt file contained in each site folder within the input folder.
+ * (e.g. input folder "crawled" could contain folders 00001 to 01465. Each contains a dump.txt)
+ * Each dump.txt could contain the text contents for an entire site, or for individual pages.
+ * This class then uses class TextDumpPage to parse each webpage within a dump.txt,
+ * which parses out the actual text body content of each webpage's section within a dump.txt.
+ * Finally, MaoriTextDetector is run over that to determine whether the full body text is
+ * likely to be in Maori or not.
+ *
+ * Potential issues: since a web page's text is dumped out by nutch with neither paragraph
+ * nor even newline separator, it's hard to be sure that the entire page is in language.
+ * If it's in multiple languages, there's no way to be sure there aren't promising Maori language
+ * paragraphs contained in a page, if the majority/the remainder happen to be in English.
+ * 
+ * So if we're looking for any paragraphs in Maori to store in a DB, perhaps it's better to run
+ * the MaoriTextDetector.isTextInMaori(BufferedReader reader) over two "lines" at a time,
+ * instead of running it over the entire html body's text.
+ *
+ * TO COMPILE OR RUN, FIRST DO:
+ *    cd maori-lang-detection/apache-opennlp-1.9.1
+ *    export OPENNLP_HOME=`pwd`
+ *    cd maori-lang-detection/src
+ *
+ * TO COMPILE:
+ *    maori-lang-detection/src$
+ *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/*" org/greenstone/atea/NutchTextDumpProcessor.java
+ *
+ * TO RUN:
+ *    maori-lang-detection/src$
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor
+ *
+*/
 public class NutchTextDumpProcessor {
     private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
 
-    private static MaoriTextDetector maoriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
+    private final MaoriTextDetector maoriTxtDetector;
     
     public final String siteID; // is this necessary?
@@ -17 +54 @@
     
     
-    public NutchTextDumpProcessor(String siteID, File txtDumpFile) {
+    public NutchTextDumpProcessor(MaoriTextDetector maoriTxtDetector, String siteID, File txtDumpFile) {
     // siteID is of the form %5d (e.g. 00020) and is just the name of a site folder
     this.siteID = siteID;
-
-
+    this.maoriTxtDetector = maoriTxtDetector;
+    
     pages = new ArrayList<TextDumpPage>();
 
@@ -37 +74 @@
             pageDump.append("\n");
         } else {
-            TextDumpPage page = new TextDumpPage(pageDump.toString());
+            TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
             // parses the fields and body text of a webpage in nutch's txt dump of entire site
             //page.parseFields();
@@ -57 +94 @@
 
     String text = getTextForPage(pageID);
+    // QTODO: what to do when page body text is empty?
+    if(text.equals("")) return false;
     return maoriTxtDetector.isTextInMaori(text);
     }
@@ -120 +159 @@
 
     try {
+        MaoriTextDetector mriTxtDetector = new MaoriTextDetector(false); // false: run non-silent
         File[] sites = sitesDir.listFiles();
         for(File siteDir : sites) { // e.g. 00001
         // look for dump.txt
-        File txtDumpFile = new File(siteDir, dump.txt);
+        File txtDumpFile = new File(siteDir, "dump.txt");
         if(!txtDumpFile.exists()) {
             error("Text dump file " + txtDumpFile + " did not exist");
@@ -131 +171 @@
         else {
             String siteID = siteDir.getName();
-            NutchTextDumpProcessor nutchTxtDump = NutchTextDumpProcessor(siteID, txtDumpFile);
+            NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);
             
         }

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java

@@ -5 +5 @@
 import java.util.Map;
 
+import org.apache.log4j.Logger;
+
+
 public class TextDumpPage {
-
+    private static Logger logger = Logger.getLogger(org.greenstone.atea.TextDumpPage.class.getName());
+    
     public static final String TEXT_START_MARKER="text:start:";
     public static final String TEXT_END_MARKER="text:end:";
     
-    private Map tuples;
+    private Map<String, String> tuples;
     
     public TextDumpPage(String siteID, String unparsedPageDump) {
-    tuples = new HashMap();
+    tuples = new HashMap<String, String>();
     
     try (
@@ -25 +29 @@
         String pageURL = line.substring(endIndex);
         
-        tuples.add("pageURL", pageURL.trim());
+        tuples.put("pageURL", pageURL.trim());
 
         String key = line.substring(endIndex);
-        tuples.add("key", key.trim());
+        tuples.put("key", key.trim());
 
         boolean readingPageText = false;
@@ -42 +46 @@
             String k = line.substring(0, endIndex);
             String v = line.substring(endIndex+1);
-            tuples.add(k.trim(), v.trim());
+            tuples.put(k.trim(), v.trim());
             }
             
             else if(line.equals(TEXT_START_MARKER)) { // dealing with the page body text
-            StringBuilder pageText = new StringBuilder();
+            pageText = new StringBuilder();
             readingPageText = true;
             }                   
@@ -56 +60 @@
             // finished with a page body            
             // remove any FINAL artificial newline we introduced
-            tuples.add("pageText", pageText.toString().trim());
+            tuples.put("pageText", pageText.toString().trim());
             readingPageText = false;
             pageText = null;
@@ -71 +75 @@
         // If the page had no pageText, add a "pageText" -> "" mapping
         if(!tuples.containsKey("pageText")) {
-        tuples.add("pageText", "");
+        tuples.put("pageText", "");
         }
         
@@ -91 +95 @@
     return tuples.get(key);
     }
+
+    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
+    public static void info(String msg) {
+    System.err.println(msg);
+    logger.info(msg);
+    }
+    public static void debug(String msg) {
+    System.err.println(msg);
+    logger.debug(msg);
+    }
+    public static void warn(String msg) {
+    System.err.println(msg);
+    logger.warn(msg);
+    }
+    public static void error(String msg) {
+    System.err.println(msg);
+    logger.error(msg);
+    }
+    public static void error(String msg, Exception e) {
+    logger.error(msg, e);
+    System.err.println("\n"+msg);
+    e.printStackTrace();
+    }
+    
 }