Changeset 33615 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java

Timestamp:

2019-10-31T20:03:55+13:00 (4 years ago)

Author:

ak19

Message:

1. Worked out how to configure log4j to log both to console and logfile, so modified the existing laboured code to use this better way. 2. Added some Mongodb links under MoreReading.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java

@@ -49 +49 @@
 */
 public class NutchTextDumpProcessor {
-    private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
+    static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
 
     static boolean DEBUG_MODE = true;
@@ -91 +91 @@
     if(DEBUG_MODE) {
         // START DEBUG
-        debug("__________________________________________");
-        debug("@@@ Found page entry: ");
-        debug("__________________________________________");
-        debug(pageDump.toString());
-        debug("------------------------------------------");
+        logger.debug("__________________________________________");
+        logger.debug("@@@ Found page entry: ");
+        logger.debug("__________________________________________");
+        logger.debug(pageDump.toString());
+        logger.debug("------------------------------------------");
         // END DEBUG
     }
@@ -174 +174 @@
         
     } catch (IOException ioe) {
-        error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
+        logger.error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
     }
 
@@ -302 +302 @@
     
     
-    info("------------- " + this.siteID + " SITE STATS -----------");
-
-    info("SITE DOMAIN: " + this.domainOfSite);
-    info("Total number of web pages in site: " + pages.size());
-    info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
+    logger.info("------------- " + this.siteID + " SITE STATS -----------");
+
+    logger.info("SITE DOMAIN: " + this.domainOfSite);
+    logger.info("Total number of web pages in site: " + pages.size());
+    logger.info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
     
     if(pagesInMRI.size() > 0) {
-        info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
+        logger.info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
         for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
-        info(mriWebPageInfo.toString());
-        }
-    }
-
-    info("                      -----------                   ");
+        logger.info(mriWebPageInfo.toString());
+        }
+    }
+
+    logger.info("                      -----------                   ");
     if(pagesContainingMRI.size() > 0) {     
-        info("The following pages weren't detected as primarily being in Māori");
-        info("But still contained sentences detected as Māori");
+        logger.info("The following pages weren't detected as primarily being in Māori");
+        logger.info("But still contained sentences detected as Māori");
         for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
-        info(mriWebPageInfo.toString());
+        logger.info(mriWebPageInfo.toString());
         }
         
     } else {
-        info("No further pages detected as containing any sentences in MRI");      
-    }
-    info("                      -----------                   ");
+        logger.info("No further pages detected as containing any sentences in MRI");       
+    }
+    logger.info("                      -----------                   ");
     }
 
@@ -358 +358 @@
     
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-    System.err.println(msg);
-    logger.info(msg);
-    }
-    public static void debug(String msg) {
-    System.err.println(msg);
-    logger.debug(msg);
-    }
-    public static void warn(String msg) {
-    System.err.println(msg);
-    logger.warn(msg);
-    }
-    public static void error(String msg) {
-    System.err.println(msg);
-    logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-    logger.error(msg, e);
-    System.err.println("\n"+msg);
-    e.printStackTrace();
-    }
-    
+   
     public static void printUsage() {
-    info("Run this program as:");
-    info("\tNutchTextDumpProcessor <path to 'crawled' folder>");
+    System.err.println("Run this program as:");
+    System.err.println("\tNutchTextDumpProcessor <path to 'crawled' folder>");
     }
     
@@ -393 +372 @@
     File sitesDir = new File(args[0]);
     if(!sitesDir.exists() || !sitesDir.isDirectory()) {
-        error("Error: " + args[0] + " does not exist or is not a directory");
+        logger.error("Error: " + args[0] + " does not exist or is not a directory");
         return;
     }
@@ -430 +409 @@
             File txtDumpFile = new File(siteDir, "dump.txt");
             if(!txtDumpFile.exists()) {
-            error("Text dump file " + txtDumpFile + " did not exist");
+            logger.error("Text dump file " + txtDumpFile + " did not exist");
             continue;
             }
@@ -439 +418 @@
             String siteID = siteDir.getName();
             long lastModified = siteDir.lastModified();
-            debug("Found siteID: " + siteID);           
+            logger.debug("Found siteID: " + siteID);            
             NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(
                  webpagesCSVPrinter, mriSentencesCSVPrinter, mriTxtDetector,
@@ -458 +437 @@
         // can get an exception when instantiating CCWETProcessor instance
         // or with CSV file
-        error(e.getMessage(), e);
+        logger.error(e.getMessage(), e);
     }
     }