Changeset 33615 for gs3-extensions

Timestamp:

2019-10-31T20:03:55+13:00 (4 years ago)

Author:

ak19

Message:

1. Worked out how to configure log4j to log both to console and logfile, so modified the existing laboured code to use this better way. 2. Added some Mongodb links under MoreReading.

Location:

gs3-extensions/maori-lang-detection

Files:

• MoreReading/crawling-Nutch.txt (modified) (1 diff)
• conf/config.properties (modified) (1 diff)
• conf/log4j.properties (modified) (1 diff)
• src/org/greenstone/atea/CCWETProcessor.java (modified) (28 diffs)
• src/org/greenstone/atea/MaoriTextDetector.java (modified) (4 diffs)
• src/org/greenstone/atea/NutchTextDumpProcessor.java (modified) (9 diffs)
• src/org/greenstone/atea/TextDumpPage.java (modified) (5 diffs)
• src/org/greenstone/atea/WETProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt

@@ -332 +332 @@
 https://ipinfo.info/html/ip_checker.php
 
+
+
+----------
+MongoDB
+Installation:
+    https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
+        https://docs.mongodb.com/manual/administration/install-on-linux/
+    https://hevodata.com/blog/install-mongodb-on-ubuntu/
+    https://www.digitalocean.com/community/tutorials/how-to-install-mongodb-on-ubuntu-16-04
+    CENTOS (Analytics): https://tecadmin.net/install-mongodb-on-centos/
+    FROM SOURCE: https://github.com/mongodb/mongo/wiki/Build-Mongodb-From-Source
+GUI:
+    https://robomongo.org/
+    Robomongo is Robo 3T now
+
+https://www.tutorialspoint.com/mongodb/mongodb_java.htm
+JAR FILE:
+    http://central.maven.org/maven2/org/mongodb/mongo-java-driver/
+    https://mongodb.github.io/mongo-java-driver/

gs3-extensions/maori-lang-detection/conf/config.properties

@@ -21 +21 @@
 WETprocessor.min.num.words=20
 WETprocessor.max.words.camelcase=10
+
+
+mongodb.user=admin
+mongodb.pwd=pinky
+#CHANGEME
+

gs3-extensions/maori-lang-detection/conf/log4j.properties

@@ -12 +12 @@
 
 # Let's log everything from the most basic DEBUG msgs on (so INFO, WARN, ERROR too)
-log4j.rootCategory=DEBUG, mldlog
+# Also want to log to the console, see
+# https://stackoverflow.com/questions/3382985/how-to-make-log4j-to-write-to-the-console-as-well
 
-# copied from GS3
-log4j.category.web.servlets.DirectInput=INFO, mldlog
-# Prevent logging duplicating its messages into rootLogger:
-# https://stackoverflow.com/questions/9208710/java-log4j-log-to-both-category-and-root-logger
-log4j.additivity.web.servlets.DirectInput=false
+log4j.rootCategory=DEBUG, console, mldlog
+
+# Define Console Appender
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+# Define the layout for console appender. If you do not define it, you will get an error
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.Target=System.err
 
 # We're saying our log "mldlog" is some file that gets appended to,

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -150 +150 @@
         
     } catch(Exception e) {
-        error("Exception attempting to read properties from config.properties.", e);
+        logger.error("Exception attempting to read properties from config.properties.", e);
     }
 
     if(configProperties.size() == 0) {
-        warn("*** Warning: no values read into config properties. Using defaults.");
+        logger.warn("*** Warning: no values read into config properties. Using defaults.");
     }
     
@@ -190 +190 @@
 
     // prepare our blacklist, greylist (for inspection) and whitelist
-    info("Loading blacklist.");
+    logger.info("Loading blacklist.");
     blackList = new HashMap<String, Integer>();
     initURLFilterList(blackList, "url-blacklist-filter.txt");
     
-    info("Loading greylist.");
+    logger.info("Loading greylist.");
     greyList = new HashMap<String, Integer>();
     initURLFilterList(greyList, "url-greylist-filter.txt");
     
-    info("Loading whitelist.");
+    logger.info("Loading whitelist.");
     whiteList = new HashMap<String, Integer>();
     initURLFilterList(whiteList, "url-whitelist-filter.txt");
 
     // Create the map of topSites
-    info("Loading map of topsites with regex of allowable url patterns for each topsite.");
+    logger.info("Loading map of topsites with regex of allowable url patterns for each topsite.");
     topSitesMap = new HashMap<String, String>();
     
@@ -226 +226 @@
         topSitesMap.put(topsite, allowed_url_pattern);
 
-        //debug("@@@@ topsite: " + topsite + " - " + allowed_url_pattern);
+        //logger.debug("@@@@ topsite: " + topsite + " - " + allowed_url_pattern);
         
         }
     } catch(Exception e) {
-        error("@@@@@@@@@ Error attempting to parse CSV format of text file " + topSitesCSVData, e);
+        logger.error("@@@@@@@@@ Error attempting to parse CSV format of text file " + topSitesCSVData, e);
     }   
  
-    //debug("Prematurely terminating for testing purposes.");
+    //logger.debug("Prematurely terminating for testing purposes.");
     //System.exit(-1);
     }
@@ -323 +323 @@
         // for later manual inspection
         if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
+            /*
             if(!possibleProductDomains.contains(domainWithProtocol)) {
 
@@ -333 +334 @@
                 countryCode = ""; // forces domain to be included for inspection
                 
-                error("Could not check if domain " + domainWithProtocol
+                logger.error("Could not check if domain " + domainWithProtocol
                   + " was in country: " + countryCode,
                   exceptObj);
@@ -348 +349 @@
                 possibleProductSitesWriter.write("\t" + url + "\n");
             }
-            } /*else {
+            }*/ /*else {
             // already wrote out domain to file at some point, write just the URL out to file
             possibleProductSitesWriter.write("\t" + url + "\n");
@@ -355 +356 @@
         }
     } catch (IOException ioe) {
-        error("@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile, ioe);
+        logger.error("@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile, ioe);
     }
 
@@ -393 +394 @@
     String value = topSitesMap.get("wikipedia.org");
     if(value == null) {
-        debug("### wikipedia.org had null value");
+        logger.debug("### wikipedia.org had null value");
     } else {
-        debug("### wikipedia.org had value: " + value);
+        logger.debug("### wikipedia.org had value: " + value);
     } // DEBUG
         */
@@ -407 +408 @@
 
         /*if(domain.contains("docs.google.com")) {
-            debug("domain with protocol: " + domainWithProtocol);
-            debug("domain: " + domain);
+            logger.debug("domain with protocol: " + domainWithProtocol);
+            logger.debug("domain: " + domain);
             }*/
         
@@ -531 +532 @@
             
         } catch (IOException ioe) {
-            error("@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile, ioe);
+            logger.error("@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile, ioe);
         }
         
@@ -537 +538 @@
 
     } catch (IOException ioe) {
-        error("\n@@@@@@@@@ Error writing to one of:\n\t" + seedURLsFile
+        logger.error("\n@@@@@@@@@ Error writing to one of:\n\t" + seedURLsFile
                    + "\n\t" + urlFilterFile
                    + "\n\t" + domainURLsFile
@@ -545 +546 @@
     /*
     // BEGIN DEBUG
-    debug("@@@@ TopSitesMap contains: ");
+    logger.debug("@@@@ TopSitesMap contains: ");
     for(Map.Entry<String, String> entry : topSitesMap.entrySet()) {
         String topSite = entry.getKey();
         String urlPattern = entry.getValue();       
-        debug(topSite + " - " + urlPattern);
+        logger.debug(topSite + " - " + urlPattern);
     } // END DEBUG
     */
@@ -620 +621 @@
         if(allowed_url_pattern != null) { // if topSitesMap.containsKey(domain);
         // there's an entry for the URL in the topSitesMap
-        debug("##### A top site matches URL domain " + domain);
+        logger.debug("##### A top site matches URL domain " + domain);
 
         // if we're dealing with SUBDOMAIN-COPY, then the fullSeedDomain, with or without
@@ -687 +688 @@
     String domainWithoutProtocol = getDomainForURL(url, false); // remove protocol
     if(!isBlackListed && url.contains("jasmin")) {
-        warn("### Blacklisting additional domain (likely an adult site): " + domainWithoutProtocol);
+        logger.warn("### Blacklisting additional domain (likely an adult site): " + domainWithoutProtocol);
         blackList.put(domainWithoutProtocol, LIST_ENTRY_CONTAINS);
     }
@@ -722 +723 @@
     // if filterListFilename does not exist in the conf folder, just return
     if(MY_CLASSLOADER.getResource(filterListFilename) == null) {
-        warn("Filter list filename: " + filterListFilename + " does not exist");
+        logger.warn("Filter list filename: " + filterListFilename + " does not exist");
         return;     
     }
@@ -744 +745 @@
             filter = filter.substring(1);
             list.put(filter, LIST_ENTRY_STARTSWITH);
-            //debug("Match filter startswith: " + filter);
+            //logger.debug("Match filter startswith: " + filter);
         }
         else if(filter.endsWith("$")) {
             filter = filter.substring(0, filter.length()-1);
             list.put(filter, LIST_ENTRY_ENDSWITH);
-            //debug("@@@ Match filter endswith: " + filter);
+            //logger.debug("@@@ Match filter endswith: " + filter);
         }
         else {
             list.put(filter, LIST_ENTRY_CONTAINS);
         }
-        //debug("Got filter: " + filter);
+        //logger.debug("Got filter: " + filter);
         }
         
     } catch (IOException ioe) {
-        error("@@@@@@@@@ Error reading into map from file " + filterListFilename, ioe);
+        logger.error("@@@@@@@@@ Error reading into map from file " + filterListFilename, ioe);
     }
     
@@ -781 +782 @@
     for(int i = 0; i < WETFiles.length; i++) {
         File WETFile = WETFiles[i];     
-        debug("Processing WETfile: " + WETFile);
+        logger.debug("Processing WETfile: " + WETFile);
 
         // Any .gz files listed means they haven't been unzipped yet. So unzip.
@@ -797 +798 @@
 
         if(!WETFile.exists() || !WETFile.isFile()) {        
-        error("Error: " + WETFile + " does not exist (failure to unzip?)");
+        logger.error("Error: " + WETFile + " does not exist (failure to unzip?)");
         return;
         }
@@ -803 +804 @@
         // Finally, we can process this WETFile's records into the keep and discard pile
         wetFileCount++;
-        debug("Off to process " + WETFile);
+        logger.debug("Off to process " + WETFile);
         String crawlID = ccrawlWETFileDir.getName(); // something like CC-MAIN-YYYY-##-wet-files
         crawlID = crawlID.substring("CC-MAIN-".length(), crawlID.indexOf("-wet-files")); // YYYY-##     
@@ -817 +818 @@
 
 
-    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-    System.err.println(msg);
-    logger.info(msg);
-    }
-    public static void debug(String msg) {
-    System.err.println(msg);
-    logger.debug(msg);
-    }
-    public static void warn(String msg) {
-    System.err.println(msg);
-    logger.warn(msg);
-    }
-    public static void error(String msg) {
-    System.err.println(msg);
-    logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-    logger.error(msg, e);
-    System.err.println("\n"+msg);
-    e.printStackTrace();
-    }
-    
+    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //    
     public static void printUsage() {
-    info("Run this program as:");
-    info("\tCCWetProcessor <path to 'ccrawl-data' folder> <output folder path>");   
+    System.err.println("Run this program as:");
+    System.err.println("\tCCWetProcessor <path to 'ccrawl-data' folder> <output folder path>"); 
     }
 
@@ -852 +831 @@
     public boolean accept(File dir, String name) {
         if(name.endsWith(".warc.wet")) {
-        debug("Will include " + name + " for processing.");
+        logger.debug("Will include " + name + " for processing.");
         return true;
         }
@@ -860 +839 @@
         File unzippedVersion = new File(dir, nameWithoutGZext);
         if(unzippedVersion.exists()) {
-            debug("--- Unzipped version " + unzippedVersion + " exists.");
-            debug("Skipping " + name);
+            logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
+            logger.debug("Skipping " + name);
             return false; // don't count gzipped version if unzipped version exists.
         }
         else {
-            debug("Only zipped version " + name + " exists.");
+            logger.debug("Only zipped version " + name + " exists.");
             return true; // No unzipped version, so have to work with gzipped version
         }
@@ -871 +850 @@
 
         // we're not even interested in any other file extensions
-        debug("Not a WET file. Skipping " + name);
+        logger.debug("Not a WET file. Skipping " + name);
         return false;
     }
@@ -887 +866 @@
         }
         else {
-        info("File " + f + " is not a directory");
+        logger.info("File " + f + " is not a directory");
         }
         return false;                  
@@ -901 +880 @@
     File commoncrawlDir = new File(args[0]);
     if(!commoncrawlDir.exists() || !commoncrawlDir.isDirectory()) {
-        error("Error: " + args[0] + " does not exist or is not a directory");
+        logger.error("Error: " + args[0] + " does not exist or is not a directory");
         return;
     }
@@ -907 +886 @@
     File outFolder = new File(args[1]);
     if(!outFolder.exists() || !outFolder.isDirectory()) {
-        error("Error: " + args[1] + " does not exist or is not a directory.");
+        logger.error("Error: " + args[1] + " does not exist or is not a directory.");
         return;
     }   
@@ -918 +897 @@
     for(int i = 0; i < ccrawlFolders.length; i++) {
         File ccrawlFolder = ccrawlFolders[i];
-        info("About to process commoncrawl WET files folder: " + ccrawlFolder);
+        logger.info("About to process commoncrawl WET files folder: " + ccrawlFolder);
         ccWETFilesProcessor.processAllWETFilesOfCrawl(ccrawlFolder);        
     }
@@ -933 +912 @@
     ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile, domainURLsFile, topSitesMatchedFile, possibleProductSitesFile);
 
-    info("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
-
-    info("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
+    logger.info("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
+
+    logger.info("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
     
     
     } catch(Exception e) {
     // can get an exception when instantiating CCWETProcessor instance
-    error(e.getMessage(), e);
+    logger.error(e.getMessage(), e);
     }
     

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java

@@ -12 +12 @@
 package org.greenstone.atea;
 
-import java.io.*;
+
 import opennlp.tools.langdetect.*;
 import opennlp.tools.sentdetect.*;
 import opennlp.tools.util.*;
 
+import java.io.*;
 import java.util.ArrayList;
+
+//import org.apache.log4j.Logger;
+
 
 /**
@@ -27 +31 @@
  *
  * Then, to compile this program, do the following from the "src" folder (the folder containing this java file):
- *    maori-lang-detection/src$ javac -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/MaoriTextDetector.java
+ *    maori-lang-detection/src$ javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/MaoriTextDetector.java
  *
  * To run this program, issue one of the following commands from the "src" folder (the folder containing this java file):
  *
- *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --help
- *
- *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --file <full/path/to/textfile>
- *
- *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector -
+ *    maori-lang-detection/src$ java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --help
+ *
+ *    maori-lang-detection/src$ java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector --file <full/path/to/textfile>
+ *
+ *    maori-lang-detection/src$ java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org.greenstone.atea.MaoriTextDetector -
  *       Press enter. This variant of the program expects text to stream in from standard input.
  *       If entering text manually, then remember to press Ctrl-D to indicate the usual end of StdIn.
@@ -43 +47 @@
  */
 public class MaoriTextDetector extends TextLanguageDetector {
+    //static Logger logger = Logger.getLogger(org.greenstone.atea.MaoriTextDetector.class.getName());
+    
     /** The 3 letter language code for Maori in ISO 639-2 or ISO 639-3 */ 
     public static final String MAORI_3LETTER_CODE = "mri";
@@ -135 +141 @@
     }
     public static void doPrintErr(boolean runSilent, String msg) {
-    if(!runSilent) System.err.println(msg); 
+    if(!runSilent) System.err.println(msg);
     }
     

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java

@@ -49 +49 @@
 */
 public class NutchTextDumpProcessor {
-    private static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
+    static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpProcessor.class.getName());
 
     static boolean DEBUG_MODE = true;
@@ -91 +91 @@
     if(DEBUG_MODE) {
         // START DEBUG
-        debug("__________________________________________");
-        debug("@@@ Found page entry: ");
-        debug("__________________________________________");
-        debug(pageDump.toString());
-        debug("------------------------------------------");
+        logger.debug("__________________________________________");
+        logger.debug("@@@ Found page entry: ");
+        logger.debug("__________________________________________");
+        logger.debug(pageDump.toString());
+        logger.debug("------------------------------------------");
         // END DEBUG
     }
@@ -174 +174 @@
         
     } catch (IOException ioe) {
-        error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
+        logger.error("@@@@@@@@@ Error reading in nutch txtdump file " + txtDumpFile, ioe);
     }
 
@@ -302 +302 @@
     
     
-    info("------------- " + this.siteID + " SITE STATS -----------");
-
-    info("SITE DOMAIN: " + this.domainOfSite);
-    info("Total number of web pages in site: " + pages.size());
-    info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
+    logger.info("------------- " + this.siteID + " SITE STATS -----------");
+
+    logger.info("SITE DOMAIN: " + this.domainOfSite);
+    logger.info("Total number of web pages in site: " + pages.size());
+    logger.info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
     
     if(pagesInMRI.size() > 0) {
-        info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
+        logger.info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
         for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
-        info(mriWebPageInfo.toString());
-        }
-    }
-
-    info("                      -----------                   ");
+        logger.info(mriWebPageInfo.toString());
+        }
+    }
+
+    logger.info("                      -----------                   ");
     if(pagesContainingMRI.size() > 0) {     
-        info("The following pages weren't detected as primarily being in Māori");
-        info("But still contained sentences detected as Māori");
+        logger.info("The following pages weren't detected as primarily being in Māori");
+        logger.info("But still contained sentences detected as Māori");
         for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
-        info(mriWebPageInfo.toString());
+        logger.info(mriWebPageInfo.toString());
         }
         
     } else {
-        info("No further pages detected as containing any sentences in MRI");      
-    }
-    info("                      -----------                   ");
+        logger.info("No further pages detected as containing any sentences in MRI");       
+    }
+    logger.info("                      -----------                   ");
     }
 
@@ -358 +358 @@
     
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-    System.err.println(msg);
-    logger.info(msg);
-    }
-    public static void debug(String msg) {
-    System.err.println(msg);
-    logger.debug(msg);
-    }
-    public static void warn(String msg) {
-    System.err.println(msg);
-    logger.warn(msg);
-    }
-    public static void error(String msg) {
-    System.err.println(msg);
-    logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-    logger.error(msg, e);
-    System.err.println("\n"+msg);
-    e.printStackTrace();
-    }
-    
+   
     public static void printUsage() {
-    info("Run this program as:");
-    info("\tNutchTextDumpProcessor <path to 'crawled' folder>");
+    System.err.println("Run this program as:");
+    System.err.println("\tNutchTextDumpProcessor <path to 'crawled' folder>");
     }
     
@@ -393 +372 @@
     File sitesDir = new File(args[0]);
     if(!sitesDir.exists() || !sitesDir.isDirectory()) {
-        error("Error: " + args[0] + " does not exist or is not a directory");
+        logger.error("Error: " + args[0] + " does not exist or is not a directory");
         return;
     }
@@ -430 +409 @@
             File txtDumpFile = new File(siteDir, "dump.txt");
             if(!txtDumpFile.exists()) {
-            error("Text dump file " + txtDumpFile + " did not exist");
+            logger.error("Text dump file " + txtDumpFile + " did not exist");
             continue;
             }
@@ -439 +418 @@
             String siteID = siteDir.getName();
             long lastModified = siteDir.lastModified();
-            debug("Found siteID: " + siteID);           
+            logger.debug("Found siteID: " + siteID);            
             NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(
                  webpagesCSVPrinter, mriSentencesCSVPrinter, mriTxtDetector,
@@ -458 +437 @@
         // can get an exception when instantiating CCWETProcessor instance
         // or with CSV file
-        error(e.getMessage(), e);
+        logger.error(e.getMessage(), e);
     }
     }

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java

@@ -38 +38 @@
         tuples.put("key", key.trim());
         //} else {
-        //debug("@@@@ no key for pageURL: " + pageURL);
+        //logger.debug("@@@@ no key for pageURL: " + pageURL);
         //}
         /*
         if(pageURL.contains(TEXT_END_MARKER)) {
-        debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
-        debug("+++++++++");
-        debug(unparsedPageDump);
-        debug("+++++++++");
+        logger.debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
+        logger.debug("+++++++++");
+        logger.debug(unparsedPageDump);
+        logger.debug("+++++++++");
         }
         */
@@ -87 +87 @@
             } else {
                 if(NutchTextDumpProcessor.DEBUG_MODE) {
-                error("No meta key for meta: " + line);
-                error(unparsedPageDump);
+                logger.error("No meta key for meta: " + line);
+                logger.error(unparsedPageDump);
                 }
             }
@@ -103 +103 @@
         
     } catch (IOException ioe) {
-        error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
+        logger.error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
     }
 
@@ -115 +115 @@
     public void debugTuples() {
     if(NutchTextDumpProcessor.DEBUG_MODE) {
-        debug("__________________________________________");
+        logger.debug("__________________________________________");
         for(Map.Entry<String, String> entry : tuples.entrySet()) {
         String key = entry.getKey();
         String value = entry.getValue();        
-        debug(key + " - " + value);
+        logger.debug(key + " - " + value);
         }
-        debug("__________________________________________");
+        logger.debug("__________________________________________");
     }
     }
@@ -164 +164 @@
     }
     
-    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
-    public static void info(String msg) {
-    System.err.println(msg);
-    logger.info(msg);
-    }
-    public static void debug(String msg) {
-    System.err.println(msg);
-    logger.debug(msg);
-    }
-    public static void warn(String msg) {
-    System.err.println(msg);
-    logger.warn(msg);
-    }
-    public static void error(String msg) {
-    System.err.println(msg);
-    logger.error(msg);
-    }
-    public static void error(String msg, Exception e) {
-    logger.error(msg, e);
-    System.err.println("\n"+msg);
-    e.printStackTrace();
-    }
-    
 }

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -77 +77 @@
 
     String fileID = inFile.getName();
-    //debug("*** Processing wetfile: " + fileID);           
+    //logger.debug("*** Processing wetfile: " + fileID);            
     fileID = fileID.substring(fileID.lastIndexOf("0")+1);
     if(fileID.startsWith(".")) { // took off too many zeroes, as happens with *000000.warc.wet
@@ -195 +195 @@
                   String recordURI, String record)
     {
-    info("CrawlID: CC-MAIN-" + this.crawlID
+    logger.info("CrawlID: CC-MAIN-" + this.crawlID
                + " WET #" + this.WETFileID
                + " record #" + recordID
                + " - contentLength: " + contentLength
                + " - lineCount: " + lineCount);
-    info("URI: " + recordURI);
-    //debug(record);
-    //info("--------------------------");
+    logger.info("URI: " + recordURI);
+    //logger.debug(record);
+    //logger.info("--------------------------");
 
     File parentFolder = null;
@@ -215 +215 @@
         else if(batchProcessor.isGreylisted(recordURI)) {
         parentFolder = batchProcessor.greyListedFolder;
-        debug("@@@GREYLISTED");
+        logger.debug("@@@GREYLISTED");
         }
         else { // url was only blacklisted
         parentFolder = batchProcessor.discardFolder;
-        debug("@@@DISCARDING - blacklisted");
+        logger.debug("@@@DISCARDING - blacklisted");
         }
     }
@@ -229 +229 @@
         else {
         parentFolder = batchProcessor.greyListedFolder;
-        debug("@@@GREYLISTED");
+        logger.debug("@@@GREYLISTED");
         }
     }
@@ -274 +274 @@
         if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
         parentFolder = batchProcessor.discardFolder;
-        debug("@@@DISCARDING - CAMELCASED CONTENTS");
+        logger.debug("@@@DISCARDING - CAMELCASED CONTENTS");
         }
         else*/
@@ -282 +282 @@
         if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
         parentFolder = batchProcessor.keepFolder;
-        debug("@@@KEEPING");
+        logger.debug("@@@KEEPING");
         }
     }
@@ -289 +289 @@
     if(parentFolder == null) {
         parentFolder = batchProcessor.discardFolder;
-        debug("@@@DISCARDING");
+        logger.debug("@@@DISCARDING");
     }
 
@@ -301 +301 @@
         }
     } catch(Exception e) {
-        debug("Unable to write URL");
+        logger.debug("Unable to write URL");
         e.printStackTrace();
     }
    
-    debug("--------------------------");
+    logger.debug("--------------------------");
 
     // outFilename will look something like YYYY-##-####
@@ -319 +319 @@
     } catch(IOException ioe) {
         ioe.printStackTrace();
-        error("@@@@@@@@@ Error writing to file " + outFile, ioe);
+        logger.error("@@@@@@@@@ Error writing to file " + outFile, ioe);
     }
     }
 
 
-    public void info(String msg) {
-    System.err.println(msg);
-    logger.info(msg);
-    }
-    public void debug(String msg) {
-    System.err.println(msg);
-    logger.debug(msg);
-    }
-    public void warn(String msg) {
-    System.err.println(msg);
-    logger.warn(msg);
-    }
-    public void error(String msg) {
-    System.err.println(msg);
-    logger.error(msg);
-    }
-    public void error(String msg, Exception e) {
-    logger.error(msg, e);
-    System.err.println("\n"+msg);
-    e.printStackTrace();
-    }
 }