Changeset 33615 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

Timestamp:

2019-10-31T20:03:55+13:00 (4 years ago)

Author:

ak19

Message:

1. Worked out how to configure log4j to log both to console and logfile, so modified the existing laboured code to use this better way. 2. Added some Mongodb links under MoreReading.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -77 +77 @@
 
     String fileID = inFile.getName();
-    //debug("*** Processing wetfile: " + fileID);           
+    //logger.debug("*** Processing wetfile: " + fileID);            
     fileID = fileID.substring(fileID.lastIndexOf("0")+1);
     if(fileID.startsWith(".")) { // took off too many zeroes, as happens with *000000.warc.wet
@@ -195 +195 @@
                   String recordURI, String record)
     {
-    info("CrawlID: CC-MAIN-" + this.crawlID
+    logger.info("CrawlID: CC-MAIN-" + this.crawlID
                + " WET #" + this.WETFileID
                + " record #" + recordID
                + " - contentLength: " + contentLength
                + " - lineCount: " + lineCount);
-    info("URI: " + recordURI);
-    //debug(record);
-    //info("--------------------------");
+    logger.info("URI: " + recordURI);
+    //logger.debug(record);
+    //logger.info("--------------------------");
 
     File parentFolder = null;
@@ -215 +215 @@
         else if(batchProcessor.isGreylisted(recordURI)) {
         parentFolder = batchProcessor.greyListedFolder;
-        debug("@@@GREYLISTED");
+        logger.debug("@@@GREYLISTED");
         }
         else { // url was only blacklisted
         parentFolder = batchProcessor.discardFolder;
-        debug("@@@DISCARDING - blacklisted");
+        logger.debug("@@@DISCARDING - blacklisted");
         }
     }
@@ -229 +229 @@
         else {
         parentFolder = batchProcessor.greyListedFolder;
-        debug("@@@GREYLISTED");
+        logger.debug("@@@GREYLISTED");
         }
     }
@@ -274 +274 @@
         if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
         parentFolder = batchProcessor.discardFolder;
-        debug("@@@DISCARDING - CAMELCASED CONTENTS");
+        logger.debug("@@@DISCARDING - CAMELCASED CONTENTS");
         }
         else*/
@@ -282 +282 @@
         if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
         parentFolder = batchProcessor.keepFolder;
-        debug("@@@KEEPING");
+        logger.debug("@@@KEEPING");
         }
     }
@@ -289 +289 @@
     if(parentFolder == null) {
         parentFolder = batchProcessor.discardFolder;
-        debug("@@@DISCARDING");
+        logger.debug("@@@DISCARDING");
     }
 
@@ -301 +301 @@
         }
     } catch(Exception e) {
-        debug("Unable to write URL");
+        logger.debug("Unable to write URL");
         e.printStackTrace();
     }
    
-    debug("--------------------------");
+    logger.debug("--------------------------");
 
     // outFilename will look something like YYYY-##-####
@@ -319 +319 @@
     } catch(IOException ioe) {
         ioe.printStackTrace();
-        error("@@@@@@@@@ Error writing to file " + outFile, ioe);
+        logger.error("@@@@@@@@@ Error writing to file " + outFile, ioe);
     }
     }
 
 
-    public void info(String msg) {
-    System.err.println(msg);
-    logger.info(msg);
-    }
-    public void debug(String msg) {
-    System.err.println(msg);
-    logger.debug(msg);
-    }
-    public void warn(String msg) {
-    System.err.println(msg);
-    logger.warn(msg);
-    }
-    public void error(String msg) {
-    System.err.println(msg);
-    logger.error(msg);
-    }
-    public void error(String msg, Exception e) {
-    logger.error(msg, e);
-    System.err.println("\n"+msg);
-    e.printStackTrace();
-    }
 }