Changeset 33573 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

Timestamp:

2019-10-16T21:39:56+13:00 (5 years ago)

Author:

ak19

Message:

Forgot to document that spaces were also allowed as separator in the input of crawl-site ids to the batchcrawl.sh script

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -77 +77 @@
 
     String fileID = inFile.getName();
-    //System.err.println("*** Processing wetfile: " + fileID);          
+    //debug("*** Processing wetfile: " + fileID);           
     fileID = fileID.substring(fileID.lastIndexOf("0")+1);
     if(fileID.startsWith(".")) { // took off too many zeroes, as happens with *000000.warc.wet
@@ -195 +195 @@
                   String recordURI, String record)
     {
-    System.err.println("CrawlID: CC-MAIN-" + this.crawlID
+    info("CrawlID: CC-MAIN-" + this.crawlID
                + " WET #" + this.WETFileID
                + " record #" + recordID
                + " - contentLength: " + contentLength
                + " - lineCount: " + lineCount);
-    System.err.println("URI: " + recordURI);
-    //System.err.println(record);
-    //System.err.println("--------------------------");
+    info("URI: " + recordURI);
+    //debug(record);
+    //info("--------------------------");
 
     File parentFolder = null;
@@ -215 +215 @@
         else if(batchProcessor.isGreylisted(recordURI)) {
         parentFolder = batchProcessor.greyListedFolder;
-        System.err.println("@@@GREYLISTED");
+        debug("@@@GREYLISTED");
         }
         else { // url was only blacklisted
         parentFolder = batchProcessor.discardFolder;
-        System.err.println("@@@DISCARDING - blacklisted");
+        debug("@@@DISCARDING - blacklisted");
         }
     }
@@ -229 +229 @@
         else {
         parentFolder = batchProcessor.greyListedFolder;
-        System.err.println("@@@GREYLISTED");
+        debug("@@@GREYLISTED");
         }
     }
@@ -274 +274 @@
         if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
         parentFolder = batchProcessor.discardFolder;
-        System.err.println("@@@DISCARDING - CAMELCASED CONTENTS");
+        debug("@@@DISCARDING - CAMELCASED CONTENTS");
         }
         else*/
@@ -282 +282 @@
         if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
         parentFolder = batchProcessor.keepFolder;
-        System.err.println("@@@KEEPING");
+        debug("@@@KEEPING");
         }
     }
@@ -289 +289 @@
     if(parentFolder == null) {
         parentFolder = batchProcessor.discardFolder;
-        System.err.println("@@@DISCARDING");
+        debug("@@@DISCARDING");
     }
 
@@ -301 +301 @@
         }
     } catch(Exception e) {
-        System.err.println("Unable to write URL");
+        debug("Unable to write URL");
         e.printStackTrace();
     }
    
-    System.err.println("--------------------------");
+    debug("--------------------------");
 
     // outFilename will look something like YYYY-##-####
@@ -319 +319 @@
     } catch(IOException ioe) {
         ioe.printStackTrace();
-        System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
-    }
+        error("@@@@@@@@@ Error writing to file " + outFile, ioe);
+    }
+    }
+
+
+    public void info(String msg) {
+    System.err.println(msg);
+    logger.info(msg);
+    }
+    public void debug(String msg) {
+    System.err.println(msg);
+    logger.debug(msg);
+    }
+    public void warn(String msg) {
+    System.err.println(msg);
+    logger.warn(msg);
+    }
+    public void error(String msg) {
+    System.err.println(msg);
+    logger.error(msg);
+    }
+    public void error(String msg, Exception e) {
+    logger.error(msg, e);
+    System.err.println("\n"+msg);
+    e.printStackTrace();
     }
 }