Changeset 33573 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

Timestamp:

2019-10-16T21:39:56+13:00 (5 years ago)

Author:

ak19

Message:

Forgot to document that spaces were also allowed as separator in the input of crawl-site ids to the batchcrawl.sh script

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java (modified) (5 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -761 +761 @@
         // Finally, we can process this WETFile's records into the keep and discard pile
         wetFileCount++;
-        logger.debug("Off to process " + WETFile);
+        debug("Off to process " + WETFile);
         String crawlID = ccrawlWETFileDir.getName(); // something like CC-MAIN-YYYY-##-wet-files
         crawlID = crawlID.substring("CC-MAIN-".length(), crawlID.indexOf("-wet-files")); // YYYY-##     
@@ -794 +794 @@
     public static void error(String msg, Exception e) {
     logger.error(msg, e);
-    System.err.println(msg);
+    System.err.println("\n"+msg);
     e.printStackTrace();
     }
@@ -810 +810 @@
     public boolean accept(File dir, String name) {
         if(name.endsWith(".warc.wet")) {
-        logger.debug("Will include " + name + " for processing.");
+        debug("Will include " + name + " for processing.");
         return true;
         }
@@ -818 +818 @@
         File unzippedVersion = new File(dir, nameWithoutGZext);
         if(unzippedVersion.exists()) {
-            logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
-            logger.debug("Skipping " + name);
+            debug("--- Unzipped version " + unzippedVersion + " exists.");
+            debug("Skipping " + name);
             return false; // don't count gzipped version if unzipped version exists.
         }
         else {
-            logger.debug("Only zipped version " + name + " exists.");
+            debug("Only zipped version " + name + " exists.");
             return true; // No unzipped version, so have to work with gzipped version
         }
@@ -829 +829 @@
 
         // we're not even interested in any other file extensions
-        logger.debug("Not a WET file. Skipping " + name);
+        debug("Not a WET file. Skipping " + name);
         return false;
     }