Changeset 33573 for gs3-extensions

Timestamp:

2019-10-16T21:39:56+13:00 (5 years ago)

Author:

ak19

Message:

Forgot to document that spaces were also allowed as separator in the input of crawl-site ids to the batchcrawl.sh script

Location:

gs3-extensions/maori-lang-detection

Files:

• hdfs-cc-work/scripts/batchcrawl.sh (modified) (2 diffs)
• src/org/greenstone/atea/CCWETProcessor.java (modified) (5 diffs)
• src/org/greenstone/atea/WETProcessor.java (modified) (9 diffs)

gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh

@@ -165 +165 @@
     echo "  $0 -all|<ids>"
     echo "    where an id is a folder name in to_crawl/sites"
-    echo "    and ids can be a comma separated list of"
+    echo "    and ids can be a comma or space separated list of"
     echo "    individual ids and/or ranges"
     echo "  Examples:"
-    echo "    $0 00008-00022,00025,00026,00028-00034"
+    echo "    $0 00008-00022,00025,00026,00028-00034 00050"
     echo "    $0 -all"
     echo "    $0 00312"
@@ -189 +189 @@
     fi
     
-    # works - split args on comma
+    # works - split args on comma or space
     # https://stackoverflow.com/questions/918886/how-do-i-split-a-string-on-a-delimiter-in-bash
     IFS=', ' read -ra IDS <<< "$args"

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -761 +761 @@
         // Finally, we can process this WETFile's records into the keep and discard pile
         wetFileCount++;
-        logger.debug("Off to process " + WETFile);
+        debug("Off to process " + WETFile);
         String crawlID = ccrawlWETFileDir.getName(); // something like CC-MAIN-YYYY-##-wet-files
         crawlID = crawlID.substring("CC-MAIN-".length(), crawlID.indexOf("-wet-files")); // YYYY-##     
@@ -794 +794 @@
     public static void error(String msg, Exception e) {
     logger.error(msg, e);
-    System.err.println(msg);
+    System.err.println("\n"+msg);
     e.printStackTrace();
     }
@@ -810 +810 @@
     public boolean accept(File dir, String name) {
         if(name.endsWith(".warc.wet")) {
-        logger.debug("Will include " + name + " for processing.");
+        debug("Will include " + name + " for processing.");
         return true;
         }
@@ -818 +818 @@
         File unzippedVersion = new File(dir, nameWithoutGZext);
         if(unzippedVersion.exists()) {
-            logger.debug("--- Unzipped version " + unzippedVersion + " exists.");
-            logger.debug("Skipping " + name);
+            debug("--- Unzipped version " + unzippedVersion + " exists.");
+            debug("Skipping " + name);
             return false; // don't count gzipped version if unzipped version exists.
         }
         else {
-            logger.debug("Only zipped version " + name + " exists.");
+            debug("Only zipped version " + name + " exists.");
             return true; // No unzipped version, so have to work with gzipped version
         }
@@ -829 +829 @@
 
         // we're not even interested in any other file extensions
-        logger.debug("Not a WET file. Skipping " + name);
+        debug("Not a WET file. Skipping " + name);
         return false;
     }

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -77 +77 @@
 
     String fileID = inFile.getName();
-    //System.err.println("*** Processing wetfile: " + fileID);          
+    //debug("*** Processing wetfile: " + fileID);           
     fileID = fileID.substring(fileID.lastIndexOf("0")+1);
     if(fileID.startsWith(".")) { // took off too many zeroes, as happens with *000000.warc.wet
@@ -195 +195 @@
                   String recordURI, String record)
     {
-    System.err.println("CrawlID: CC-MAIN-" + this.crawlID
+    info("CrawlID: CC-MAIN-" + this.crawlID
                + " WET #" + this.WETFileID
                + " record #" + recordID
                + " - contentLength: " + contentLength
                + " - lineCount: " + lineCount);
-    System.err.println("URI: " + recordURI);
-    //System.err.println(record);
-    //System.err.println("--------------------------");
+    info("URI: " + recordURI);
+    //debug(record);
+    //info("--------------------------");
 
     File parentFolder = null;
@@ -215 +215 @@
         else if(batchProcessor.isGreylisted(recordURI)) {
         parentFolder = batchProcessor.greyListedFolder;
-        System.err.println("@@@GREYLISTED");
+        debug("@@@GREYLISTED");
         }
         else { // url was only blacklisted
         parentFolder = batchProcessor.discardFolder;
-        System.err.println("@@@DISCARDING - blacklisted");
+        debug("@@@DISCARDING - blacklisted");
         }
     }
@@ -229 +229 @@
         else {
         parentFolder = batchProcessor.greyListedFolder;
-        System.err.println("@@@GREYLISTED");
+        debug("@@@GREYLISTED");
         }
     }
@@ -274 +274 @@
         if(numCamelCaseWords >= batchProcessor.MAX_WORDS_CAMELCASE) {
         parentFolder = batchProcessor.discardFolder;
-        System.err.println("@@@DISCARDING - CAMELCASED CONTENTS");
+        debug("@@@DISCARDING - CAMELCASED CONTENTS");
         }
         else*/
@@ -282 +282 @@
         if(validWordCount >= batchProcessor.MIN_NUM_WORDS) { // otherwise, keep anything with a sufficient number of valid words
         parentFolder = batchProcessor.keepFolder;
-        System.err.println("@@@KEEPING");
+        debug("@@@KEEPING");
         }
     }
@@ -289 +289 @@
     if(parentFolder == null) {
         parentFolder = batchProcessor.discardFolder;
-        System.err.println("@@@DISCARDING");
+        debug("@@@DISCARDING");
     }
 
@@ -301 +301 @@
         }
     } catch(Exception e) {
-        System.err.println("Unable to write URL");
+        debug("Unable to write URL");
         e.printStackTrace();
     }
    
-    System.err.println("--------------------------");
+    debug("--------------------------");
 
     // outFilename will look something like YYYY-##-####
@@ -319 +319 @@
     } catch(IOException ioe) {
         ioe.printStackTrace();
-        System.err.println("\n@@@@@@@@@ Error writing to file " + outFile);
-    }
+        error("@@@@@@@@@ Error writing to file " + outFile, ioe);
+    }
+    }
+
+
+    public void info(String msg) {
+    System.err.println(msg);
+    logger.info(msg);
+    }
+    public void debug(String msg) {
+    System.err.println(msg);
+    logger.debug(msg);
+    }
+    public void warn(String msg) {
+    System.err.println(msg);
+    logger.warn(msg);
+    }
+    public void error(String msg) {
+    System.err.println(msg);
+    logger.error(msg);
+    }
+    public void error(String msg, Exception e) {
+    logger.error(msg, e);
+    System.err.println("\n"+msg);
+    e.printStackTrace();
     }
 }