Index: /gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt	(revision 33568)
+++ /gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt	(revision 33569)
@@ -54,7 +54,15 @@
 # NOT TOP SITES, BUT SITES WE INSPECTED AND WANT TO CONTROL SIMILARLY TO TOP SITES
 00.gs,SINGLEPAGE
-
-# May be a large site
+# May be a large site with only seedURLs of real relevance
 topographic-map.com,SINGLEPAGE
+ami-media.net,SINGLEPAGE
+# 2 pages of declarations of human rights in Maori, rest in other languages
+anitra.net,SINGLEPAGE
+# special case
+mi.centr-zashity.ru,SINGLEPAGE
+
+# TOP SITE BUT NOT TOP 500
+www.tumblr.com,SINGLEPAGE
+
 
 # TOP SITES
@@ -74,4 +82,5 @@
 # The page's containing folder is whitelisted in case the photos are there.
 korora.econ.yale.edu,SINGLEPAGE
+
 
 000webhost.com
Index: /gs3-extensions/maori-lang-detection/conf/url-blacklist-filter.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/url-blacklist-filter.txt	(revision 33568)
+++ /gs3-extensions/maori-lang-detection/conf/url-blacklist-filter.txt	(revision 33569)
@@ -28,4 +28,5 @@
 zh-min-nan.wiktionary.org
 
+######
 # unwanted domains
 .video-chat.
@@ -69,4 +70,36 @@
 acba.osb-land.com
 
+
+# just get rid of any URL containing "livejasmin"
+## livejasmin
+# Actually: do that in the code (CCWETProcessor) with a log message,
+# since we actually need to get rid of any sites in entirety that contain
+# any url with the string "livejasmin"
+# So run the program once, check the log for messages mentioning "additional"
+# adult sites found and add their domains in here.
+anigma-beauty.com
+adultfeet.com
+atopian.org
+bellydancingvideo.net
+bmmodelsagency.com
+brucknergallery.com
+fuckvidz.org
+photobattle.net
+votekat.info
+
+# Similar to above, the following contained the string "jasmin" in the URL
+teenycuties.com
+a.tiles.mapbox.com
+blazingteens.net
+redtubeporn.info
+osb-land.com
+totallyhotmales.com
+babeevents.com
+talkserver.de
+hehechat.org
+fetish-nights.com
+lesslove.com
+hebertsvideo.com
+
 # sounds like some pirating site
 ^http://pirateguides.com/
@@ -85,2 +118,5 @@
 # not sure about the domain name and/or full url seems like it belongs here
 abcutie.com
+
+# only had a single seedURL and it quickly redirected to an adult site
+apparactes.gq
Index: /gs3-extensions/maori-lang-detection/conf/url-greylist-filter.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/url-greylist-filter.txt	(revision 33568)
+++ /gs3-extensions/maori-lang-detection/conf/url-greylist-filter.txt	(revision 33569)
@@ -18,4 +18,6 @@
 abacre.com
 cn-huafu.net
+apteka.social
+
 
 # not product stores but autotranslated?
@@ -25,7 +27,9 @@
 1videosmusica.com
 256file.com
-7773033.ru
-abali.ru
-allbeautyone.ru
+# already in greylisting of all .ru 
+#7773033.ru
+#abali.ru
+#allbeautyone.ru
+aqualuz.org
 
 # if page doesn't load and can't be tested
@@ -33,6 +37,10 @@
 www.kiterewa.pl
 
-# license plate site?
-eba.com.ru
+
+
+# MANUALLY INSPECTED URLS AND ADDED TO GREYLIST
+
+# license plate site? - already in greylisting of all .ru
+#eba.com.ru
 
 # As per archive.org, there's just a photo on the defunct page at this site
@@ -42,2 +50,16 @@
 # seems to be Indonesian or Malaysian Bible rather than in Maori or any Polynesian language
 alkitab.life:2022
+
+# appears defunct
+alixira.com
+
+# single seedURL was not a page in Maori, but global languages.
+# And the rest of the domain appears to be in English
+anglican.org
+
+
+### TLDs that we greylist - any exceptions will be in the whitelist
+# Our list of .ru and .pl domains were not relevant
+.ru/
+.pl/
+.tk/
Index: /gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt	(revision 33568)
+++ /gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt	(revision 33569)
@@ -10,2 +10,7 @@
 # some particular other urls on yale.edu
 http://korora.econ.yale.edu/phillips/archive/hauraki.htm
+
+# We've added .ru$ sites to the blacklist, but the following 
+# Russian website contains actual Maori language content
+http://www.krassotkin.ru/sites/prayer.su/maori/
+https://mi.centr-zashity.ru/
Index: /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh	(revision 33568)
+++ /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh	(revision 33569)
@@ -74,5 +74,11 @@
      	echo "CRAWL FAILED." 2>&1 | tee -a ${siteDir}log.out
     fi
-    
+
+
+    # move the peripheral crawl products (the log.out and UNFINISHED files)
+    # from the input to the output folder. This way we can re-run the crawl and
+    # the original output will still have been preserved
+    mv ${siteDir}log.out $outputDir/$crawlId/log.out
+    mv ${siteDir}UNFINISHED $outputDir/$crawlId/UNFINISHED
 }
 
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33568)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33569)
@@ -150,11 +150,9 @@
 	    
 	} catch(Exception e) {
-	    System.err.println("Exception attempting to read properties from config.properties.");
-	    logger.error("Exception attempting to read properties from config.properties.");
-	    e.printStackTrace();
+	    error("Exception attempting to read properties from config.properties.", e);
 	}
 
 	if(configProperties.size() == 0) {
-	    System.err.println("*** Warning: no values read into config properties. Using defaults.");
+	    warn("*** Warning: no values read into config properties. Using defaults.");
 	}
 	
@@ -192,18 +190,18 @@
 
 	// prepare our blacklist, greylist (for inspection) and whitelist
-	System.err.println("Loading blacklist.");
+	info("Loading blacklist.");
 	blackList = new HashMap<String, Integer>();
 	initURLFilterList(blackList, "url-blacklist-filter.txt");
 	
-	System.err.println("Loading greylist.");
+	info("Loading greylist.");
 	greyList = new HashMap<String, Integer>();
 	initURLFilterList(greyList, "url-greylist-filter.txt");
 	
-	System.err.println("Loading whitelist.");
+	info("Loading whitelist.");
 	whiteList = new HashMap<String, Integer>();
 	initURLFilterList(whiteList, "url-whitelist-filter.txt");
 
 	// Create the map of topSites
-	System.err.println("Loading map of topsites with regex of allowable url patterns for each topsite.");
+	info("Loading map of topsites with regex of allowable url patterns for each topsite.");
 	topSitesMap = new HashMap<String, String>();
 	
@@ -228,15 +226,12 @@
 		topSitesMap.put(topsite, allowed_url_pattern);
 
-		//System.err.println("@@@@ topsite: " + topsite + " - " + allowed_url_pattern);
+		//debug("@@@@ topsite: " + topsite + " - " + allowed_url_pattern);
 		
 	    }
 	} catch(Exception e) {
-	    e.printStackTrace();
-	    System.err.println("\n@@@@@@@@@ Error attempting to parse CSV format of text file " + topSitesCSVData);
-	}
-	
-	
+	    error("@@@@@@@@@ Error attempting to parse CSV format of text file " + topSitesCSVData, e);
+	}	
  
-	//System.err.println("Prematurely terminating for testing purposes.");
+	//debug("Prematurely terminating for testing purposes.");
 	//System.exit(-1);
     }
@@ -309,5 +304,5 @@
 		    urlsSet = new TreeSet<String>();
 		    urlsSet.add(url);
-		    domainsToURLsMap.put(domainWithProtocol, urlsSet);
+		    domainsToURLsMap.put(domainWithProtocol, urlsSet);		    
 		} else {
 		    urlsSet = domainsToURLsMap.get(domainWithProtocol);
@@ -317,6 +312,5 @@
 	    }
 	} catch (IOException ioe) {
-	    ioe.printStackTrace();
-	    System.err.println("\n@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile);
+	    error("@@@@@@@@@ Error reading in urls from file " + this.keepURLsFile, ioe);
 	}
 
@@ -356,7 +350,7 @@
 	String value = topSitesMap.get("wikipedia.org");
 	if(value == null) {
-	    System.err.println("### wikipedia.org had null value");
+	    debug("### wikipedia.org had null value");
 	} else {
-	    System.err.println("### wikipedia.org had value: " + value);
+	    debug("### wikipedia.org had value: " + value);
 	} // DEBUG
 	    */
@@ -370,6 +364,6 @@
 
 		/*if(domain.contains("docs.google.com")) {
-		    System.err.println("domain with protocol: " + domainWithProtocol);
-		    System.err.println("domain: " + domain);
+		    debug("domain with protocol: " + domainWithProtocol);
+		    debug("domain: " + domain);
 		    }*/
 		
@@ -495,6 +489,5 @@
 		    
 		} catch (IOException ioe) {
-		    ioe.printStackTrace();
-		    System.err.println("\n@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile);
+		    error("@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile, ioe);
 		}
 		
@@ -502,19 +495,17 @@
 
 	} catch (IOException ioe) {
-	    ioe.printStackTrace();
-	    System.err.println("\n@@@@@@@@@ Error writing to one of: ");
-	    System.err.println("\t" + seedURLsFile);
-	    System.err.println("\t" + urlFilterFile);
-	    System.err.println("\t" + domainURLsFile);	
-	    System.err.println("\t" + topSiteMatchesFile);    
+	    error("\n@@@@@@@@@ Error writing to one of:\n\t" + seedURLsFile
+			       + "\n\t" + urlFilterFile
+			       + "\n\t" + domainURLsFile
+			       + "\n\t" + topSiteMatchesFile, ioe);
 	}	
 
 	/*
 	// BEGIN DEBUG
-	System.err.println("@@@@ TopSitesMap contains: ");
+	debug("@@@@ TopSitesMap contains: ");
 	for(Map.Entry<String, String> entry : topSitesMap.entrySet()) {
 	    String topSite = entry.getKey();
 	    String urlPattern = entry.getValue();	    
-	    System.err.println(topSite + " - " + urlPattern);
+	    debug(topSite + " - " + urlPattern);
 	} // END DEBUG
 	*/
@@ -587,5 +578,5 @@
 	    if(allowed_url_pattern != null) { // if topSitesMap.containsKey(domain);
 		// there's an entry for the URL in the topSitesMap
-		System.err.println("##### A top site matches URL domain " + domain);
+		debug("##### A top site matches URL domain " + domain);
 
 		// if we're dealing with SUBDOMAIN-COPY, then the fullSeedDomain, with or without
@@ -648,5 +639,14 @@
      */
     public boolean isBlacklisted(String url) {
-	return isListedInFilterList(blackList, url);
+	boolean isBlackListed = isListedInFilterList(blackList, url);
+
+	// if any portion of the URL contains the word "livejasmin", or even "jasmin" actually,
+	// then it's an adult site, so blacklist the entire domain if it wasn't already blacklisted
+	String domainWithoutProtocol = getDomainForURL(url, false); // remove protocol
+	if(!isBlackListed && url.contains("jasmin")) {
+	    warn("### Blacklisting additional domain (likely an adult site): " + domainWithoutProtocol);
+	    blackList.put(domainWithoutProtocol, LIST_ENTRY_CONTAINS);
+	}
+	return isBlackListed;
     }
     
@@ -680,5 +680,5 @@
 	// if filterListFilename does not exist in the conf folder, just return
 	if(MY_CLASSLOADER.getResource(filterListFilename) == null) {
-	    System.err.println(filterListFilename + " does not exist");
+	    warn("Filter list filename: " + filterListFilename + " does not exist");
 	    return;	    
 	}
@@ -702,19 +702,19 @@
 		    filter = filter.substring(1);
 		    list.put(filter, LIST_ENTRY_STARTSWITH);
-		    System.err.println("Match filter startswith: " + filter);
+		    //debug("Match filter startswith: " + filter);
 		}
 		else if(filter.endsWith("$")) {
 		    filter = filter.substring(0, filter.length()-1);
 		    list.put(filter, LIST_ENTRY_ENDSWITH);
+		    //debug("@@@ Match filter endswith: " + filter);
 		}
 		else {
 		    list.put(filter, LIST_ENTRY_CONTAINS);
 		}
-		//System.err.println("Got filter: " + filter);
+		//debug("Got filter: " + filter);
 	    }
 		
 	} catch (IOException ioe) {
-	    ioe.printStackTrace();
-	    System.err.println("\n@@@@@@@@@ Error reading into map from file " + filterListFilename);
+	    error("@@@@@@@@@ Error reading into map from file " + filterListFilename, ioe);
 	}
 	
@@ -739,5 +739,5 @@
 	for(int i = 0; i < WETFiles.length; i++) {
 	    File WETFile = WETFiles[i];	    
-	    logger.debug("Processing WETfile: " + WETFile);
+	    debug("Processing WETfile: " + WETFile);
 
 	    // Any .gz files listed means they haven't been unzipped yet. So unzip.
@@ -754,7 +754,6 @@
 	    // Check the unzipped WETFile exists	    
 
-	    if(!WETFile.exists() || !WETFile.isFile()) {
-		System.err.println("Error: " + WETFile + " does not exist (failure to unzip?)");
-		logger.error("Error: " + WETFile + " does not exist (failure to unzip?)");
+	    if(!WETFile.exists() || !WETFile.isFile()) {		
+		error("Error: " + WETFile + " does not exist (failure to unzip?)");
 		return;
 	    }
@@ -777,7 +776,29 @@
 
     // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
+    public static void info(String msg) {
+	System.err.println(msg);
+	logger.info(msg);
+    }
+    public static void debug(String msg) {
+	System.err.println(msg);
+	logger.debug(msg);
+    }
+    public static void warn(String msg) {
+	System.err.println(msg);
+	logger.warn(msg);
+    }
+    public static void error(String msg) {
+	System.err.println(msg);
+	logger.error(msg);
+    }
+    public static void error(String msg, Exception e) {
+	logger.error(msg, e);
+	System.err.println(msg);
+	e.printStackTrace();
+    }
+    
     public static void printUsage() {
-	System.err.println("Run this program as:");
-	System.err.println("\tWetProcessor <folder containing wet(.gz) files> <output folder path>");	
+	info("Run this program as:");
+	info("\tWetProcessor <folder containing wet(.gz) files> <output folder path>");	
     }
 
@@ -824,5 +845,5 @@
 	    }
 	    else {
-		System.err.println("File " + f + " is not a directory");
+		info("File " + f + " is not a directory");
 	    }
 	    return false;				   
@@ -838,5 +859,5 @@
 	File commoncrawlDir = new File(args[0]);
 	if(!commoncrawlDir.exists() || !commoncrawlDir.isDirectory()) {
-	    System.out.println("Error: " + args[0] + " does not exist or is not a directory");
+	    error("Error: " + args[0] + " does not exist or is not a directory");
 	    return;
 	}
@@ -844,5 +865,5 @@
 	File outFolder = new File(args[1]);
 	if(!outFolder.exists() || !outFolder.isDirectory()) {
-	    System.out.println("Error: " + args[1] + " does not exist or is not a directory.");
+	    error("Error: " + args[1] + " does not exist or is not a directory.");
 	    return;
 	}	
@@ -855,5 +876,5 @@
 	for(int i = 0; i < ccrawlFolders.length; i++) {
 	    File ccrawlFolder = ccrawlFolders[i];
-	    System.err.println("About to process commoncrawl WET files folder: " + ccrawlFolder);
+	    info("About to process commoncrawl WET files folder: " + ccrawlFolder);
 	    ccWETFilesProcessor.processAllWETFilesOfCrawl(ccrawlFolder);	    
 	}
@@ -869,13 +890,12 @@
 	ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile, domainURLsFile, topSitesMatchedFile);
 
-	System.out.println("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
-
-	System.out.println("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
+	info("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
+
+	info("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
 	
 	
     } catch(Exception e) {
 	// can get an exception when instantiating CCWETProcessor instance
-	e.printStackTrace();
-	System.err.println(e.getMessage());
+	error(e.getMessage(), e);
     }
 	
