Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33559)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33560)
@@ -65,4 +65,17 @@
     public final int MIN_NUM_WORDS;
     public final int MAX_WORDS_CAMELCASE;
+
+    // constants for the possible fixed values in sites-too-big-to-exhaustively-crawl.txt file
+    public final String SUBDOMAIN_COPY = "SUBDOMAIN-COPY";
+    public final String SINGLEPAGE = "SINGLEPAGE";
+    
+    /**
+     * Characters that need escaping if used as a string literal in a regex
+     * https://stackoverflow.com/questions/399078/what-special-characters-must-be-escaped-in-regular-expressions
+     * https://www.regular-expressions.info/refcharacters.html
+    */
+    //public final String[] ESCAPE_CHARS_FOR_RE = [".", "^", "$", "*", "+", "?", "(", ")", "[", "{", "\\", "|"];
+    // put the \\ at start so we don't the escape character for chars escaped earlier
+    public final String ESCAPE_CHARS_FOR_RE = "\\.^$*+?()[{|";
     
     private Properties configProperties = new Properties();
@@ -220,13 +233,31 @@
      * This retains any www. or subdomain prefix.
      */
-    private String getDomainForURL(String url) {
-	int startIndex = url.indexOf("//"); // http:// or https:// prefix
+    private String getDomainForURL(String url, boolean withProtocol) {
+	int startIndex = startIndex = url.indexOf("//"); // for http:// or https:// prefix
 	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+	// the keep the URL around in case param withProtocol=true
+	String protocol = (startIndex == -1) ? "" : url.substring(0, startIndex);
+	
 	String domain = url.substring(startIndex);
 	int endIndex = domain.indexOf("/");
 	if(endIndex == -1) endIndex = domain.length();
 	domain = domain.substring(0, endIndex);
+
+	if(withProtocol) {
+	    // now that we have the domain (everything to the first / when there is no protocol)
+	    // can glue the protocol back on
+	    domain = protocol + domain;
+	}
 	
 	return domain;
+    }
+
+    /** Utility function to help escape regex characters in URL to go into regex-urlfilter.txt */
+    private String escapeStringForRegex(String str) {	
+	for(int i = 0; i < ESCAPE_CHARS_FOR_RE.length(); i++) {
+	    char c = ESCAPE_CHARS_FOR_RE.charAt(i);
+	    str = str.replace(Character.toString(c), "\\"+c);
+	}
+	return str;	
     }
     
@@ -238,15 +269,14 @@
     public void createSeedURLsFiles(File seedURLsFile, File urlFilterFile,
 				    File domainURLsFile, File topSiteMatchesFile) {
-	// Maintain Sets of unique domains and urls
+	// Maintain a Map of unique domains mapped to seed urls at that domain
 	// TreeSet: by default, "the elements are ordered using their natural ordering"
 	// (or by a Comparator provided at set creation time).
 	// Whereas HashSet doesn't guarantee ordering.
 	// So we get alphabetic sorting for free. And guaranteed log(n) for basic operations.
-
-	//Set<String> domainsSet = new TreeSet<String>();
-	//Set<String> urlsSet = new TreeSet<String>();
+	// Would be a similar distinction for Maps.
 	domainsToURLsMap = new TreeMap<String, Set<String>>();
-    
-	final String FILTER_REGEX_PREFIX = "+https?://([a-z0-9-]+\\.)*"; // https?://([a-z0-9-]+\.)* for nutch's regex-urlfilter.txt
+
+	final String PROTOCOL_REGEX_PREFIX = "+^https?://";
+	final String FILTER_REGEX_PREFIX = PROTOCOL_REGEX_PREFIX + "([a-z0-9-]+\\.)*"; // https?://([a-z0-9-]+\.)* for nutch's regex-urlfilter.txt
 	
 	try (
@@ -256,19 +286,18 @@
 	    // read a URL at a time from urlsFile
 	    String url = null;
-	    String domain = null;
+	    String domainWithProtocol = null;
 	    while((url = reader.readLine()) != null) { // readLine removes newline separator
 		
-		// work out domain. This retains any www. or subdomain prefix		
-		domain = getDomainForURL(url);
-
-		//urlsSet.add(url);
-		//domainsSet.add(domain);
+		// work out domain. This retains any www. or subdomain prefix
+		// passing true to further also retain the http(s) protocol
+		domainWithProtocol = getDomainForURL(url, true);
+
 		Set<String> urlsSet;
-		if(!domainsToURLsMap.containsKey(domain)) {
+		if(!domainsToURLsMap.containsKey(domainWithProtocol)) {
 		    urlsSet = new TreeSet<String>();
 		    urlsSet.add(url);
-		    domainsToURLsMap.put(domain, urlsSet);
+		    domainsToURLsMap.put(domainWithProtocol, urlsSet);
 		} else {
-		    urlsSet = domainsToURLsMap.get(domain);
+		    urlsSet = domainsToURLsMap.get(domainWithProtocol);
 		    urlsSet.add(url);
 		}
@@ -322,5 +351,11 @@
 	    
 	    while(domainIterator.hasNext()) {
-		String domain = domainIterator.next();
+		String domainWithProtocol = domainIterator.next();
+		int startIndex = domainWithProtocol.indexOf("//"); // http:// or https:// prefix
+		startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+		String domain = domainWithProtocol.substring(startIndex);
+		
+		System.err.println("domain with protocol: " + domainWithProtocol);
+		System.err.println("domain: " + domain);
 		
 		String allowedURLPatternRegex = isURLinTopSitesMap(domain);		
@@ -330,4 +365,5 @@
 		// domain and seedURLs to the topSiteMatchesFile.		
 		if(allowedURLPatternRegex != null && allowedURLPatternRegex.equals("")) {
+		    
 		    // topsite, but we don't (yet) know what portion can be crawled
 		    // Append the top site and url to a global/toplevel file that
@@ -335,5 +371,5 @@
 		    // won't go into any other file hereafter
 
-		    Set<String> urlsForDomainSet = domainsToURLsMap.get(domain);
+		    Set<String> urlsForDomainSet = domainsToURLsMap.get(domainWithProtocol);
 		    Iterator<String> urlIterator = urlsForDomainSet.iterator();
 		    while(urlIterator.hasNext()) {
@@ -353,5 +389,5 @@
 		
 		// write out the domain		
-		//seedURLsWriter.write(domain + "\n");
+		//seedURLsWriter.write(domainWithProtocol + "\n");
 		
 
@@ -367,4 +403,6 @@
 
 		    // write all sorted unique domains into global domains file
+		    // Using the domain withuot protocol since the global domains file is for
+		    // informational purposes
 		    domainURLsWriter.write(domain + "\n");
 
@@ -373,13 +411,13 @@
 		    // files (and write regexed domain into each sites/0000#/regex-urlfilter.txt)
 		    // If we ever run nutch on a single seedURLs listing containing
-		    // all seed pages to crawl sites from, the above two files will work for that.
-		    
+		    // all seed pages to crawl sites from, the above two files will work for that.  
 		    
 		    if(allowedURLPatternRegex == null) { // entire site can be crawled
-			siteURLsWriter.write(domain + "\n");
+			siteURLsWriter.write(domainWithProtocol + "\n");
 			
 			// Write out filter in the following form for a site, e.g. for nutch.apache.org:
 			// nutch.apache.org => +^https?://([a-z0-9-]+\.)*nutch\.apache\.org/
-			String regexed_domain = FILTER_REGEX_PREFIX + domain.replace(".", "\\.") + "/";		
+			String regexed_domain = FILTER_REGEX_PREFIX + escapeStringForRegex(domain) + "/";		
+			//String regexed_domain = FILTER_REGEX_PREFIX + domain.replace(".", "\\.") + "/";		
 			urlFilterWriter.write(regexed_domain + "\n"); //global file
 			siteRegexWriter.write(regexed_domain + "\n"); // site file
@@ -387,30 +425,37 @@
 		    else { // domain belongs to a top site where only portion of site can be crawled
 			
-			if(allowedURLPatternRegex.equals("COPY")) { // COPY existing domain as url-filter
-			    siteURLsWriter.write(domain + "\n");
+			if(allowedURLPatternRegex.equals(SUBDOMAIN_COPY)) { // COPY existing domain as url-filter
+			    siteURLsWriter.write(domainWithProtocol + "\n");
 			    // e.g. pinky.blogspot.com will add a filter for pinky.blogspot.com
 			    // and not for all of blogspot.com
 			    
-			   String regexed_domain = "+https?://"+domain.replace(".", "\\.") + "/";
-			   urlFilterWriter.write(regexed_domain + "\n");
-			   siteRegexWriter.write(regexed_domain + "\n");
-			   
-			} else if(allowedURLPatternRegex.equals("SINGLEPAGE")) {
+			    String regexed_domain = PROTOCOL_REGEX_PREFIX+escapeStringForRegex(domain) + "/";
+			    //String regexed_domain = PROTOCOL_REGEX_PREFIX+domain.replace(".", "\\.") + "/";
+			    urlFilterWriter.write(regexed_domain + "\n");
+			    siteRegexWriter.write(regexed_domain + "\n");
+			    
+			} else if(allowedURLPatternRegex.equals(SINGLEPAGE)) {
 			    // don't write out domain. We want individual pages
-			    //DON'T DO: siteURLsWriter.write(domain + "\n");
+			    //DON'T DO THIS HERE: siteURLsWriter.write(domainWithProtocol + "\n");
 			    
-			    // don't write out domain as a regex expression url filter
+			    // don't write out domain as a regex expression url filter either,
 			    // write out the individual seed urls for the domain instead
 			    // since we will only be downloading the single page
 			    
-			    Set<String> urlsForDomainSet = domainsToURLsMap.get(domain);
-			    for(String urlInDomain : urlsForDomainSet) {			    
-				String regexed_url = "+^"+urlInDomain.replace(".", "\\.");
+			    Set<String> urlsForDomainSet = domainsToURLsMap.get(domainWithProtocol);
+			    for(String urlInDomain : urlsForDomainSet) {
+				// don't append slash to end this time
+				String regexed_url = "+^"+escapeStringForRegex(urlInDomain);
+				//String regexed_url = "+^"+urlInDomain.replace(".", "\\.");
 				urlFilterWriter.write(regexed_url + "\n");
 				siteRegexWriter.write(regexed_url + "\n");
 			    }
 			} else { // allowedURLPatternRegex is a url-form - convert to regex
-			    String regexed_pattern = "+^https?://"+allowedURLPatternRegex.replace(".", "\\.");
-			    siteURLsWriter.write(domain + "\n");
+			    if(!allowedURLPatternRegex.endsWith("/")) {
+				allowedURLPatternRegex += "/";
+			    }
+			    String regexed_pattern = PROTOCOL_REGEX_PREFIX+escapeStringForRegex(allowedURLPatternRegex);
+			    //String regexed_pattern = PROTOCOL_REGEX_PREFIX+allowedURLPatternRegex.replace(".", "\\.");
+			    siteURLsWriter.write(domainWithProtocol + "\n");
 			    urlFilterWriter.write(regexed_pattern + "\n");
 			    siteRegexWriter.write(regexed_pattern + "\n");
@@ -420,16 +465,15 @@
 		    
 		    // next write out the urls for the domain into the sites/0000x/seedURLs.txt file
-		    // also write into the global seeds file
-		    // (with a tab prefixed to each url?)
-		    Set<String> urlsForDomainSet = domainsToURLsMap.get(domain);
+		    // also write into the global seeds file (with a tab prefixed to each?)
+		    Set<String> urlsForDomainSet = domainsToURLsMap.get(domainWithProtocol);
 		    Iterator<String> urlIterator = urlsForDomainSet.iterator();
 		    while(urlIterator.hasNext()) {
 			String url = urlIterator.next();
-			seedURLsWriter.write("\t" + url + "\n"); // global seedURLs file
-			siteURLsWriter.write("\t" + url + "\n");
+			seedURLsWriter.write(url + "\n"); // global seedURLs file
+			siteURLsWriter.write(url + "\n");
 		    }
 		} catch (IOException ioe) {
 		    ioe.printStackTrace();
-		    System.err.println("\n@@@@@@@@@ Error writing to " + siteSeedsFile + " or " + siteRegexFile);
+		    System.err.println("\n@@@@@@@@@ Error writing to one of:" + siteSeedsFile + " or " + siteRegexFile);
 		}
 		
@@ -438,22 +482,10 @@
 	} catch (IOException ioe) {
 	    ioe.printStackTrace();
-	    System.err.println("\n@@@@@@@@@ Error writing to " + seedURLsFile + " or " + urlFilterFile);
-	}
-	
-	// write out domains as regular expressions into "regex-urlfilter.txt" file
-	try (BufferedWriter urlFilterWriter = new BufferedWriter(new FileWriter(urlFilterFile))) {
-	    Set<String> domainsSet = domainsToURLsMap.keySet();	    
-	    Iterator<String> i = domainsSet.iterator();
-	    // nutch.apache.org => +^https?://([a-z0-9-]+\.)*nutch\.apache\.org/
-	    while(i.hasNext()) {
-		String domain = i.next();
-		domain = FILTER_REGEX_PREFIX + domain.replace(".", "\\.") + "/";		
-		urlFilterWriter.write(domain + "\n");
-	    }
-	    
-	} catch (IOException ioe) {
-	    ioe.printStackTrace();
-	    System.err.println("\n@@@@@@@@@ Error writing to " + urlFilterFile);
-	}
+	    System.err.println("\n@@@@@@@@@ Error writing to one of: ");
+	    System.err.println("\t" + seedURLsFile);
+	    System.err.println("\t" + urlFilterFile);
+	    System.err.println("\t" + domainURLsFile);	
+	    System.err.println("\t" + topSiteMatchesFile);    
+	}	
 
 	/*
@@ -475,15 +507,53 @@
 	return url;
     }
-    
+
+
+    /** 
+     * @return true when a seedURL's domain exactly matches a topsite such as blogspot.com,
+     * with or without www. prefix. This method tests for such as case as it would be dangerous
+     * to do a SUBDOMAIN-COPY on such a site and thereby crawl that entire domain.
+     */
+    private boolean isExactDomainMatch(String seedURLDomain, String domain) {
+	// check for an exact match as-is
+	if(seedURLDomain.equals(domain)) {
+	    return true;
+	}
+
+	// else check if with or without a www. prefix we have an exact match with domain
+	if(seedURLDomain.startsWith("www.")) {
+	    if(seedURLDomain.substring(4).equals(domain)) {
+		return true;
+	    }
+	} else {
+	    if(domain.equals("www."+seedURLDomain)) {
+		return true;
+	    }
+	}
+
+	return false;
+     }
+    
+
     /**
-     * Check if the domain of the url, either in its entirety or when stripped of www/subdomains,
-     * is in the list of top sites.
+     * Check if the domain of the seedurl, either in its entirety or when stripped of 
+     * www/subdomains, is in the list of top sites.
      * If it is, and the given url matches the regex for that topsite, then add the url to the
      * whitelist and a regex disallowing the rest of the topsite to the url regex filter file.
-     
-     */
-    private String isURLinTopSitesMap(String domain) {
+     * @param fullSeedDomain: domain of seedURL without the protocol. May include www. prefix.
+     * @return one of the following values:
+     *  - This function returns null if the seedURL's domain does not match any of the topsites.
+     *  - The empty String is returned if the seedURL's domain matched a topsite but no (allowed-
+     * url-pattern) value was defined for it. The empty String is also returned if the seedURL's
+     * domain exactly matched a topsite and had a value of SUBDOMAIN-COPY, because we still don't
+     * want to blindly crawl a topsite (as would happen with SUBDOMAIN-COPY).
+     *  - A non-emptry String is returned if the seedURL's domain matched a topsite and a value
+     * was defined for it. (The value will be one of "SUBDOMAIN-COPY", "SINGLEPAGE" or an allowed
+     * URL pattern.
+     */
+    private String isURLinTopSitesMap(String fullSeedDomain) {
 	boolean keepLooping = true;
 
+	String domain = fullSeedDomain;
+	
 	// domain aprameter will have retained www or subdomains, but is stripped of protocol
 	
@@ -492,20 +562,29 @@
 	// If no match at all, return null.
 	do {
-	    if(domain.contains("pinterest.com")) {
-		System.err.println("@@@@@@@@@ Checking for url " + domain + " in the top sites map");
-	    }
 	    
 	    String allowed_url_pattern = topSitesMap.get(domain);
 	    if(allowed_url_pattern != null) { // if topSitesMap.containsKey(domain);
 		// there's an entry for the URL in the topSitesMap
-		System.err.println("##### A top site matches URL domain " + domain);		
-		return allowed_url_pattern;		
+		System.err.println("##### A top site matches URL domain " + domain);
+
+		// if we're dealing with SUBDOMAIN-COPY, then the fullSeedDomain, with or without
+		// www prefix, should not exactly match the topSitesMap domain
+		// e.g. we don't want to crawl a seed URL with domain www.blogspot.com
+		// despite it matching topsite blogspot.com with a value of SUBDOMAIN-COPY.
+		
+		if(allowed_url_pattern.equals(SUBDOMAIN_COPY) && isExactDomainMatch(fullSeedDomain, domain)) {
+		    return ""; // means don't crawl site, write url into unprocessed-topsite-matches file
+		}
+		return allowed_url_pattern;
 	    }
 	    // else, no entry for the URL in the topSitesMap
-	    // Not done: strip subDomain from URL and check it against topSitesMap
+	    // We're not done yet: strip subDomain from URL and check it against topSitesMap again
 	    
-	    String newURL = stripSubDomain(domain);
-	    if(domain.equals(newURL)) keepLooping = false;
-	    else domain = newURL;
+	    String newDomain = stripSubDomain(domain);
+	    if(domain.equals(newDomain)) {
+		keepLooping = false;
+	    } else {
+		domain = newDomain;
+	    }
 	} while(keepLooping);
 
@@ -674,5 +753,7 @@
 	this.setRecordCount(wetRecordCount);
     }
-    
+
+
+    // --------------- STATIC METHODS AND INNER CLASSED USED BY MAIN -------------- //
     public static void printUsage() {
 	System.err.println("Run this program as:");
@@ -770,5 +851,5 @@
 	System.out.println("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");
 
-	System.out.println("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites but had no regex of allowed url patterns.\n");
+	System.out.println("\n*** Check " + topSitesMatchedFile + " for sites not prepared for crawling because they matched top sites for which no regex of allowed url patterns were specified in sites-too-big-to-exhaustively-crawl.txt.\n");
 	
 	
