Changeset 33604 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

Timestamp:

2019-10-24T23:22:30+13:00 (5 years ago)

Author:

ak19

Message:

1. Better output into possible-product-sites.txt including the overseas country code prefix to help decide whether the site is worth keeping or not. 2. Updated whitelisting and top-sites filters to grab the /mi/ subsections of sites that don't appear to be autotranslated. This is done in preparation for blocking out product sites hereafter

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java (modified) (1 diff)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -324 +324 @@
         if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
             if(!possibleProductDomains.contains(domainWithProtocol)) {
-            // more expensive test, so do this only if above conditions are true:
-            if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+
+            String countryCode = "";
+            try {
+                // more expensive test, so do this only if above conditions are true:
+                countryCode = Utility.getCountryCodeOfDomain(domainWithProtocol, geoLiteCityDatFile);
+                System.err.println("@@@@ Got country code: " + countryCode);
+            } catch(Exception exceptObj) {
+                countryCode = ""; // forces domain to be included for inspection
+                
+                error("Could not check if domain " + domainWithProtocol
+                  + " was in country: " + countryCode,
+                  exceptObj);
+            }
+
+            boolean isInNZ = countryCode.toLowerCase().equals("nz");
+            
+
+            //if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+            if(!isInNZ) {
                 possibleProductDomains.add(domainWithProtocol);
-                // write both domain and URL out to file
-                possibleProductSitesWriter.write(domainWithProtocol + "\n");
+                // write both domain and a sample URL on that site out to file
+                possibleProductSitesWriter.write(countryCode + " : " + domainWithProtocol + "\n");              
                 possibleProductSitesWriter.write("\t" + url + "\n");
             }
-            } else {
-            // already wrote out domain to file, write just the URL out to file
+            } /*else {
+            // already wrote out domain to file at some point, write just the URL out to file
             possibleProductSitesWriter.write("\t" + url + "\n");
-            }
+            }*/
         }
         }