Changeset 33604

Timestamp:

2019-10-24T23:22:30+13:00 (5 years ago)

Author:

ak19

Message:

1. Better output into possible-product-sites.txt including the overseas country code prefix to help decide whether the site is worth keeping or not. 2. Updated whitelisting and top-sites filters to grab the /mi/ subsections of sites that don't appear to be autotranslated. This is done in preparation for blocking out product sites hereafter

Location:

gs3-extensions/maori-lang-detection

Files:

• conf/sites-too-big-to-exhaustively-crawl.txt (modified) (1 diff)
• conf/url-whitelist-filter.txt (modified) (2 diffs)
• src/org/greenstone/atea/CCWETProcessor.java (modified) (1 diff)
• src/org/greenstone/atea/Utility.java (modified) (1 diff)

gs3-extensions/maori-lang-detection/conf/sites-too-big-to-exhaustively-crawl.txt

@@ -61 +61 @@
 # special case
 mi.centr-zashity.ru,SINGLEPAGE
+
+martinvrijland.nl,martinvrijland.nl/mi/
+csunplugged.org,csunplugged.org/mi/
+gpedia.com,gpedia.com/mi/
 
 # TOP SITE BUT NOT TOP 500

gs3-extensions/maori-lang-detection/conf/url-whitelist-filter.txt

@@ -2 +2 @@
 # whitelist overrides blacklist and greylist.
 # FORMAT:
-# precede URL by ^ to greylist urls that match the given prefix
-# succeed URL by $ to greylist urls that match the given suffix
-# ^url$ will greylist urls that match the given url completely
-# Without either ^ or $ symbol, urls containing the given url will get greylisted
+# precede URL by ^ to whitelist urls that match the given prefix
+# succeed URL by $ to whitelist urls that match the given suffix
+# ^url$ will whitelist urls that match the given url completely
+# Without either ^ or $ symbol, urls containing the given url will get whitelisted
 
 # Special exception for this url on yale.edu, since we needed to blacklist
@@ -15 +15 @@
 http://www.krassotkin.ru/sites/prayer.su/maori/
 https://mi.centr-zashity.ru/
+
+
+
+# WHITELIST WEBSITES THAT HAVE NON-AUTOMATED /mi/ SUBSECTIONS
+# WE CONTROL WHAT PART OF THEM WILL BE DOWNLOADED (THE /mi SUBSECTION)
+# IN sites-too-big-to-exhaustively-crawl.txt
+#https://www.martinvrijland.nl/mi/te-mana-hinengaro/Ko-te-nuinga-ake-o-nga-tangata-kei-te-timata-ki-te-kite-kei-te-noho-tatou-i-roto-i-te-whakaata-ko-te-aha-tenei/
+#https://www.csunplugged.org/mi/principles/
+#http://www.gpedia.com/mi/gpedia/Reo_M%C4%81ori
+
+https://www.martinvrijland.nl
+https://www.csunplugged.org
+http://www.gpedia.com
+
+

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -324 +324 @@
         if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
             if(!possibleProductDomains.contains(domainWithProtocol)) {
-            // more expensive test, so do this only if above conditions are true:
-            if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+
+            String countryCode = "";
+            try {
+                // more expensive test, so do this only if above conditions are true:
+                countryCode = Utility.getCountryCodeOfDomain(domainWithProtocol, geoLiteCityDatFile);
+                System.err.println("@@@@ Got country code: " + countryCode);
+            } catch(Exception exceptObj) {
+                countryCode = ""; // forces domain to be included for inspection
+                
+                error("Could not check if domain " + domainWithProtocol
+                  + " was in country: " + countryCode,
+                  exceptObj);
+            }
+
+            boolean isInNZ = countryCode.toLowerCase().equals("nz");
+            
+
+            //if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+            if(!isInNZ) {
                 possibleProductDomains.add(domainWithProtocol);
-                // write both domain and URL out to file
-                possibleProductSitesWriter.write(domainWithProtocol + "\n");
+                // write both domain and a sample URL on that site out to file
+                possibleProductSitesWriter.write(countryCode + " : " + domainWithProtocol + "\n");              
                 possibleProductSitesWriter.write("\t" + url + "\n");
             }
-            } else {
-            // already wrote out domain to file, write just the URL out to file
+            } /*else {
+            // already wrote out domain to file at some point, write just the URL out to file
             possibleProductSitesWriter.write("\t" + url + "\n");
-            }
+            }*/
         }
         }

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java

@@ -54 +54 @@
      * version I'm using: https://github.com/maxmind/geoip-api-java
      * Newer version: https://maxmind.github.io/GeoIP2-java/
+     *
+     * @return 2 letter countrycode in uppercase or an exception
      */
+    public static String getCountryCodeOfDomain(String domainWithProtocol, File geoLiteCityDatFile)
+    throws Exception
+    {   
+    int startIndex = domainWithProtocol.indexOf("//"); // http:// or https:// prefix
+    startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+    String domain = domainWithProtocol.substring(startIndex);   
+    
+    // pass in the GeoLiteCity.dat file to be able to do the location lookup for domain's IP
+    LookupService cl = new LookupService(geoLiteCityDatFile, LookupService.GEOIP_MEMORY_CACHE);
+        
+    // get IP for domain
+    InetAddress inetAddress = InetAddress.getByName(domain);
+    String ipAddress = inetAddress.getHostAddress();
+    
+    // get location object for IP
+    Location location = cl.getLocation(ipAddress);
+    
+    if(location == null) {
+        throw new Exception("@@@@ No location info in DB for: " + domain);
+    } else {
+        return location.countryCode;
+    }
+    
+    }
+    
     public static boolean isDomainInCountry(String domainWithProtocol,
                         String countryCode, File geoLiteCityDatFile)