Changeset 33603

Timestamp:

2019-10-24T22:04:37+13:00 (4 years ago)

Author:

ak19

Message:

Incorporating Dr Nichols suggestion to help weed out product sites: if tld of seed URL addresses containing /mi/ is outside NZ, add to list of possible-product-sites.txt. This should be a smaller number hopefully than all urls containing /mi and, because they're located outside nz, more likely to be a product site than not.

Location:

gs3-extensions/maori-lang-detection

Files:

• MoreReading/crawling-Nutch.txt (modified) (1 diff)
• conf/GeoLiteCity.dat (added)
• lib/geoip-api-1.2.10.jar (added)
• src/org/greenstone/atea/CCWETProcessor.java (modified) (6 diffs)
• src/org/greenstone/atea/Utility.java (modified) (2 diffs)

gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt

@@ -314 +314 @@
 
 
-
+-------------------
+Dr Nichols's suggestion: can store listing of potential product sites to inspect by checking url for /mi in combination with whether the domain's IP geolocates to OUTSIDE New Zealand (tld nz).
+* https://stackoverflow.com/questions/1415851/best-way-to-get-geo-location-in-java
+  - https://mvnrepository.com/artifact/com.maxmind.geoip/geoip-api/1.2.10
+  - older .dat.gz file is archived at https://web.archive.org/web/20180917084618/http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
+  - and newer geo country data at https://dev.maxmind.com/geoip/geoip2/geolite2/
+* https://dev.maxmind.com/geoip/geoip2/geolite2/
+* older GeoIp API (has LookupService): https://github.com/maxmind/geoip-api-java
+* Newer GeoIp2 API: https://dev.maxmind.com/geoip/geoip2/downloadable/#MaxMind_APIs
+    and https://maxmind.github.io/GeoIP2-java/doc/v2.12.0/
+* https://maxmind.github.io/GeoIP2-java/
+* https://github.com/AtlasOfLivingAustralia/ala-hub/issues/11
+
+
+---
+https://check-host.net/ip-info
+https://ipinfo.info/html/ip_checker.php
+

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -275 +275 @@
      */
     public void createSeedURLsFiles(File seedURLsFile, File urlFilterFile,
-                    File domainURLsFile, File topSiteMatchesFile) {
+                    File domainURLsFile, File topSiteMatchesFile,
+                    File possibleProductSitesFile) {
     // Maintain a Map of unique domains mapped to seed urls at that domain
     // TreeSet: by default, "the elements are ordered using their natural ordering"
@@ -286 +287 @@
     final String PROTOCOL_REGEX_PREFIX = "+^https?://";
     final String FILTER_REGEX_PREFIX = PROTOCOL_REGEX_PREFIX + "([a-z0-9-]+\\.)*"; // https?://([a-z0-9-]+\.)* for nutch's regex-urlfilter.txt
+
+    // keep an eye out on URLs we need to inspect later
+    Set<String> possibleProductDomains = new TreeSet<String>();
+    File geoLiteCityDatFile = new File(MY_CLASSLOADER.getResource("GeoLiteCity.dat").getFile());
     
     try (
          BufferedReader reader = new BufferedReader(new FileReader(this.keepURLsFile));
+         BufferedWriter possibleProductSitesWriter = new BufferedWriter(new FileWriter(possibleProductSitesFile));
          ) {
         
@@ -310 +316 @@
         }
 
+        // Dr Nichols said that a url that was located outside the country and
+        // which had /mi/ URLs was more likely to be an autotranslated (product) site.
+        // Following Dr Nichols' idea, let's keep a look out for more product sites:
+        // if any URL contains /mi AND the tld of its domain is outside of New Zealand
+        // then add that domain (if not already added) and that url into a file
+        // for later manual inspection
+        if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
+            if(!possibleProductDomains.contains(domainWithProtocol)) {
+            // more expensive test, so do this only if above conditions are true:
+            if(!Utility.isDomainInCountry(domainWithProtocol, "nz", geoLiteCityDatFile)) {
+                possibleProductDomains.add(domainWithProtocol);
+                // write both domain and URL out to file
+                possibleProductSitesWriter.write(domainWithProtocol + "\n");
+                possibleProductSitesWriter.write("\t" + url + "\n");
+            }
+            } else {
+            // already wrote out domain to file, write just the URL out to file
+            possibleProductSitesWriter.write("\t" + url + "\n");
+            }
+        }
         }
     } catch (IOException ioe) {
@@ -334 +360 @@
          BufferedWriter seedURLsWriter = new BufferedWriter(new FileWriter(seedURLsFile));
          BufferedWriter urlFilterWriter = new BufferedWriter(new FileWriter(urlFilterFile));
-         BufferedWriter topSiteMatchesWriter = new BufferedWriter(new FileWriter(topSiteMatchesFile))
+         BufferedWriter topSiteMatchesWriter = new BufferedWriter(new FileWriter(topSiteMatchesFile));       
          ) {
 
@@ -427 +453 @@
             siteURLsWriter.write(url + "\n");
             }
-            
             
             if(allowedURLPatternRegex == null) { // entire site can be crawled
@@ -887 +912 @@
     File domainURLsFile = new File(outFolder, "all-domain-urls.txt");
     File topSitesMatchedFile = new File(outFolder, "unprocessed-topsite-matches.txt");
-    
-    ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile, domainURLsFile, topSitesMatchedFile);
+    File possibleProductSitesFile = new File(outFolder, "possible-product-sites.txt");
+    
+    ccWETFilesProcessor.createSeedURLsFiles(seedURLsFile, urlFilterFile, domainURLsFile, topSitesMatchedFile, possibleProductSitesFile);
 
     info("\n*** Inspect urls in greylist at " + ccWETFilesProcessor.greyListedFile + "\n");

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java

@@ -2 +2 @@
 
 import java.io.*;
+import java.net.InetAddress;
 import java.util.zip.GZIPInputStream;
+
+import com.maxmind.geoip.*; // for LookupService and Location
 
 import org.apache.log4j.Logger;
@@ -41 +44 @@
     }
 
+    /**
+     * Attribution following below is as per https://dev.maxmind.com/geoip/geoip2/geolite2/
+     *
+     * This product includes GeoLite2 data created by MaxMind, available from
+     * <a href="https://www.maxmind.com">https://www.maxmind.com</a>.
+     *
+     * Usage:
+     * https://stackoverflow.com/questions/1415851/best-way-to-get-geo-location-in-java
+     * version I'm using: https://github.com/maxmind/geoip-api-java
+     * Newer version: https://maxmind.github.io/GeoIP2-java/
+     */
+    public static boolean isDomainInCountry(String domainWithProtocol,
+                        String countryCode, File geoLiteCityDatFile)
+    {
+    countryCode = countryCode.toUpperCase();
+    
+    int startIndex = domainWithProtocol.indexOf("//"); // http:// or https:// prefix
+    startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+    String domain = domainWithProtocol.substring(startIndex);   
+    
+    boolean result = false;
+    try {
+        // pass in the GeoLiteCity.dat file to be able to do the location lookup for domain's IP
+        LookupService cl = new LookupService(geoLiteCityDatFile, LookupService.GEOIP_MEMORY_CACHE);
+        
+        // get IP for domain
+        InetAddress inetAddress = InetAddress.getByName(domain);
+        String ipAddress = inetAddress.getHostAddress();
+        
+        // get location object for IP
+        Location location = cl.getLocation(ipAddress);
+
+        if(location != null) {
+        // compare country code with tld parameter
+        System.err.println("@@@@ Got country code: " + location.countryCode);
+        result = location.countryCode.equals(countryCode);
+        } else {
+        System.err.println("@@@@ No location info in DB for: " + domainWithProtocol);
+        }
+    } catch(Exception e) {
+        e.printStackTrace();
+        System.err.println("Could not check if domain " + domain + " was in country: " + countryCode);
+    } finally {
+        return result;
+    }
+    }
 }