Changeset 33808 for other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

Timestamp:

2019-12-17T19:31:28+13:00 (4 years ago)

Author:

ak19

Message:

Storing not just whether /mi(/) suffix is in path, but also whether http(s):mi. is in path, as storing these can help reducing number of auto-translated sites too in a similar way.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (3 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -74 +74 @@
     
     private String geoLocationCountryCode = null; /** 2 letter country code */
-    private boolean urlContainsLangCodeInPath = false; /** If any URL on this site contains a /mi(/) in its URL */
+    private boolean urlContainsLangCodeInPathSuffix = false; /** If any URL on this site contains a /mi(/) in its URL */
+    private boolean urlContainsLangCodeInPathPrefix = false; /** If any URL on this site contains a http(s)://mi.* in its URL */
     
     private String domainOfSite;
@@ -191 +192 @@
         // contains /mi(/) in its URL
         String url = page.getPageURL();
-        if(!this.urlContainsLangCodeInPath && (url.contains("/mi/") || url.endsWith("/mi"))) {
-            this.urlContainsLangCodeInPath = true;
+        if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
+            this.urlContainsLangCodeInPathSuffix = true;
+        }
+        if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
+            this.urlContainsLangCodeInPathPrefix = true;
         }
         }
@@ -328 +332 @@
           this.numPagesInMRI, this.numPagesContainingMRI,
           this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl,
-          this.geoLocationCountryCode, this.urlContainsLangCodeInPath);
+          this.geoLocationCountryCode, this.urlContainsLangCodeInPathSuffix, this.urlContainsLangCodeInPathPrefix);
 
     //mongodbAccess.insertWebsiteInfo(website);