Changeset 33808 for other-projects

Timestamp:

2019-12-17T19:31:28+13:00 (4 years ago)

Author:

ak19

Message:

Storing not just whether /mi(/) suffix is in path, but also whether http(s):mi. is in path, as storing these can help reducing number of auto-translated sites too in a similar way.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• NutchTextDumpToMongoDB.java (modified) (3 diffs)
• morphia/WebsiteInfo.java (modified) (3 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -74 +74 @@
     
     private String geoLocationCountryCode = null; /** 2 letter country code */
-    private boolean urlContainsLangCodeInPath = false; /** If any URL on this site contains a /mi(/) in its URL */
+    private boolean urlContainsLangCodeInPathSuffix = false; /** If any URL on this site contains a /mi(/) in its URL */
+    private boolean urlContainsLangCodeInPathPrefix = false; /** If any URL on this site contains a http(s)://mi.* in its URL */
     
     private String domainOfSite;
@@ -191 +192 @@
         // contains /mi(/) in its URL
         String url = page.getPageURL();
-        if(!this.urlContainsLangCodeInPath && (url.contains("/mi/") || url.endsWith("/mi"))) {
-            this.urlContainsLangCodeInPath = true;
+        if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
+            this.urlContainsLangCodeInPathSuffix = true;
+        }
+        if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
+            this.urlContainsLangCodeInPathPrefix = true;
         }
         }
@@ -328 +332 @@
           this.numPagesInMRI, this.numPagesContainingMRI,
           this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl,
-          this.geoLocationCountryCode, this.urlContainsLangCodeInPath);
+          this.geoLocationCountryCode, this.urlContainsLangCodeInPathSuffix, this.urlContainsLangCodeInPathPrefix);
 
     //mongodbAccess.insertWebsiteInfo(website);

other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/WebsiteInfo.java

@@ -21 +21 @@
     
     public final String geoLocationCountryCode;
-    public final boolean urlContainsLangCodeInPath;
+    public final boolean urlContainsLangCodeInPathSuffix;
+    public final boolean urlContainsLangCodeInPathPrefix;
     
     public WebsiteInfo(/*int siteCount,*/ String siteFolderName, String domainOfSite,
@@ -27 +28 @@
                int numPagesInMRI, int numPagesContainingMRI,
                long siteCrawledTimestamp, boolean siteCrawlUnfinished, boolean redoCrawl,
-               String geoLocationCountryCode, boolean urlContainsLangCodeInPath)
+               String geoLocationCountryCode, boolean urlContainsLangCodeInPathSuffix, boolean urlContainsLangCodeInPathPrefix)
     {
     //this.id = siteCount;
@@ -44 +45 @@
     
     this.geoLocationCountryCode = geoLocationCountryCode;
-    this.urlContainsLangCodeInPath = urlContainsLangCodeInPath;
+    this.urlContainsLangCodeInPathSuffix = urlContainsLangCodeInPathSuffix;
+    this.urlContainsLangCodeInPathPrefix = urlContainsLangCodeInPathPrefix;
     }
 }