Changeset 33811 for other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

Timestamp:

2019-12-18T16:51:34+13:00 (4 years ago)

Author:

ak19

Message:

Returning to using a single variable, urlContainsLangCodeInPath, to record both whether any page on a site contains /mi(/) OR http(s):mi.* in its URL path.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (4 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -74 +74 @@
     
     private String geoLocationCountryCode = null; /** 2 letter country code */
-    private boolean urlContainsLangCodeInPathSuffix = false; /** If any URL on this site contains a /mi(/) in its URL */
-    private boolean urlContainsLangCodeInPathPrefix = false; /** If any URL on this site contains a http(s)://mi.* in its URL */
+    private boolean urlContainsLangCodeInPath = false; /** If any URL on this site contains a /mi(/) or http(s)://mi.* in its URL path */
     
     private String domainOfSite;
@@ -208 +207 @@
     }
     
-    /*      
-    // for every site, we just need to work out if any of its pages
-    // contains /mi(/) in its URL
+    /* No need to loop again through all pages. Instead, just inspectPageURLPath() as each page is created above.
+    // For any site, we just need to work out if any of its pages contains /mi(/) or http(s)://mi.* in its URL path
     for(TextDumpPage aPage : pages) {
-        String url = aPage.getPageURL();
-        logger.debug("@@@@ pageURL: " + url);
-        if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
-        this.urlContainsLangCodeInPathSuffix = true;
-        logger.info("*********** URL CONTAINS SUFFIX");
-        }
-        // And if any contains http(s)://mi. in its URL
-        if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
-        this.urlContainsLangCodeInPathPrefix = true;
-        }
+        inspectPageURLPath(aPage);
     }
     */
@@ -233 +222 @@
     //logger.debug("@@@@ pageURL: " + url);
     
-    // check if each page in site contains /mi(/) in URL, and if so set a site-level variable accordingly
-    if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
-        this.urlContainsLangCodeInPathSuffix = true;
-    }
-    // And if any page contains http(s)://mi. in its URL, then set site level variable for this accordingly
-    if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
-        this.urlContainsLangCodeInPathPrefix = true;
+    if(!this.urlContainsLangCodeInPath) { // if not already set to true for any previous page in this site,
+        // check if this page of the site contains /mi(/) or http(s)://mi in its URL path
+        if(url.contains("/mi/") || url.endsWith("/mi") || url.startsWith("https://mi.") || url.startsWith("http://mi.")) {
+        this.urlContainsLangCodeInPath = true;
+        }
     }
     }
@@ -356 +343 @@
           this.numPagesInMRI, this.numPagesContainingMRI,
           this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl,
-          this.geoLocationCountryCode, this.urlContainsLangCodeInPathSuffix, this.urlContainsLangCodeInPathPrefix);
+          this.geoLocationCountryCode, this.urlContainsLangCodeInPath);
 
     //mongodbAccess.insertWebsiteInfo(website);