Changeset 33810

Timestamp:

2019-12-17T21:48:08+13:00 (4 years ago)

Author:

ak19

Message:

Bugfix: mi in url path should be checked for for each page of site, not just last page. Got closing bracket in loop in the wrong place.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (5 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -55 +55 @@
     static Logger logger = Logger.getLogger(org.greenstone.atea.NutchTextDumpToMongoDB.class.getName());
 
-    static boolean DEBUG_MODE = true;
+    static boolean DEBUG_MODE = true; // this is set to false in main() at the end of this class
 
     /** Counter for number of sites.
@@ -159 +159 @@
             //page.getText();
             pages.add(page);
+            inspectPageURLPath(page);
             pageDump = null;
 
@@ -176 +177 @@
         // can throw away any newlines between text start and end markers.
         
-        prevLine = line;        
+        prevLine = line;
         }
 
@@ -187 +188 @@
         TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
         pages.add(page);
+        inspectPageURLPath(page);
         pageDump = null;
-
-        // for every site, we just need to work out if any of its pages
-        // contains /mi(/) in its URL
-        String url = page.getPageURL();
-        if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
-            this.urlContainsLangCodeInPathSuffix = true;
-        }
-        if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
-            this.urlContainsLangCodeInPathPrefix = true;
-        }
-        }
+        }   
+
         
     } catch (IOException ioe) {
@@ -215 +208 @@
     }
     
-    
+    /*      
+    // for every site, we just need to work out if any of its pages
+    // contains /mi(/) in its URL
+    for(TextDumpPage aPage : pages) {
+        String url = aPage.getPageURL();
+        logger.debug("@@@@ pageURL: " + url);
+        if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
+        this.urlContainsLangCodeInPathSuffix = true;
+        logger.info("*********** URL CONTAINS SUFFIX");
+        }
+        // And if any contains http(s)://mi. in its URL
+        if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
+        this.urlContainsLangCodeInPathPrefix = true;
+        }
+    }
+    */
     prepareSiteStats(mongodbAccess);
     }
 
+    /** for every site, we just need to work out if any of its pages contains /mi(/) or http(s)://mi. in its URL.
+      * This method is called on each page of a site as the page is created. */
+    private void inspectPageURLPath(TextDumpPage page) {
+    String url = page.getPageURL();
+    //logger.debug("@@@@ pageURL: " + url);
+    
+    // check if each page in site contains /mi(/) in URL, and if so set a site-level variable accordingly
+    if(!this.urlContainsLangCodeInPathSuffix && (url.contains("/mi/") || url.endsWith("/mi"))) {
+        this.urlContainsLangCodeInPathSuffix = true;
+    }
+    // And if any page contains http(s)://mi. in its URL, then set site level variable for this accordingly
+    if(!this.urlContainsLangCodeInPathPrefix && (url.startsWith("https://mi.") || url.startsWith("http://mi."))) {
+        this.urlContainsLangCodeInPathPrefix = true;
+    }
+    }
+    
     
     private void prepareSiteStats(MongoDBAccess mongodbAccess) throws IOException {