Changeset 33801 for other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

Timestamp:

2019-12-13T18:40:46+13:00 (4 years ago)

Author:

ak19

Message:

1. NutchTextDumpToMongoDB Added an extra field to each document in Websites mongodb collection: numPagesContainingMRI. 2. Bugfix to yesterday's commit: performing a substring() was off by one.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (5 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -78 +78 @@
     private String domainOfSite;
     private int numPagesInMRI = 0;
+    private int numPagesContainingMRI = 0;
     
     /** keep a list to store the text of each page */
@@ -86 +87 @@
     /** Number of language and confidence results to return for storing in MongoDB
      * MongoDB runs out of space if storing too many, as we store this info per sentence
-     * and a long text document becomes a very large MongoDB document presumable*/
+     * and a long text document becomes a very large MongoDB document presumably */
     private static final int NUM_TOP_LANGUAGES = 3; // 103 max, in current version of opennlp lang model
 
@@ -262 +263 @@
         webpage.setMRISentenceCount(numSentencesInMRI);
         webpage.setContainsMRI((numSentencesInMRI > 0));
-        
+        if(numSentencesInMRI > 0) { // if(numSentencesInMRI >= 5) {
+            // Not sure if we can trust that a single sentence detected as Maori on a page is really Maori
+            // But if at least 5 sentences are detected as Maori, it is more likely to be the case to be MRI?
+            numPagesContainingMRI++;
+        }
+            
         //mongodbAccess.insertWebpageInfo(webpage);
         // Uses morphia to save to mongodb, see https://www.baeldung.com/mongodb-morphia
@@ -298 +304 @@
     } catch(Exception e) {      
         logger.error("*** For SiteID " + siteID + ", got exception: "  + e.getMessage(), e);
-        this.geoLocationCountryCode = "UNKNOWN"; // couldn't get the country code, so should also be UNKNOWN not null
+
+        //if(this.domainOfSite.endsWith(".nz")) { // nz TLDs are worth counting
+        //this.geoLocationCountryCode = "NZ"; 
+        //}
+
+        // Help along identification of domain's country by construing TLDs if 2 letters after last period mark
+        int periodIndex = domainOfSite.length()-3;
+        // .com|org etc extensions that have 3 chars afte period mark will remain unknown
+        // 2 letter extensions will be considered TLD
+        if(periodIndex >=0 && domainOfSite.charAt(periodIndex) == '.' && ((periodIndex+1) < domainOfSite.length())) {
+        // has a 2 letter TLD. Make it uppercase to match return value of Utility.getCountryCodeOfDomain() above
+        String TLD = domainOfSite.substring(periodIndex+1);
+        this.geoLocationCountryCode = TLD.toUpperCase();
+        } else {
+        this.geoLocationCountryCode = "UNKNOWN"; // couldn't get the country code, so should also be UNKNOWN not null
+        }
     }
 
@@ -304 +325 @@
 
     WebsiteInfo website = new WebsiteInfo(/*SITE_COUNTER,*/ this.siteID, this.domainOfSite,
-          totalPages, this.countOfWebPagesWithBodyText, this.numPagesInMRI, 
+          totalPages, this.countOfWebPagesWithBodyText,
+          this.numPagesInMRI, this.numPagesContainingMRI,
           this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl,
           this.geoLocationCountryCode, this.urlContainsLangCodeInPath);