Changeset 33906

Timestamp:

2020-02-05T23:36:37+13:00 (4 years ago)

Author:

ak19

Message:

Code is intermediate state. 1. Introduced basicDomain field to MongoDB and recreated the MongoDB tables/collections, this will help discount duplicated domains under http and https, with and without www. Though webpage URLs may potentially still be unique and not duplicated across all 4 possible variants, I want them counted under the same base domain name. 2. Another issue noticed now is that some of the sites appear to be hosted on multiple countries servers, and so slightly different country code counts and domainlistings are returned. 3. So added code modifications (untested) to sort the domains alphabetically after stripping protocol and www to allow comparing the old domainListing results of MongoDB's now renamed oldWebsites and oldWebpages collections to the new versions of these collections and to then update the differences in manual counts.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (11 diffs)
• NutchTextDumpToMongoDB.java (modified) (3 diffs)
• WebPageURLsListing.java (modified) (5 diffs)
• morphia/WebsiteInfo.java (modified) (3 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -26 +26 @@
 import com.mongodb.Block;
 
+import org.bson.BsonArray;
+import org.bson.BsonString;
 import org.bson.Document;
 import org.bson.conversions.Bson;
@@ -217 +219 @@
         .append("siteFolderName", website.siteFolderName) 
         .append("domain", website.domain) 
+        .append("basicDomain", website.basicDomain) 
         .append("totalPages", website.totalPages)
         .append("numPagesWithBodyText", website.countOfWebPagesWithBodyText)
@@ -389 +392 @@
      *  and https://stackoverflow.com/questions/47979978/ambiguous-reference-to-foreach-when-listing-mongodbs-database-in-java
      *
-     *
+     * Count by country code of non-NZ websites containing a positive number of sentences in MRI,
+     * listing all the base domain strings (no protocol or www) in ALPHABETICAL ORDER
+     * and total counts of numPagesInMRI and numPagesContainingMRI across all these
+     * matching sites.
+     * 
      * The mongodb aggregate() we want to run this time:
      *
@@ -405 +412 @@
           $group: {
             _id: "nz",
-            count: { $sum: 1 },
-            domain: { $addToSet: '$domain' }
+            count: { $sum: 1 },            
+        domain: { $addToSet: '$basicDomain' } // domain: {$push: "$basicDomain" }
           }
     },
@@ -431 +438 @@
          match(andQuery),
          unwind("$geoLocationCountryCode"),
-         group("NZ", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
+         group("NZ", Arrays.asList(sum("count", 1),
+                   addToSet("domain", "$basicDomain"))),
          sort(BasicDBObject.parse("{count : -1}"))
      )).forEach((Block<Document>)doc -> writeDoc(doc, writer));
@@ -441 +449 @@
     
     /**
+     * Count of NZ (incl .nz TLD)  websites containing a positive number of sentences in MRI, 
+     * listing all the base domain strings (no protocol or www) in ALPHABETICAL ORDER
+     * and total counts of numPagesInMRI and numPagesContainingMRI across all these
+     * matching sites.
+     *
      * The aggregate() we want to run this time:
      *
@@ -459 +472 @@
             _id: {$toLower: '$geoLocationCountryCode'},
             count: { $sum: 1 },
-            domain: { $addToSet: '$domain' }
+        domain: { $addToSet: '$basicDomain' } // domain: {$push: "$basicDomain" }
           }
      },
@@ -484 +497 @@
         orQuery);
 
-
     collection.aggregate(Arrays.asList(
          match(andQuery),  //match(BasicDBObject.parse(matchQuery))
          // match((List<DBObject>)JSON.parse(matchQuery)),
          unwind("$geoLocationCountryCode"),
-         group("$geoLocationCountryCode", Arrays.asList(sum("count", 1), addToSet("domain", "$domain"))),
+         group("$geoLocationCountryCode", Arrays.asList(sum("count", 1), 
+                        addToSet("domain", "$basicDomain"))),
          sort(BasicDBObject.parse("{count : -1}"))
        )).forEach((Block<Document>)doc -> writeDoc(doc, writer));
@@ -639 +652 @@
      */    
     public void writeDoc(Document doc, Writer writer) throws UncheckedIOException {
+
+    // If there's a domain field in the json Doc, sort this domain listing alphabetically
+    Object domainList = doc.remove("domain");
+    if(domainList != null) {
+        doc.put("domain", sortAlphabetically(domainList));
+    }
+    
     //OLD WAY: writer.write(doc.toJson(new JsonWriterSettings(JsonMode.STRICT, true)) + NEWLINE);
     // Can't control json output to add newlines after each array element,
@@ -655 +675 @@
     // Have to use gson's pretty print to produce a json string that contains
     // newlines after every array element in the json:
+    
     String jsonStr = prettyPrintJson(doc.toJson());
     //System.err.println(jsonStr);
@@ -663 +684 @@
         throw new UncheckedIOException(ex);
     }       
+    }
+
+    private List sortAlphabetically(Object list) {
+    BsonArray domainList = (BsonArray)list;
+    //for(String domain : domainList) {
+    for(int i = domainList.size() - 1; i >= 0; i--) {
+        BsonString domain = domainList.get(i).asString();
+        String domainStr = Utility.stripProtocolAndWWWFromURL(domain.toString());
+        domainList.set(i, new BsonString(domainStr));       
+    }
+
+    return domainList;
     }
     

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -75 +75 @@
     private String geoLocationCountryCode = null; /** 2 letter country code */
     private boolean urlContainsLangCodeInPath = false; /** If any URL on this site contains a /mi(/) or http(s)://mi.* in its URL path */
-    
+
     private String domainOfSite;
+    private String baseSiteDomain; // domainOfSite stripped of any http(s)://www.
     private int numPagesInMRI = 0;
     private int numPagesContainingMRI = 0;
@@ -202 +203 @@
         String url = firstPage.getPageURL();
         this.domainOfSite = Utility.getDomainForURL(url, true);
+        this.baseSiteDomain = Utility.stripProtocolAndWWWFromURL(this.domainOfSite);
     }
     else {
         this.domainOfSite = "UNKNOWN";
+        this.baseSiteDomain = "UNKNOWN";
     }
     
@@ -339 +342 @@
     int totalPages = pages.size();  
 
-    WebsiteInfo website = new WebsiteInfo(/*SITE_COUNTER,*/ this.siteID, this.domainOfSite,
+    WebsiteInfo website = new WebsiteInfo(/*SITE_COUNTER,*/ this.siteID,
+          this.domainOfSite, this.baseSiteDomain,
           totalPages, this.countOfWebPagesWithBodyText,
           this.numPagesInMRI, this.numPagesContainingMRI,

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -57 +57 @@
     
     public void produceURLsForPagesInMRI(File domainsFile) {
-    ArrayList<Tuple> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
     File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
     writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -66 +66 @@
     
     public void produceURLsForPagesContainingMRI(File domainsFile) {
-    ArrayList<Tuple> urlsList = getURLsForWebPages(MongoDBAccess.CONTAINS_MRI, domainsFile);    
+    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.CONTAINS_MRI, domainsFile);    
     File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
     writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -74 +74 @@
     }
     
-    private ArrayList<Tuple> getURLsForWebPages(int filterType, File domainsFile) {
+    private ArrayList<Tuple> getURLsForAllWebPagesInSiteListing(int filterType, File domainsFile) {
     ArrayList<Tuple> urlsList = new ArrayList<Tuple>();
     
@@ -120 +120 @@
     }
     
-    /** Given a hand curated list of NZ sites with positive numPagesContainingMRI,
-     * get a listing of all their web pages IN_MRI (or CONTAINS_MRI?). 
-     * Total all these pages in MRI (N), then work out the correct sample size (n)
+    /** Given a hand curated list of all sites with positive numPagesContainingMRI
+     * determined by manual inspection, get a listing of all their web pages that
+     * are IN_MRI (or CONTAINS_MRI?). 
+     * Total all these pages that are inMRI (N), then work out the correct sample size (n)
      * at 90% confidence with 5% margin of error. Then generate a random listing
      * of n of these pages in MRI of these trusted sites and output to a file
-     * for manual inspection. */
+     * for manual inspection of the sample webpage URLs at page-level. */
     /* OLD: Given a hand curated list of non-NZ sites that CONTAINS_MRI, get a listing
      * of all their web pages IN_MRI (or CONTAINS_MRI).
@@ -138 +139 @@
 
     // 0. get a list of all the web pages in the given domain listing where isMRI = true
-    ArrayList<Tuple> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
         // produceURLsForPagesInMRI(domainsFile);
     

other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/WebsiteInfo.java

@@ -9 +9 @@
     public final String siteFolderName;
     public final String domain;
+    public final String basicDomain; // domain without protocol and www. prefix
     
     public final int totalPages;
@@ -23 +24 @@
     public final boolean urlContainsLangCodeInPath;
     
-    public WebsiteInfo(/*int siteCount,*/ String siteFolderName, String domainOfSite,
+    public WebsiteInfo(/*int siteCount,*/ String siteFolderName,
+               String domainOfSite, String baseSiteDomain,
                int totalPages, int countOfWebPagesWithBodyText,
                int numPagesInMRI, int numPagesContainingMRI,
@@ -32 +34 @@
     this.siteFolderName = siteFolderName;
     this.domain = domainOfSite;
+    this.basicDomain = baseSiteDomain;
     
     this.totalPages = totalPages;