Changeset 33909 for other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

Timestamp:

2020-02-12T19:02:44+13:00 (4 years ago)

Author:

ak19

Message:

1. Implementing tables 3 to 5. 2. Rolled back the introduction of the basicDomain field (domain stripped of http/https and www prefixes) as the code can create and sort this field alphabetically, whereas it didn't sort properly in mongodb. 3. The code now does sort the domains stripped of protocol and www for the mongodb queries producing domain results and ensures the domain list is unique. 4. Split the MongoDBAccess class into 2, with the connection code in MongoDBAccess.java and the querying code in MongoDBQueryer (a subclass of MongoDBAccess) that is so far exclusively used by WebPageURLsListing.java

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java (modified) (14 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -29 +29 @@
     static private final long FIXED_SEED = 1000;
     
-    private final MongoDBAccess mongodbAccess;
+    private final MongoDBQueryer mongodbQueryer;
     private File outFolder;
 
@@ -49 +49 @@
     
     
-    public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
+    public WebPageURLsListing(MongoDBQueryer mongodbQueryer, File outFolder)
     {
-    this.mongodbAccess = mongodbAccess;
+    this.mongodbQueryer = mongodbQueryer;
     this.outFolder = outFolder;
     }
@@ -57 +57 @@
     
     public void produceURLsForPagesInMRI(File domainsFile) {
-    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
+    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
     File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
     writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -66 +66 @@
     
     public void produceURLsForPagesContainingMRI(File domainsFile) {
-    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.CONTAINS_MRI, domainsFile);    
+    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.CONTAINS_MRI, domainsFile);   
     File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
     writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -97 +97 @@
             domain = domain.substring(0, index);
             }
-            ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
+            ArrayList<String> moreURLs = mongodbQueryer.queryAllMatchingURLsFilteredBy(domain, filterType);
 
             // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
-            if(moreURLs.size() == 0 && filterType == MongoDBAccess.IS_MRI) {
+            if(moreURLs.size() == 0 && filterType == MongoDBQueryer.IS_MRI) {
             System.out.println("   " + countryCode + " domain " + domain + " had no isMRI webpages - only containsMRI.");
             }
@@ -133 +133 @@
     public void mriWebPageListingForDomainListing(File domainsFile) {
 
-    int filterType = MongoDBAccess.IS_MRI;
+    int filterType = MongoDBQueryer.IS_MRI;
     
     // for overseas websites, 
@@ -139 +139 @@
 
     // 0. get a list of all the web pages in the given domain listing where isMRI = true
-    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBAccess.IS_MRI, domainsFile);
+    ArrayList<Tuple> urlsList = getURLsForAllWebPagesInSiteListing(MongoDBQueryer.IS_MRI, domainsFile);
         // produceURLsForPagesInMRI(domainsFile);
     
@@ -148 +148 @@
     // 2. write all the URLs in urlsList to a file
     //File outFolder = domainsFile.getParentFile();
-    String fileName = (filterType == MongoDBAccess.IS_MRI) ? "isMRI_" : "containsMRI_";
+    String fileName = (filterType == MongoDBQueryer.IS_MRI) ? "isMRI_" : "containsMRI_";
     File outFile = new File(outFolder, fileName+domainsFile.getName());
 
@@ -219 +219 @@
 
     /* ---------------------------------------- */
-
     /** 
-     * Create the file 5counts_tentativeNonAutotranslatedSites.json
+     * Create the file 5counts_containsMRISites_allNZGrouped.json
+     * that contains the count and domains for NZ sites (NZ origin or nz TLD) with pages
+     * that CONTAIN_MRI, followed by counts and domains listing for overseas sites
+     * that CONTAIN_MRI.
+     * @return full path of file generated
+     */
+    public String writeContainsMRISites_nzSitesAndTLDsGrouped() {
+    
+    File outFile = new File(outFolder, "5counts_containsMRISites_allNZGrouped.json");
+
+    String filename = Utility.getFilePath(outFile);
+    
+    try (
+         Writer writer = new BufferedWriter(new FileWriter(outFile));
+         ) {
+        // first write out NZ sites and .nz TLD count and domains
+        mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
+        // next write out all overseas sites (not NZ origin or .nz TLD)
+        // that have no "mi" in the URL path as mi.* or */mi
+        boolean isMiInURLPath = false;
+        mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI);
+        
+    } catch(Exception e) {
+        logger.error("Unable to write to file " + filename);
+        logger.error(e.getMessage(), e);
+    }
+    
+    System.err.println("*** Wrote file: " + filename);
+
+    return filename;
+    }
+    
+    /** 
+     * Create the file 5a_counts_tentativeNonAutotranslatedSites.json
      * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
      * followed by counts and domain listing for overseas sites that are either from Australia
@@ -237 +269 @@
          ) {
         // first write out NZ sites and .nz TLD count and domains
-        mongodbAccess.aggregateContainsMRIForNZ(writer, MongoDBAccess.CONTAINS_MRI);
+        mongodbQueryer.aggregateContainsMRIForNZ(writer, MongoDBQueryer.CONTAINS_MRI);
         // next write out all overseas sites (not NZ origin or .nz TLD)
         // that have no "mi" in the URL path as mi.* or */mi
         boolean isMiInURLPath = false;
-        mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
+        mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
         
     } catch(Exception e) {
@@ -254 +286 @@
 
     /**
+     * Create the file 5b_counts_overseasSitesWithMiInPath.json
      * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
      * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path.
@@ -267 +300 @@
          ) {
         boolean isMiInURLPath = true;
-        mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
+        mongodbQueryer.aggregateContainsMRIForOverseas(writer, MongoDBQueryer.CONTAINS_MRI, isMiInURLPath);
 
     } catch(Exception e) {
@@ -300 +333 @@
 
     try (
-         MongoDBAccess mongodb = new MongoDBAccess();
+         MongoDBQueryer mongodb = new MongoDBQueryer();
          ) {
 
@@ -335 +368 @@
         //System.err.println("For N = " + 4360 + ", n = " + listing.calcSampleSize(4360));
         //System.err.println("For N = " + 681 + ", n = " + listing.calcSampleSize(681));
-        
-        String filename = listing.writeTentativeNonAutotranslatedSites();
+
+        // get all sites where >0 pages have containsMRI=true
+        // grouping NZ sites and .nz TLDs together and remainder under overseas
+        // geolocations.
+        String filename = listing.writeContainsMRISites_nzSitesAndTLDsGrouped();
+
+        // separately:
+        // - all NZ containsMRI + overseas tentative non-product sites with containMRI
+        // - overseas tentative product sites with containMRI
+        filename = listing.writeTentativeNonAutotranslatedSites();
         filename = listing.writeOverseasSitesWithMiInURLPath();