Changeset 33884 for other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

Timestamp:

2020-01-31T22:21:40+13:00 (4 years ago)

Author:

ak19

Message:

0. Previous commit had lots of modifications, and only 2 files matched the simple commit message of clarifications. The code changes in the prev commit were to incorporate the processing of a domains File (of curated sites) and write out all webPages in ach of those sites where isMRI=true. And then calculate a representative sample size n out of N total isMRI webPages, then shuffle that list of isMRI webPages and write out the first n webPage URLs in that list. 1. This commit: incorporating country code alongside URLs as Dr Bainbridge requested.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java (modified) (10 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -30 +30 @@
     private File outFolder;
 
+
+    
+    public static class Tuple {
+    public final String url;
+    public final String countryCode;
+    
+    public Tuple(String url, String countryCode) {
+        this.url = url;
+        this.countryCode = countryCode;
+    }
+
+    public String toString() {
+        return this.url + "," + countryCode;
+    }
+    }
+    
     
     public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
@@ -46 +62 @@
     
     public void produceURLsForPagesInMRI(File domainsFile) {
-    ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+    ArrayList<Tuple> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
     File outFile = new File(outFolder, "isMRI_"+domainsFile.getName());
     writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -55 +71 @@
     
     public void produceURLsForPagesContainingMRI(File domainsFile) {
-    ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.CONTAINS_MRI, domainsFile);   
+    ArrayList<Tuple> urlsList = getURLsForWebPages(MongoDBAccess.CONTAINS_MRI, domainsFile);    
     File outFile = new File(outFolder, "containsMRI_"+domainsFile.getName());
     writeURLsToFile(urlsList, outFile, urlsList.size());
@@ -63 +79 @@
     }
     
-    private ArrayList<String> getURLsForWebPages(int filterType, File domainsFile) {
-    ArrayList<String> urlsList = new ArrayList<String>();
+    private ArrayList<Tuple> getURLsForWebPages(int filterType, File domainsFile) {
+    ArrayList<Tuple> urlsList = new ArrayList<Tuple>();
     
     // 1. read each url from the domainsFile
@@ -79 +95 @@
         domain = domain.trim();
         if(!domain.equals("")) {
+            
+            String countryCode = "";
+            int index = domain.lastIndexOf(",");
+            if(index != -1) {
+            countryCode = domain.substring(index+1).trim();
+            domain = domain.substring(0, index);
+            }
             ArrayList<String> moreURLs = mongodbAccess.queryAllMatchingURLsFilteredBy(domain, filterType);
-            urlsList.addAll(moreURLs);
+
+            // Print out whether there were no isMRI pages for the domain (only containsMRI). A useful thing to know
             if(moreURLs.size() == 0 && filterType == MongoDBAccess.IS_MRI) {
-            System.out.println("Domain " + domain + " had no isMRI webpages (only containsMRI).");
+            System.out.println("   " + countryCode + " domain " + domain + " had no isMRI webpages - only containsMRI.");
             }
+
+            //urlsList.addAll(moreURLs);
+            for(int i = 0; i < moreURLs.size(); i++) {
+            urlsList.add(new Tuple(moreURLs.get(i), countryCode));
+            }
+            
         }
         }
+        System.err.println("");
     } catch(Exception e) {
         logger.error("Unable to read URLs from file " + getFilePath(domainsFile));
@@ -112 +143 @@
 
     // 0. get a list of all the web pages in the given domain listing where isMRI = true
-    ArrayList<String> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
+    ArrayList<Tuple> urlsList = getURLsForWebPages(MongoDBAccess.IS_MRI, domainsFile);
         // produceURLsForPagesInMRI(domainsFile);
     
@@ -174 +205 @@
      * Writes out the first n URLs in urlsList into outFile.
      */
-    private void writeURLsToFile(ArrayList<String> urlsList, File outFile, final int n) {
+    private void writeURLsToFile(ArrayList<Tuple> urlsList, File outFile, final int n) {
     try (
          Writer writer = new BufferedWriter(new FileWriter(outFile));
@@ -180 +211 @@
 
         for (int i=0; i < n; i++) {
-        String url = urlsList.get(i);
+        Tuple urlInfo = urlsList.get(i);
+        
         //System.out.println(list.get(i));
-        writer.write(url + "\n");
+        writer.write(urlInfo + "\n"); // calls toString() on tuple of url -> countryCode
         }
     } catch(Exception e) {
@@ -281 +313 @@
         WebPageURLsListing listing = new WebPageURLsListing(mongodb, outFolder);
 
+        System.out.println("*************************************");
+        
         
         if(args.length >= 1) {
@@ -311 +345 @@
         // TODO: generate the tables
         }
-        
+
+        System.out.println("*************************************");
     } catch(Exception e) {
         logger.error(e.getMessage(), e);