Changeset 34000

Timestamp:

2020-03-09T18:55:01+13:00 (4 years ago)

Author:

ak19

Message:

Some debugging and other minor changes

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• AllDomainCount.java (modified) (2 diffs)
• CountryCodeCountsMapData.java (modified) (1 diff)
• Utility.java (modified) (1 diff)

other-projects/maori-lang-detection/src/org/greenstone/atea/AllDomainCount.java

@@ -30 +30 @@
     private Set<String> uniqueURLs = new TreeSet<String>(); // stripped of protocol and www
 
+    // just for debugging
+    private Map<String, String> domainToInfile = new HashMap<String,String>();
+    
     private int lineCount = 0;
     private final String[] filenames = {"discardURLs.txt", "greyListed.txt", "keepURLs.txt"};;
@@ -64 +67 @@
         // don't care about real uniqueness of domains - Set just ensures that each
         // domain is listed once. Keeps http(s) and www distinct
-        String domainWithProtocol = Utility.getDomainForURL(url, true);     
+        String domainWithProtocol = Utility.getDomainForURL(url, true);
         domains.add(domainWithProtocol);
+        
+        // START DEBUGGING - to inspect why domains tend to be repeated
+        // between (a) discardURLs and greylisted/keepURLs and
+        // (b) greylisted and keepURLs.
+        // Because (a) some pages on greylisted or even accepted sites contain
+        // less than minimum content and end up in discarded urls list.
+        // And (b) some domains contain URLs with /product-... suffix
+        // and are greylisted but the same domain can contain URLs without a /product
+        // pattern and end up as part of keepURLs.
+        if(!domainToInfile.containsKey(domainWithProtocol)) {
+            domainToInfile.put(domainWithProtocol, infile.getPath());
+        } else {
+            String oldInfile = domainToInfile.get(domainWithProtocol);
+            if(!oldInfile.equals(infile.getPath())) {
+            System.err.println("\t*** " + domainWithProtocol + " is repeated between old " + oldInfile + " and new " + infile.getPath());
+            }
+        }
+        // END DEBUGGING
         
         // unique domains - so stripped of http(s) and www

other-projects/maori-lang-detection/src/org/greenstone/atea/CountryCodeCountsMapData.java

@@ -708 +708 @@
     // https://developer.mozilla.org/en-US/docs/Tools/Browser_Console
     String webConsoleCommand = ":screenshot --selector \".map\" --file --filename \"" + outputFilePath + "\"";  
-    System.out.println("The following command should be copied to your clipboard. Else copy it.");
+    System.out.println("The following command should have been copied to your clipboard. Else copy it.");
     System.out.println("Run it in the opened Firefox tab's Web Console command line (press F12 and undock web console first):\n");
     System.out.println(webConsoleCommand);

other-projects/maori-lang-detection/src/org/greenstone/atea/Utility.java

@@ -93 +93 @@
     url = stripProtocolFromURL(url);
     
-    if(url.startsWith("www.")) { // strip any "wwww." at start as well too
+    if(url.startsWith("www.")) { // also strip any "www." at start
         url = url.substring(4);
     }