Changeset 33657 for other-projects

Timestamp:

2019-11-12T21:33:57+13:00 (4 years ago)

Author:

ak19

Message:

Some fixes after brief testing against 1/3 of the crawl. Restarted processing of crawledNode2 set of crawls.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (2 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -171 +171 @@
         // process final webpage record:
         //debugPageDump(pageDump);
-        
-        TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
-        pages.add(page);
-        pageDump = null;
-
-        // for every site, we just need to work out if any of its pages
-        // contains /mi(/) in its URL
-        String url = page.getPageURL();
-        if(!this.urlContainsLangCodeInPath && (url.contains("/mi/") || url.endsWith("/mi"))) {
-        this.urlContainsLangCodeInPath = true;
+
+        if(pageDump == null) {
+        logger.warn("siteID " + siteID + " had an empty dump.txt file. Reinspect site.");
+        } else {
+        TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
+        pages.add(page);
+        pageDump = null;
+
+        // for every site, we just need to work out if any of its pages
+        // contains /mi(/) in its URL
+        String url = page.getPageURL();
+        if(!this.urlContainsLangCodeInPath && (url.contains("/mi/") || url.endsWith("/mi"))) {
+            this.urlContainsLangCodeInPath = true;
+        }
         }
         
@@ -306 +310 @@
     File geoLiteCityDatFile = new File(this.getClass().getClassLoader().getResource("GeoLiteCity.dat").getFile());
     try {
-        this.geoLocationCountryCode = Utility.getCountryCodeOfDomain(this.domainOfSite, geoLiteCityDatFile);
-    } catch(Exception e) {
-        e.printStackTrace();
+        if(this.domainOfSite.equals("UNKNOWN")) {
+        this.geoLocationCountryCode = "UNKNOWN";
+        } else {
+        this.geoLocationCountryCode = Utility.getCountryCodeOfDomain(this.domainOfSite, geoLiteCityDatFile);
+        }
+    } catch(Exception e) {      
+        logger.error("*** For SiteID " + siteID + ", got exception: "  + e.getMessage(), e);
         this.geoLocationCountryCode = null;
     }