Index: /other-projects/maori-lang-detection/mongodb-data/pieChart01c_obtainingSeedURLs.svg
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/pieChart01c_obtainingSeedURLs.svg	(revision 34006)
+++ /other-projects/maori-lang-detection/mongodb-data/pieChart01c_obtainingSeedURLs.svg	(revision 34006)
@@ -0,0 +1,1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" style="font-family:'lucida grande', 'lucida sans unicode', arial, helvetica, sans-serif;font-size:12px;" xmlns="http://www.w3.org/2000/svg" width="600" height="400"><desc>Created with Highcharts 4.2.7</desc><defs><clipPath id="highcharts-47"><rect x="1" y="1" width="578" height="297"></rect></clipPath></defs><rect x="10" y="49" width="580" height="299" fill="#ffffff"></rect><rect x="10" y="49" width="580" height="299" stroke="#026FC1" stroke-width="2" fill="none"></rect><g class="highcharts-series-group"><g class="highcharts-series highcharts-series-0" transform="translate(10,49) scale(1 1)"><path fill="#ff0000" d="M 296.9835024705007 69.00000168005236 A 81 81 0 0 1 377.6113491996138 157.92530000807037 L 297 150 A 0 0 0 0 0 297 150 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#a9a9a9" d="M 377.6033835952554 158.00590738118507 A 81 81 0 0 1 366.3072677789028 191.92257903592596 L 297 150 A 0 0 0 0 0 297 150 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#DDDF00" d="M 366.265310553223 191.9918653308659 A 81 81 0 0 1 366.265310553223 191.9918653308659 L 297 150 A 0 0 0 0 0 297 150 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#3ebe52" d="M 366.22328406223835 192.06110963394397 A 81 81 0 1 1 296.8874926250162 69.00007813528106 L 297 150 A 0 0 0 1 0 297 150 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path></g><g class="highcharts-markers highcharts-series-0" transform="translate(10,49) scale(1 1)"></g></g><text x="300" text-anchor="middle" class="highcharts-title" style="color:#000000;font-size:10px;font-family:'arial', sans-serif;fill:#000000;width:536px;" y="17"><tspan>Processing the 38724 out of &gt;11.4 billion URLs in the 12-month CommonCrawl data which had</tspan><tspan dy="13" x="300">content_language=MRI</tspan><title>Processing the 38724 out of &gt;11.4 billion URLs in the 12-month CommonCrawl data which had content_language=MRI</title></text><g class="highcharts-data-labels highcharts-series-0" transform="translate(10,49) scale(1 1)" opacity="1"><path fill="none" d="M 383.76938279873923 74.98323239757032 C 378.76938279873923 74.98323239757032 372.8400579123436 80.35381024279047 364.6872361935496 87.73835477996819 L 356.53441447475564 95.12289931714591" stroke="#ff0000" stroke-width="1"></path><path fill="none" d="M 406.79522007421485 184.6271494648037 C 401.79522007421485 184.6271494648037 394.2063753841814 182.0954630168899 383.7717139353853 178.61439415100847 L 373.33705248658924 175.13332528512703" stroke="#a9a9a9" stroke-width="1"></path><path fill="none" d="M 396.018571483199 207.6999969482422 C 391.018571483199 207.6999969482422 384.5512261393511 202.94138448203648 375.1476971341777 197.2340521587122 L 365.74416812900427 191.5267198353879" stroke="#DDDF00" stroke-width="1"></path><path fill="none" d="M 194.76027515154266 203.92816951032583 C 199.76027515154266 203.92816951032583 206.73250757305308 200.00541855462666 216.31932715262994 194.61163599054032 L 225.9061467322068 189.21785342645398" stroke="#3ebe52" stroke-width="1"></path><g style="cursor:pointer;" transform="translate(389,65)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>discarded: 10290 URLs - 26.6%</tspan></text></g><g style="cursor:pointer;" transform="translate(412,175)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>greyListed: 2751 URLs - 7.1%</tspan></text></g><g style="cursor:pointer;" transform="translate(401,198)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>further pruned away: 4 URLs - 0.0%</tspan></text></g><g style="cursor:pointer;" transform="translate(0,194)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>final crawl seeds: 25679 URLs - 66.3%</tspan></text></g></g><g class="highcharts-legend" transform="translate(65,360)"><g><g><g class="highcharts-legend-item" transform="translate(8,3)"><text x="21" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start" y="15">discarded</text><rect x="0" y="4" width="16" height="12" fill="#ff0000"></rect></g><g class="highcharts-legend-item" transform="translate(103,3)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start">greyListed</text><rect x="0" y="4" width="16" height="12" fill="#a9a9a9"></rect></g><g class="highcharts-legend-item" transform="translate(201,3)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>further pruned away</tspan></text><rect x="0" y="4" width="16" height="12" fill="#DDDF00"></rect></g><g class="highcharts-legend-item" transform="translate(351,3)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>final crawl seeds</tspan></text><rect x="0" y="4" width="16" height="12" fill="#3ebe52"></rect></g></g></g></g><g class="highcharts-tooltip" display="none" style="cursor:default;padding:0;pointer-events:none;white-space:nowrap;" transform="translate(0,0)"><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.049999999999999996" stroke-width="5" transform="translate(1, 1)"></path><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.09999999999999999" stroke-width="3" transform="translate(1, 1)"></path><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.15" stroke-width="1" transform="translate(1, 1)"></path><path fill="rgb(249, 249, 249)" fill-opacity=" .85" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"></path><text x="8" style="font-size:12px;color:#000000;fill:#000000;" y="20"></text></g><text x="590" text-anchor="end" style="cursor:pointer;color:#909090;font-size:9px;fill:#909090;" y="395">meta-chart.com</text></svg>
Index: /other-projects/maori-lang-detection/mongodb-data/piechart_data.txt
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/piechart_data.txt	(revision 34005)
+++ /other-projects/maori-lang-detection/mongodb-data/piechart_data.txt	(revision 34006)
@@ -1,4 +1,4 @@
 https://www.rapidtables.com/tools/pie-chart.html
-https://www.meta-chart.com/pie#/data
+https://www.meta-chart.com/pie#/data (more powerful: can choose colours, display labels)
 
 "11.5 billion CC URLs" 
@@ -264,6 +264,6 @@
 
 
-wharariki:[143]/Scratch/ak19/maori-lang-detection/src>wc -l ../mongodb-data/InfoOnEmptyPagesNotInMongoDB.txt 
-589179 ../mongodb-data/InfoOnEmptyPagesNotInMongoDB.txt
+wharariki:[143]/Scratch/ak19/maori-lang-detection/src>wc -l ../mongodb-data/InfoOnEmptyPagesNotInMongoDB.csv 
+589179 ../mongodb-data/InfoOnEmptyPagesNotInMongoDB.csv
 
 - 17 lines at start that aren't about empty web pages in dump.txt = 589162 empty web pages
@@ -274,39 +274,125 @@
 Inspecting the csv file:
 
-wharariki:[198]/Scratch/ak19/maori-lang-detection/src>wc -l InfoOnEmptyPagesNotInMongoDB.txt 
-587082 InfoOnEmptyPagesNotInMongoDB.txt
+
+wharariki:[198]/Scratch/ak19/maori-lang-detection/src>wc -l InfoOnEmptyPagesNotInMongoDB.csv 
+587082 InfoOnEmptyPagesNotInMongoDB.csv
 -1 for column headings = 
 587081 empty pages
 
-wharariki:[183]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | wc
+
+# Listing of the nutch crawl status values:
+# https://nutch.apache.org/apidocs/apidocs-2.0/org/apache/nutch/crawl/CrawlStatus.html
+# But the only ones used are: status_unfetched|status_fetched|status_gone|status_redir|status_notmodified
+# Remainder are status (null). See examples in siteID 00154 later in this file.
+
+
+	wharariki:[298]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_unfetched" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	 555167 1117894 60067623
+	wharariki:[299]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	   3441   21326  579499
+	wharariki:[300]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	   5907   17929 1059096
+	wharariki:[301]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_notmodified" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	    291     873   51684
+	wharariki:[302]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_redir" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	  10959   32941 1927067
+
+	UNKNOWN STATUS (no status, protocolStatus or parseStatus info) forthe remainder:
+	wharariki:[291]/Scratch/ak19/maori-lang-detection/mongodb-data>egrep -v "status_unfetched|status_fetched|status_gone|status_redir|status_notmodified" InfoOnEmptyPagesNotInMongoDB.csv | less
+
+	wharariki:[304]/Scratch/ak19/maori-lang-detection/mongodb-data>egrep -v "status_unfetched|status_fetched|status_gone|status_redir|status_notmodified" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	  11317-1 (column heading)   22633  874662
+
+=> unfetched + fetched + gone + notmodified + redir + (UNKNOWN cause)
+=> 555167+3441+5907+291+10959+11316 = 587081 empty pages (CHECKED)
+
+wharariki:[183]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | wc
    3441   21326  579499
 
-	OF WHICH fetched but parseException:
-		wharariki:[187]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep "ParseException" | wc
+	wharariki:[315]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | grep "success/ok" | wc
+	   2065   10325  289719
+
+	wharariki:[317]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | grep "success/redirect" | wc
+	    150     750   33234
+
+	wharariki:[316]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | grep "failed/exception" | wc
+	    939    9390  219818
+[
+	all status_fetched with failed/exception are parseExceptions:
+		wharariki:[187]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep "ParseException" | wc
 		    939    9390  219818
-
-	wharariki:[214]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep -v "ParseException" | wc
-   		2502   11936  359681
-
-	ONLY OTHER OPTION FOR status_fetched IS SUCCESS:
-		wharariki:[211]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | egrep -v "ParseException|SUCCESS" | wc
-		      0       0       0
-
-wharariki:[188]/Scratch/ak19/maori-lang-detection/src>fgrep "status_unfetched" InfoOnEmptyPagesNotInMongoDB.txt | wc
+]
+
+All other kinds of status_fetched have no information besides SUCCESS (despite resulting in empty pages):
+	wharariki:[319]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | egrep -v "success/ok|success/redirect|failed/exception" | wc
+	    287     861   36728
+
+
+	All status_fetched that are not parseExceptions were SUCCESS:
+
+		wharariki:[214]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep -v "ParseException" | wc
+	   		2502   11936  359681
+
+		ONLY OTHER OPTION FOR status_fetched IS SUCCESS:
+			wharariki:[211]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | egrep -v "ParseException|SUCCESS" | wc
+			      0       0       0
+
+
+wharariki:[188]/Scratch/ak19/maori-lang-detection/src>fgrep "status_unfetched" InfoOnEmptyPagesNotInMongoDB.csv | wc
  555167 1117894 60067623
 
-	wharariki:[191]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.txt | wc
+	status_unfetched includes
+	- EXCEPTIONs like http error code 403 (Forbidden), 402 (Payment Required), 429 (Too Many Requests), 502 (Bad Gateway)
+	IOExceptions like unzipping issues (unzipBestEffort returned null)
+	Unknown Host Exceptions, SocketTimeoutException, ConnectionException connection refused,
+	SSL Exceptions like fatal alert/internal error, SSLHandshakeException (SSL security issues / invalid certificate), 
+	(EXCEPTION, args=[javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target])
+	- (null)
+
+	
+	wharariki:[309]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_unfetched" InfoOnEmptyPagesNotInMongoDB.csv | grep "EXCEPTION" | wc
+	   1847   11254  381055
+
+	
+
+status_redir_temp, status_redir_perm
+	- MOVED
+	- TEMP_MOVED 
+
+	wharariki:[327]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_redir" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	  10959   32941 1927067
+	wharariki:[328]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_redir_temp" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	   4872   14625  906162
+	wharariki:[329]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_redir_perm" InfoOnEmptyPagesNotInMongoDB.csv | wc
+	   6087   18316 1020905
+
+
+wharariki:[191]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv | wc
 	   5907   17929 1059096
-	wharariki:[192]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.txt  | fgrep "NOTFOUND" | wc
+
+[
+For status_gone, alternative values to NOTFOUND are GONE and ROBOTS_DENIED and ACCESS_DENIED:
+	wharariki:[200]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | fgrep -v "NOTFOUND" | less
+	wharariki:[204]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | egrep -v "NOTFOUND|GONE|ROBOTS_DENIED" | less
+
+wharariki:[342]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | egrep -v "NOTFOUND|GONE|ROBOTS_DENIED|ACCESS_DENIED" | wc
+      0       0       0
+]
+
+	wharariki:[192]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | fgrep "NOTFOUND" | wc
 	   3276    9828  695839
 
-For status_gone, alternative values to NOTFOUND are GONE and ROBOTS_DENIED and ACCESS_DENIED:
-	wharariki:[200]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.txt  | fgrep -v "NOTFOUND" | less
-	wharariki:[204]/Scratch/ak19/maori-lang-detection/src>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.txt  | egrep -v "NOTFOUND|GONE|ROBOTS_DENIED" | less
-
-
-wharariki:[196]/Scratch/ak19/maori-lang-detection/src>fgrep "status_notmodified" InfoOnEmptyPagesNotInMongoDB.txt  | wc
+	wharariki:[337]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | egrep "GONE" | wc
+	    374    1322   93428
+	wharariki:[338]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | egrep "ROBOTS_DENIED" | wc
+	   2253    6759  269069
+	wharariki:[339]/Scratch/ak19/maori-lang-detection/mongodb-data>fgrep "status_gone" InfoOnEmptyPagesNotInMongoDB.csv  | egrep "ACCESS_DENIED" | wc
+	      4      20     760
+
+= 5907
+
+wharariki:[196]/Scratch/ak19/maori-lang-detection/src>fgrep "status_notmodified" InfoOnEmptyPagesNotInMongoDB.csv  | wc
     291     873   51684
-wharariki:[197]/Scratch/ak19/maori-lang-detection/src>fgrep "status_notmodified" InfoOnEmptyPagesNotInMongoDB.txt  | fgrep "NOTMODIFIED" | wc
+wharariki:[197]/Scratch/ak19/maori-lang-detection/src>fgrep "status_notmodified" InfoOnEmptyPagesNotInMongoDB.csv  | fgrep "NOTMODIFIED" | wc
     291     873   51684
 
@@ -314,12 +400,12 @@
 ========
 
-wharariki:[222]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep -v "success/ok" | wc
+wharariki:[222]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep -v "success/ok" | wc
    1376   11001  289780
-wharariki:[223]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep "success/ok" | fgrep "ParseException" | wc
+wharariki:[223]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep "success/ok" | fgrep "ParseException" | wc
       0       0       0
 
 
-wharariki:[226]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep -v "success/ok" | fgrep -v "ParseException" | less
-wharariki:[227]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep -v "success/ok" | fgrep -v "ParseException" | wc
+wharariki:[226]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep -v "success/ok" | fgrep -v "ParseException" | less
+wharariki:[227]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep -v "success/ok" | fgrep -v "ParseException" | wc
     437    1611   69962
 
@@ -328,18 +414,73 @@
 - "failed/exception" for ParseException
 All failed/exception are ParseExceptions:
-wharariki:[233]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | fgrep "failed/exception" | fgrep -v "ParseException" | wc
+wharariki:[233]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | fgrep "failed/exception" | fgrep -v "ParseException" | wc
       0       0       0
 
 ALL THE status_fetched:
-wharariki:[234]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | wc
+wharariki:[234]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | wc
    3441   21326  579499
-wharariki:[244]/Scratch/ak19/maori-lang-detection/src>egrep "success/redirect|success/ok|failed/exception" InfoOnEmptyPagesNotInMongoDB.txt | wc
+wharariki:[244]/Scratch/ak19/maori-lang-detection/src>egrep "success/redirect|success/ok|failed/exception" InfoOnEmptyPagesNotInMongoDB.csv | wc
    3154   20465  542771
-wharariki:[245]/Scratch/ak19/maori-lang-detection/src>egrep -v "success/redirect|success/ok|failed/exception" InfoOnEmptyPagesNotInMongoDB.txt | less
-wharariki:[246]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" | egrep -v "success/redirect|success/ok|failed/exception" InfoOnEmptyPagesNotInMongoDB.txt | less
-
-wharariki:[247]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | egrep -v "success/redirect|success/ok|failed/exception" | lesswharariki:[248]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.txt | egrep -v "success/redirect|success/ok|failed/exception" | wc
+wharariki:[245]/Scratch/ak19/maori-lang-detection/src>egrep -v "success/redirect|success/ok|failed/exception" InfoOnEmptyPagesNotInMongoDB.csv | less
+wharariki:[246]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" | egrep -v "success/redirect|success/ok|failed/exception" InfoOnEmptyPagesNotInMongoDB.csv | less
+
+wharariki:[247]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | egrep -v "success/redirect|success/ok|failed/exception" | lesswharariki:[248]/Scratch/ak19/maori-lang-detection/src>fgrep "status_fetched" InfoOnEmptyPagesNotInMongoDB.csv | egrep -v "success/redirect|success/ok|failed/exception" | wc
     287     861   36728
 
 (No equivalent info to success/ok, success/redirect, failed/exception)
 
+-----------------------------
+No status information for many pages on site 00154, from the following point onwards (crawled too much of the site?):
+	http://m.biblepub.com/bibles/mb/19/81   key:    com.biblepub.m:http/bibles/mb/19/81
+	baseUrl:        null
+	status: 2 (status_fetched)
+	fetchTime:      1573978084279
+	prevFetchTime:  1571385510616
+	fetchInterval:  2592000
+	retriesSinceFetch:      0
+	modifiedTime:   0
+	prevModifiedTime:       0
+	protocolStatus: SUCCESS, args=[]
+	signature:      3e214d69ab677a676e40c2b91901acc9
+	parseStatus:    success/ok (1/0), args=[]
+	title:  Psalm 81 - Maori Bible - Bibles - BiblePub Mobile
+	score:  1.0
+	marker _injmrk_ :       y
+	marker _updmrk_ :       1571386061-31026
+	marker dist :   0
+	reprUrl:        null
+	batchId:        1571386061-31026
+	metadata CharEncodingForConversion :    utf-8
+	metadata OriginalCharEncoding :         utf-8
+	metadata _rs_ :         ^@^@^By
+	metadata _csh_ :        ^@^@^@^@
+	text:start:
+	Psalm 81 - Maori Bible - Bibles - BiblePub Mobile Maori Bible Books next back Psalm 81 1 Ki te tino kaiwhakatangi. Kititi. Na Ahapa. Kia kaha te waiata ki te Atua, ki to tatou kaha: kia hari te hamama ki 
+	te Atua o Hakopa. 2 Whakahuatia te himene, maua mai ki konei te timipera, te hapa reka me te hatere. 3 Whakatangihia te tetere i te kowhititanga marama, i te kinga o te marama, i to tatou ra hakari. 4 Ko 
+	te tikanga hoki tenei ma Iharaira, he mea whakarite na te Atua o Hakopa. 5 I whakatakotoria tenei e ia ma Hohepa hei whakaaturanga, i tona haerenga puta noa i te whenua o Ihipa: i rongo ai ahau ki reira i
+	 tetahi reo, kahore ahau i matau. 6 I tangohia mai e ahau tona pokohiwi i te pikaunga: whakarerea ake e ona ringa te kete. 7 I karanga koe ki ahau i te pouritanga, a kua ora koe i ahau; i whakahoki kupu a
+	hau ki a koe i te wahi ngaro o te whatitiri; i whakamatau i a koe ki nga wai o Meripa. (Hera. 8 Whakarongo, e taku iwi, a ka whakaatu ahau ki a koe: e Iharaira, ki te whakarongo koe ki ahau; 9 Aua tetahi 
+	atua ke i roto i a koe; kaua ano e koropiko ki te atua ke. 10 Ko Ihowa ahau, ko tou Atua, i arahina mai ai koe i te whenua o Ihipa: kia nui te kowhera o tou mangai, a maku e whakaki. 11 Otiia kihai taku i
+	wi i pai ki te whakarongo ki toku reo: kihai ano a Iharaira i aro ki ahau. 12 Na tukua atu ana ratou e ahau ki te maro o o ratou ngakau: a haere ana ratou i runga i o ratou whakaaro. 13 Aue, te whakarongo
+	 taku iwi ki ahau! Te haere a Iharaira i aku ara! 14 Penei e kore e aha kua whati i ahau te tara o o ratou hoariri: kua tahuri ano toku ringa ki o ratou hoariri. 15 Ko te hunga e kino ana ki a Ihowa kua n
+	gohengohe ki a ia: ko to ratou taima ia kua mau tonu. 16 Kua whangainga hoki ratou e ia ki te witi pai rawa, kua whakamakonatia ano koe e ahau ki te honi i roto i te kohatu. next back Contact Us - Full Si
+	te Â© 2013 BiblePub
+	text:end:
+
+	http://m.biblepub.com/bibles/mb/19/82   key:    com.biblepub.m:http/bibles/mb/19/82
+	baseUrl:        null
+	status: 1 (status_unfetched)
+	fetchTime:      1571386117381
+	prevFetchTime:  0
+	fetchInterval:  2592000
+	retriesSinceFetch:      0
+	modifiedTime:   0
+	prevModifiedTime:       0
+	protocolStatus: (null)
+	parseStatus:    (null)
+	title:  null
+	score:  0.0
+	marker dist :   1
+	reprUrl:        null
+	metadata _csh_ :        ^@^@^@^@
+
Index: /other-projects/maori-lang-detection/mongodb-data/piechart_data2.txt
===================================================================
--- /other-projects/maori-lang-detection/mongodb-data/piechart_data2.txt	(revision 34006)
+++ /other-projects/maori-lang-detection/mongodb-data/piechart_data2.txt	(revision 34006)
@@ -0,0 +1,26 @@
+https://www.rapidtables.com/tools/pie-chart.html
+
+Title: 38724 out of >11.4 billion URLs in 12-month CommonCrawl data had content_language=MRI
+data names: discarded_10290 greylisted_2751 pruned_4 crawlSeeds_25679
+data values: 10290 2751 4 25679
+slice text: (Percentage)
+
+
+------
+https://www.meta-chart.com/pie#/data
+
+Number of slices -> 4
+Series Unit: URLs
+
+Slice 1: discarded (red) 10290
+Slice 2: greyListed (grey) 2751
+Slice 3: further pruned away (yellow) 4
+Slice 4: final crawl seeds (green) 25679
+
+https://www.meta-chart.com/pie#/labels
+Graph title: Processing the 38724 out of >11.4 billion URLs in the 12-month CommonCrawl data which had content_language=MRI
+Slice Display data label display setting: Name, Value and Percent
+
+https://www.meta-chart.com/pie#/display
+Export as SVG and PNG
+Leave Sort setting at botton to "ORIG (default)"
