Index: other-projects/maori-lang-detection/mongodb-data/pieChart4b_sitesPreparedForCrawling.svg
===================================================================
--- other-projects/maori-lang-detection/mongodb-data/pieChart4b_sitesPreparedForCrawling.svg	(revision 34011)
+++ other-projects/maori-lang-detection/mongodb-data/pieChart4b_sitesPreparedForCrawling.svg	(revision 34011)
@@ -0,0 +1,1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" style="font-family:'lucida grande', 'lucida sans unicode', arial, helvetica, sans-serif;font-size:12px;" xmlns="http://www.w3.org/2000/svg" width="600" height="400"><desc>Created with Highcharts 4.2.7</desc><defs><clipPath id="highcharts-13"><rect x="1" y="1" width="578" height="296"></rect></clipPath></defs><rect x="10" y="36" width="580" height="298" fill="#ffffff"></rect><rect x="10" y="36" width="580" height="298" stroke="#026FC1" stroke-width="2" fill="none"></rect><g class="highcharts-series-group"><g class="highcharts-series highcharts-series-0" transform="translate(10,36) scale(1 1)"><path fill="#50B432" d="M 333.9883906273894 87.00000118225907 A 57 57 0 1 1 311.3209041939748 196.29396345106315 L 334 144 A 0 0 0 1 0 334 144 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#ED561B" d="M 311.26862157878634 196.27125821205743 A 57 57 0 0 1 307.7694493786801 194.6059108613053 L 334 144 A 0 0 0 0 0 334 144 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#DDDF00" d="M 307.71885659152736 194.5796550121024 A 57 57 0 0 1 307.56723611915606 194.50058409186522 L 334 144 A 0 0 0 0 0 334 144 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#7fff82" d="M 307.5167487598618 194.4741260820999 A 57 57 0 0 1 333.92082814352995 87.00005498408666 L 334 144 A 0 0 0 0 0 334 144 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path></g><g class="highcharts-markers highcharts-series-0" transform="translate(10,36) scale(1 1)"></g></g><text x="300" text-anchor="middle" class="highcharts-title" style="color:#000000;font-size:10px;font-family:'arial', sans-serif;fill:#000000;width:536px;" y="17"><tspan>Breakdown of the 1463 sites prepared for crawling</tspan></text><g class="highcharts-data-labels highcharts-series-0" transform="translate(10,36) scale(1 1)" opacity="1"><path fill="none" d="M 424.17700894533186 161.21657831317492 C 419.17700894533186 161.21657831317492 411.3446403066807 159.58746766368756 400.57513342853525 157.34744052064246 L 389.80562655038983 155.10741337759737" stroke="#50B432" stroke-width="1"></path><path fill="none" d="M 300.50157018871084 225.6999969482422 C 305.50157018871084 225.6999969482422 300.0291835781709 214.8230932562125 304.75929725715974 204.89202963825886 L 309.4894109361486 194.96096602030522" stroke="#ED561B" stroke-width="1"></path><path fill="none" d="M 267.01695533420315 204.5499973297119 C 272.01695533420315 204.5499973297119 297.4487323421958 213.53573968059135 302.5381493578394 203.7839278263318 L 307.62756637348303 194.03211597207223" stroke="#DDDF00" stroke-width="1"></path><path fill="none" d="M 244.52634259127765 122.68651389137781 C 249.52634259127765 122.68651389137781 257.2940352265625 124.600397671481 267.9746126000791 127.23198786912289 L 278.6551899735957 129.86357806676477" stroke="#7fff82" stroke-width="1"></path><g style="cursor:pointer;" transform="translate(429,151)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>completely crawled sites: 827</tspan></text></g><g style="cursor:pointer;" transform="translate(81,216)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>uncrawled irrelevant sites pruned away: 16</tspan></text></g><g style="cursor:pointer;" transform="translate(7,195)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>sites failed to properly crawl (text dump missing): 1</tspan></text></g><g style="cursor:pointer;" transform="translate(80,113)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>incompletely crawled sites: 619</tspan></text></g></g><g class="highcharts-legend" transform="translate(60,346)"><g><g><g class="highcharts-legend-item" transform="translate(8,3)"><text x="21" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start" y="15"><tspan>completely crawled sites</tspan></text><rect x="0" y="4" width="16" height="12" fill="#50B432"></rect></g><g class="highcharts-legend-item" transform="translate(182,3)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>uncrawled irrelevant sites pruned away</tspan></text><rect x="0" y="4" width="16" height="12" fill="#ED561B"></rect></g><g class="highcharts-legend-item" transform="translate(8,17)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>sites failed to properly crawl (text dump missing)</tspan></text><rect x="0" y="4" width="16" height="12" fill="#DDDF00"></rect></g><g class="highcharts-legend-item" transform="translate(309,17)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>incompletely crawled sites</tspan></text><rect x="0" y="4" width="16" height="12" fill="#7fff82"></rect></g></g></g></g><g class="highcharts-tooltip" display="none" style="cursor:default;padding:0;pointer-events:none;white-space:nowrap;" transform="translate(0,0)"><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.049999999999999996" stroke-width="5" transform="translate(1, 1)"></path><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.09999999999999999" stroke-width="3" transform="translate(1, 1)"></path><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.15" stroke-width="1" transform="translate(1, 1)"></path><path fill="rgb(249, 249, 249)" fill-opacity=" .85" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"></path><text x="8" style="font-size:12px;color:#000000;fill:#000000;" y="20"></text></g><text x="590" text-anchor="end" style="cursor:pointer;color:#909090;font-size:9px;fill:#909090;" y="395">meta-chart.com</text></svg>
Index: other-projects/maori-lang-detection/mongodb-data/pieChart5b_sitesPreparedForCrawling.svg
===================================================================
--- other-projects/maori-lang-detection/mongodb-data/pieChart5b_sitesPreparedForCrawling.svg	(revision 34011)
+++ other-projects/maori-lang-detection/mongodb-data/pieChart5b_sitesPreparedForCrawling.svg	(revision 34011)
@@ -0,0 +1,1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" style="font-family:'lucida grande', 'lucida sans unicode', arial, helvetica, sans-serif;font-size:12px;" xmlns="http://www.w3.org/2000/svg" width="600" height="400"><desc>Created with Highcharts 4.2.7</desc><defs><clipPath id="highcharts-13"><rect x="1" y="1" width="578" height="282"></rect></clipPath></defs><rect x="10" y="36" width="580" height="284" fill="#ffffff"></rect><rect x="10" y="36" width="580" height="284" stroke="#026FC1" stroke-width="2" fill="none"></rect><g class="highcharts-series-group"><g class="highcharts-series highcharts-series-0" transform="translate(10,36) scale(1 1)"><path fill="#50B432" d="M 342.9918530718522 108.00000082965548 A 40 40 0 1 1 304.81731609417767 159.91984268973619 L 343 148 A 0 0 0 1 0 343 148 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#ED561B" d="M 304.80541534481495 159.8816540522733 A 40 40 0 0 1 304.086357245126 157.25896363239656 L 343 148 A 0 0 0 0 0 343 148 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#DDDF00" d="M 304.07711773985653 157.22004536664588 A 40 40 0 0 1 304.04963279808555 157.1032354048453 L 343 148 A 0 0 0 0 0 343 148 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#24CBE5" d="M 304.0405490393799 157.0642804925178 A 40 40 0 0 1 318.9396309136573 116.0453659162092 L 343 148 A 0 0 0 0 0 343 148 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path><path fill="#0036ff" d="M 318.97159757259885 116.02132152844862 A 40 40 0 0 1 342.9444408024771 108.00003858532398 L 343 148 A 0 0 0 0 0 343 148 Z" stroke="#FFFFFF" stroke-width="1" stroke-linejoin="round" transform="translate(0,0)"></path></g><g class="highcharts-markers highcharts-series-0" transform="translate(10,36) scale(1 1)"></g></g><text x="300" text-anchor="middle" class="highcharts-title" style="color:#000000;font-size:10px;font-family:'arial', sans-serif;fill:#000000;width:536px;" y="17"><tspan>Breakdown of the 1463 sites prepared for crawling</tspan></text><g class="highcharts-data-labels highcharts-series-0" transform="translate(10,36) scale(1 1)" opacity="1"><path fill="none" d="M 404.373868915765 188.99683004119885 C 399.373868915765 188.99683004119885 392.9311410396775 184.25433517934755 384.07239021005734 177.73340474430202 L 375.21363938043714 171.2124743092565" stroke="#50B432" stroke-width="1"></path><path fill="none" d="M 277.81746402505604 183.24999809265137 C 282.81746402505604 183.24999809265137 283.20253697011185 163.87874888961804 293.8117642818662 160.97284182855677 L 304.42099159362056 158.0669347674955" stroke="#ED561B" stroke-width="1"></path><path fill="none" d="M 269.5395001146233 162.0999984741211 C 274.5395001146233 162.0999984741211 282.64460008182493 161.6854044960711 293.35281619633986 159.16863918225204 L 304.0610323108548 156.65187386843297" stroke="#DDDF00" stroke-width="1"></path><path fill="none" d="M 272.21866085153755 123.56643737687676 C 277.21866085153755 123.56643737687676 284.73652818279044 126.30170167666228 295.0735957632631 130.06269008886736 L 305.41066334373573 133.82367850107244" stroke="#24CBE5" stroke-width="1"></path><path fill="none" d="M 315.8405523462715 81.10000843613257 C 320.8405523462715 81.10000843613257 323.3730606495547 88.68857890057457 326.8552595665692 99.1228632891823 L 330.3374584835837 109.55714767779004" stroke="#0036ff" stroke-width="1"></path><g style="cursor:pointer;" transform="translate(409,179)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>crawled sites with text content: 1027  - 70.2%</tspan></text></g><g style="cursor:pointer;" transform="translate(27,173)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>uncrawled irrelevant sites pruned away: 16  - 1.1%</tspan></text></g><g style="cursor:pointer;" transform="translate(-22,152)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>sites failed to properly crawl (text dump missing): 1  - 0.1%</tspan></text></g><g style="cursor:pointer;" transform="translate(-38,114)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>crawled sites where dump.txt had no text content: 269  - 18.4%</tspan></text></g><g style="cursor:pointer;" transform="translate(55,71)"><text x="5" style="font-size: 10px; font-weight: bold; font-family: 'arial', sans-serif; color: rgb(0, 0, 0); fill: rgb(0, 0, 0); text-rendering: geometricprecision; text-shadow: none;" y="15"><tspan>crawled sites with empty dump.txt files: 150  - 10.3%</tspan></text></g></g><g class="highcharts-legend" transform="translate(27,332)"><g><g><g class="highcharts-legend-item" transform="translate(8,3)"><text x="21" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start" y="15"><tspan>crawled sites with text content</tspan></text><rect x="0" y="4" width="16" height="12" fill="#50B432"></rect></g><g class="highcharts-legend-item" transform="translate(210,3)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>uncrawled irrelevant sites pruned away</tspan></text><rect x="0" y="4" width="16" height="12" fill="#ED561B"></rect></g><g class="highcharts-legend-item" transform="translate(8,17)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>sites failed to properly crawl (text dump missing)</tspan></text><rect x="0" y="4" width="16" height="12" fill="#DDDF00"></rect></g><g class="highcharts-legend-item" transform="translate(8,31)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>crawled sites where dump.txt had no text content</tspan></text><rect x="0" y="4" width="16" height="12" fill="#24CBE5"></rect></g><g class="highcharts-legend-item" transform="translate(312,31)"><text x="21" y="15" style="color:#333;font-size:9pt;font-weight:normal;cursor:pointer;font-family:'lucida grande','lucida sans unicode',arial,helvetica,sans-serif;text-transform:none;fill:#333;" text-anchor="start"><tspan>crawled sites with empty dump.txt files</tspan></text><rect x="0" y="4" width="16" height="12" fill="#0036ff"></rect></g></g></g></g><g class="highcharts-tooltip" display="none" style="cursor:default;padding:0;pointer-events:none;white-space:nowrap;" transform="translate(0,0)"><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.049999999999999996" stroke-width="5" transform="translate(1, 1)"></path><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.09999999999999999" stroke-width="3" transform="translate(1, 1)"></path><path fill="none" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"  stroke="black" stroke-opacity="0.15" stroke-width="1" transform="translate(1, 1)"></path><path fill="rgb(249, 249, 249)" fill-opacity=" .85" d="M 3.5 0.5 L 13.5 0.5 C 16.5 0.5 16.5 0.5 16.5 3.5 L 16.5 13.5 C 16.5 16.5 16.5 16.5 13.5 16.5 L 3.5 16.5 C 0.5 16.5 0.5 16.5 0.5 13.5 L 0.5 3.5 C 0.5 0.5 0.5 0.5 3.5 0.5"></path><text x="8" style="font-size:12px;color:#000000;fill:#000000;" y="20"></text></g><text x="590" text-anchor="end" style="cursor:pointer;color:#909090;font-size:9px;fill:#909090;" y="395">meta-chart.com</text></svg>
Index: other-projects/maori-lang-detection/mongodb-data/piechart_data.txt
===================================================================
--- other-projects/maori-lang-detection/mongodb-data/piechart_data.txt	(revision 34007)
+++ other-projects/maori-lang-detection/mongodb-data/piechart_data.txt	(revision 34011)
@@ -487,2 +487,42 @@
 	metadata _csh_ :        ^@^@^@^@
 
+
+------------
+
+
+Would like to do something like:
+wharariki:[378]/Scratch/ak19/maori-lang-detection/crawled>find . -name UNFINISHED | grep -l text:start */dump.txt | wc
+
+
+Would like to find how many and which of the unfinished websites had a dump.txt with no text content
+AND how many of the completely crawled websites had a dump.txt with no text content.
+
+
+--------------
+
+
+
+wharariki:[393]/Scratch/ak19/maori-lang-detection/crawled>grep -l "text:start" */dump.txt
+
+wharariki:[388]/Scratch/ak19/maori-lang-detection/crawled>less 01461/dump.txt 
+wharariki:[389]/Scratch/ak19/maori-lang-detection/crawled>less 01453/dump.txt 
+wharariki:[390]/Scratch/ak19/maori-lang-detection/crawled>less 01447/dump.txt 
+wharariki:[391]/Scratch/ak19/maori-lang-detection/crawled>less 01446/dump.txt 
+wharariki:[392]/Scratch/ak19/maori-lang-detection/crawled>less 01445/dump.txt 
+ 
+# All the dump.txt files that are 0 bytes (no content):
+# https://stackoverflow.com/questions/15703664/find-all-zero-byte-files-in-directory-and-subdirectories
+wharariki:[396]/Scratch/ak19/maori-lang-detection/crawled>find . -name "dump.txt" -size 0 | sort | wc
+    150     150    2550
+
+
+Examples of empty dump.txt files (listed with: find . -name "dump.txt" -size 0 | sort):
+	wharariki:[400]/Scratch/ak19/maori-lang-detection/crawled>less ../tmp/to_crawl.THE_VERSION_USED/sites/00014/seedURLs.txt 
+	wharariki:[401]/Scratch/ak19/maori-lang-detection/crawled>less ../tmp/to_crawl.THE_VERSION_USED/sites/01461/seedURLs.txt 
+	wharariki:[402]/Scratch/ak19/maori-lang-detection/crawled>less ../tmp/to_crawl.THE_VERSION_USED/sites/01447/seedURLs.txt 
+	wharariki:[403]/Scratch/ak19/maori-lang-detection/crawled>less ../tmp/to_crawl.THE_VERSION_USED/sites/01422/seedURLs.txt 
+	
+
+
+
+=======
Index: other-projects/maori-lang-detection/mongodb-data/piechart_data2.txt
===================================================================
--- other-projects/maori-lang-detection/mongodb-data/piechart_data2.txt	(revision 34007)
+++ other-projects/maori-lang-detection/mongodb-data/piechart_data2.txt	(revision 34011)
@@ -106,2 +106,44 @@
 07. 11316 empty status unknown
 
+
+
+============
+
+
+1463 sites prepared for crawling
+1447 sites crawled (16 were autotranslated or otherwise irrelevant)
+1446 crawled sites contained dump.txt files (1 site was missing dump.txt) - 1446 sites in mongodb
+619 sites not finished crawling
+1027 sites where dump.txt contained text:start denoting text content, so 419 sites with no text content
+
+
+
+16 uncrawled irrelevant sites pruned away
+1 failed crawl of site (text dump missing)
+1446 crawled sites in MongoDB
+
+
+Graph title: Breakdown of the 1463 sites prepared for crawling
+* 16 uncrawled irrelevant sites pruned away
+* 1 sites failed to properly crawl (text dump missing)
+* 619 incompletely crawled sites
+* 827 completely crawled sites
+
+
+Graph title: Breakdown of the 1463 sites prepared for crawling
+* 16 uncrawled irrelevant sites pruned away
+* 1 sites failed to properly crawl (text dump missing)
+* 419 crawled sites with no text content
+	- 150 crawled sites with 0-size dump.txt files [crawled sites with empty dump.txt files] See below.
+	- 269 crawled sites where dump.txt had no text content
+* 1027 crawled sites with text content (WebPages collection in MongoDB will have webpage documents for these sites)
+
+
+
+ 
+# All the dump.txt files that are 0 bytes (no content):
+# https://stackoverflow.com/questions/15703664/find-all-zero-byte-files-in-directory-and-subdirectories
+wharariki:[396]/Scratch/ak19/maori-lang-detection/crawled>find . -name "dump.txt" -size 0 | sort | wc
+    150     150    2550
+
+
