Index: /gs3-extensions/maori-lang-detection/hdfs-cc-work/GS_README.TXT
===================================================================
--- /gs3-extensions/maori-lang-detection/hdfs-cc-work/GS_README.TXT	(revision 33607)
+++ /gs3-extensions/maori-lang-detection/hdfs-cc-work/GS_README.TXT	(revision 33608)
@@ -665,4 +665,46 @@
 
 
+--------------------------------------------------------
+K. Reading data from hbase tables and backing up hbase
+--------------------------------------------------------
+
+* Backing up HBase database:
+https://blogs.msdn.microsoft.com/data_otaku/2016/12/21/working-with-the-hbase-import-and-export-utility/
+
+* From an image at http://dwgeek.com/read-hbase-table-using-hbase-shell-get-command.html/
+to see the contents of a table, inside hbase shell, type:
+
+   scan 'tablename'
+
+e.g. scan '01066_webpage' and hit enter.
+
+
+To list tables and see their "column families" (I don't yet understand what this is):
+
+hbase shell
+hbase(main):001:0> list
+
+hbase(main):002:0> describe '01066_webpage'
+Table 01066_webpage is ENABLED                                                                                                                                                                              
+01066_webpage                                                                                                                                                                                               
+COLUMN FAMILIES DESCRIPTION                                                                                                                                                                                 
+{NAME => 'f', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOCK
+CACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                            
+{NAME => 'h', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOCK
+CACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                            
+{NAME => 'il', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOC
+KCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                           
+{NAME => 'mk', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOC
+KCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                           
+{NAME => 'mtdt', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BL
+OCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                         
+{NAME => 'ol', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOC
+KCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                           
+{NAME => 'p', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOCK
+CACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                            
+{NAME => 's', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOCK
+CACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}                                                                                                                                            
+8 row(s) in 0.1180 seconds
+
 
 -----------------------EOF------------------------
Index: /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh	(revision 33607)
+++ /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh	(revision 33608)
@@ -64,5 +64,6 @@
 	    echo "2. copy the regex-urlfilter file:" 2>&1 | tee -a ${siteDir}UNFINISHED
 	    echo "   cp $NUTCH_URLFILTER_TEMPLATE $NUTCH_URLFILTER_FILE" 2>&1 | tee -a ${siteDir}UNFINISHED
-	    echo "3. Adjust # crawl iterations in old crawl command:\n$crawl_cmd" 2>&1 | tee -a ${siteDir}UNFINISHED
+	    echo "3. Adjust # crawl iterations in old crawl command:" 2>&1 | tee -a ${siteDir}UNFINISHED
+	    echo "   $crawl_cmd" 2>&1 | tee -a ${siteDir}UNFINISHED
 	fi
 	
Index: /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/exportHBase.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/exportHBase.sh	(revision 33608)
+++ /gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/exportHBase.sh	(revision 33608)
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# This script exports the Hbase tables for the crawled websites' webpages to files called part-m-*
+# which presumably can be reimported into Hbase.
+# See https://blogs.msdn.microsoft.com/data_otaku/2016/12/21/working-with-the-hbase-import-and-export-utility/
+# We want to backup our HBase tables that were created by crawling, so we can finally retire the
+# vagrant Virtual Machines we used to crawl web sites.
+
+
+# When doing this on the command line:
+#
+# hdfs dfs -mkdir /user/vagrant
+# EXPORT CMD:
+# vagrant@node5:~/crawled$ hbase org.apache.hadoop.hbase.mapreduce.Export "01066_webpage" "/user/vagrant/01066"
+#
+# Note that when the script runs the above with double quotes around the tablename or outputdir
+# hadoop doesn't like it. But such a cmd was okay from the command line.
+#
+# OUTPUT:
+# vagrant@node5:~/crawled$ hdfs dfs -ls /user/vagrant/01066
+# Found 2 items
+# -rw-r--r--   1 vagrant supergroup          0 2019-10-30 08:29 /user/vagrant/01066/_SUCCESS
+# -rw-r--r--   1 vagrant supergroup  184788017 2019-10-30 08:29 /user/vagrant/01066/part-m-00000
+
+
+sitesDir=crawled
+echo "SITES DIR (INPUT): $sitesDir"
+
+logOut=$sitesDir/exportHBase.log
+
+# output dir is on hdfs (i.e. hdfs:///user/vagrant)
+outputDir=/user/vagrant
+
+exportCmd="hbase org.apache.hadoop.hbase.mapreduce.Export"
+
+function exportAll() {
+
+    if $(hdfs dfs -test -d "$outputDir"); then
+	echo "Output directory on hdfs $outputDir already exists."
+    else
+	echo "Creating directory on hfds $outputDir..."
+	hdfs dfs -mkdir $outputDir
+    fi
+    
+    # https://stackoverflow.com/questions/4000613/perform-an-action-in-every-sub-directory-using-bash
+    for siteDir in $sitesDir/*/; do
+	
+	# to get crawl_id like 00001 from $siteDir like to_crawl/sites/00001/
+	# Remove the $sitesDir prefix of to_crawl/sites followed by /,
+	# Next remove the / suffix that remains
+	crawlId=${siteDir#"$sitesDir/"}
+	crawlId=${crawlId%/}
+	
+	echo "$siteDir $crawlId"
+
+	fullExportCmd="$exportCmd ${crawlId}_webpage $outputDir/$crawlId"
+	
+	echo "Going to run command:"
+	#echo "   $exportCmd \"${crawlId}_webpage\" \"$outputDir/$crawlId\"" 2>&1 | tee $logOut
+	echo "    $fullExportCmd" 2>&1 | tee $logOut
+
+	# run the export from hbase - produces a file called part-m-00000 (perhaps multiple files called part-m-*)
+	$fullExportCmd 2>&1 | tee -a $logOut
+
+	
+	# if export was successful, should have file _SUCCESS in outputdir
+	# In that case, copy the outputfile(s) named part-m-* into crawl folder
+	if $(hdfs dfs -test -f "$outputDir/$crawlId/_SUCCESS"); then
+	    hdfs dfs -get $outputDir/$crawlId/part-m-* ${sitesDir}/$crawlId/.
+	else
+	    echo "ERROR: EXPORT OF $crawlId FAILED" 2>&1 | tee -a $logOut
+	fi
+
+    done
+}
+
+
+# script begins here
+exportAll
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java	(revision 33607)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MaoriTextDetector.java	(revision 33608)
@@ -81,5 +81,5 @@
 
 
-	// we'll be storing just those sentences in text that are in MÄori. 
+	// we'll be storing just those sentences in the text that are in MÄori. 
 	
 	// OpenNLP language detection works best with a minimum of 2 sentences
