Index: /gs3-extensions/maori-lang-detection/bin/script/create-uniq-WET-urls-file.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/bin/script/create-uniq-WET-urls-file.sh	(revision 33413)
+++ /gs3-extensions/maori-lang-detection/bin/script/create-uniq-WET-urls-file.sh	(revision 33413)
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Need to run this script from the top level folder of this extension
+# This script needs to be provided with the nz-only-TLDs-<date>.txt file produced by running get_commoncrawl_nz_urls.sh
+if [ -z $1 ]; then
+    echo ""
+    echo "Usage: $0 nz-only-TLDs-<date>.txt"
+    echo "   where the nz-only-TLDs-<date>.txt file is produced by running get_commoncrawl_nz_urls.sh."
+    echo ""
+    exit 1
+fi
+
+uniq_urls_file=uniq-tld-nz-WET-urls-`date +%F`.txt
+echo "Creating file $uniq_urls_file containing just the unique WET gz urls for the .nz TLD..."
+
+# cat is unnecessary: https://stackoverflow.com/questions/1915636/is-there-a-way-to-uniq-by-column/1915750
+#    "You can dump the cat! Rather than piping into <the subsequent process>, just let <the subsequent process> read the file using <. Piping through cat is a common unnecessary complication used by novices. For large amounts of data there's a performance effect to be had."
+
+#<$1 cut -d ' ' -f18 | sed -r 's@\.?"(,|})\s*$@@' | sed -r 's@^\s*"@@' > $uniq_urls_file
+#sort -u $uniq_urls_file > tmp.txt
+
+# The first cut grabs the filename field of the json, containing the relative path to the WET file
+# The sed operations remove any starting and terminating double quotes followed by , or }
+# Then we get the uniq urls out of all that.
+
+# The above is imperfect because space is not a proper field separator (some json property values
+# contain spaces)
+# https://unix.stackexchange.com/questions/155952/should-i-use-piping-or-redirection-for-input-to-sort
+tmpfile=tmp_WET_urls.txt
+
+<$1 sed -r 's@^.*"filename":\s*"@@' | sed -r 's@\.?"(,|}).*$@@' > $tmpfile
+sort -u $tmpfile > sorted_WET_urls.txt
+
+rm $tmpfile
+
+# remove all lines containing paths like /crawldiagnostics/ and /robotstxt/
+# https://stackoverflow.com/questions/17049962/delete-line-containing-one-of-multiple-strings
+
+# And convert links like
+#   https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2019-30/segments/1563195523840.34/warc/CC-MAIN-20190715175205-20190715200159-00000.warc.gz
+# to:
+#   https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2019-30/segments/1563195523840.34/wet/CC-MAIN-20190715175205-20190715200159-00000.warc.wet.gz
+
+awk '!/crawldiagnostics|robotstxt/' sorted_WET_urls.txt | sed -r 's@\/warc\/CC-MAIN-@/wet/CC-MAIN-@' | sed -r 's@\.warc\.gz@.warc.wet.gz@' > $uniq_urls_file
+
+rm sorted_WET_urls.txt
+
+# All urls need to be prefixed with:
+#ccprefix=https://commoncrawl.s3.amazonaws.com/
+
+
+# From each line of json of the input file, we want the filename value filed containing
+# the relative path to the gzipped WET file.
+# The first sed grabs everything after the filename field in each line/json
+# The second sed operation remove any terminating double quotes followed by , or }
+# Then we sort -u to get the uniq urls out of all that.
+
+echo "File $uniq_urls_file containing just the unique commoncrawl WET urls of the .nz TLD has now been created."
+num_uniq_urls=`wc -l $uniq_urls_file`
+total_urls=`wc -l $1`
+echo ""
+echo ""
+echo "Summary:"
+echo "There were $num_uniq_urls unique WET urls"
+echo "out of a total of $total_urls urls in $1."
+echo ""
+echo ""
+
+exit 0
Index: /gs3-extensions/maori-lang-detection/bin/script/create-uniq-nz-urls-file.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/bin/script/create-uniq-nz-urls-file.sh	(revision 33413)
+++ /gs3-extensions/maori-lang-detection/bin/script/create-uniq-nz-urls-file.sh	(revision 33413)
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Need to run this script from the top level folder of this extension
+# This script needs to be provided with the nz-only-TLDs-<date>.txt file produced by running get_commoncrawl_nz_urls.sh
+if [ -z $1 ]; then
+    echo ""
+    echo "Usage: $0 nz-only-TLDs-<date>.txt"
+    echo "   where the nz-only-TLDs-<date>.txt file is produced by running get_commoncrawl_nz_urls.sh."
+    echo ""
+    exit 1
+fi
+
+uniq_urls_file=uniq-tld-nz-urls-`date +%F`.txt
+echo "Creating file $uniq_urls_file containing just the unique domains and subdomains..."
+
+# cat is unnecessary: https://stackoverflow.com/questions/1915636/is-there-a-way-to-uniq-by-column/1915750
+#    "You can dump the cat! Rather than piping into <the subsequent process>, just let <the subsequent process> read the file using <. Piping through cat is a common unnecessary complication used by novices. For large amounts of data there's a performance effect to be had."
+
+#cat $1 | cut -d ' ' -f4 | cut -d/ -f3 | sed -r 's@\.?",\s*$@@' | sed -r 's@^www\.@@' | uniq > $uniq_urls_file
+<$1 cut -d ' ' -f4 | cut -d/ -f3 | sed -r 's@\.?",\s*$@@' | sed -r 's@^www\.@@' | uniq > $uniq_urls_file
+
+
+# The first cut grabs the url field of the json.
+# The second cut grabs the domain name from the url (located between first // and immediately subsequent /).
+# The first sed process then removes any trailing . (e.g. "massey.ac.nz." becomes "massey.ac.nz") followed by ", and optional spaces before the end,
+# and the final sed removes any "www." prefix.
+# Then we get the uniq urls out of all that.
+
+echo "File $uniq_urls_file containing just the unique .nz sites (domains and subdomains) to be used as seed urls has now been created."
+num_uniq_urls=`wc -l $uniq_urls_file`
+total_urls=`wc -l $1`
+echo ""
+echo ""
+echo "Summary:"
+echo "There were $num_uniq_urls unique sites"
+echo "out of a total of $total_urls urls in $1."
+echo ""
+echo ""
+
+exit 0
Index: /gs3-extensions/maori-lang-detection/bin/script/get_commoncrawl_nz_urls.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/bin/script/get_commoncrawl_nz_urls.sh	(revision 33412)
+++ /gs3-extensions/maori-lang-detection/bin/script/get_commoncrawl_nz_urls.sh	(revision 33413)
@@ -156,28 +156,28 @@
 echo ""
 
-uniq_urls_file=uniq-tld-nz-urls-`date +%F`.txt
-echo "Creating file $uniq_urls_file containing just the unique domains and subdomains..."
-#cat $outfile | cut -d ' ' -f4 | cut -d/ -f3 | sed -r 's@\.?",\s*$@@' | sed -r 's@^www\.@@' | uniq > $uniq_urls_file
-<$outfile cut -d ' ' -f4 | cut -d/ -f3 | sed -r 's@\.?",\s*$@@' | sed -r 's@^www\.@@' | uniq > $uniq_urls_file
+# uniq_urls_file=uniq-tld-nz-urls-`date +%F`.txt
+# echo "Creating file $uniq_urls_file containing just the unique domains and subdomains..."
+# #cat $outfile | cut -d ' ' -f4 | cut -d/ -f3 | sed -r 's@\.?",\s*$@@' | sed -r 's@^www\.@@' | uniq > $uniq_urls_file
+# <$outfile cut -d ' ' -f4 | cut -d/ -f3 | sed -r 's@\.?",\s*$@@' | sed -r 's@^www\.@@' | uniq > $uniq_urls_file
 
-# cat is unnecessary: https://stackoverflow.com/questions/1915636/is-there-a-way-to-uniq-by-column/1915750
-#    "You can dump the cat! Rather than piping into <the subsequent process>, just let <the subsequent process> read the file using <. Piping through cat is a common unnecessary complication used by novices. For large amounts of data there's a performance effect to be had."
+# # cat is unnecessary: https://stackoverflow.com/questions/1915636/is-there-a-way-to-uniq-by-column/1915750
+# #    "You can dump the cat! Rather than piping into <the subsequent process>, just let <the subsequent process> read the file using <. Piping through cat is a common unnecessary complication used by novices. For large amounts of data there's a performance effect to be had."
 
-# The first cut grabs the url field of the json.
-# The second cut grabs the domain name from the url (located between first // and immediately subsequent /).
-# The first sed process then removes any trailing . (e.g. "massey.ac.nz." becomes "massey.ac.nz") followed by ", and optional spaces before the end,
-# and the final sed removes any "www." prefix.
-# Then we get the uniq urls out of all that.
+# # The first cut grabs the url field of the json.
+# # The second cut grabs the domain name from the url (located between first // and immediately subsequent /).
+# # The first sed process then removes any trailing . (e.g. "massey.ac.nz." becomes "massey.ac.nz") followed by ", and optional spaces before the end,
+# # and the final sed removes any "www." prefix.
+# # Then we get the uniq urls out of all that.
 
-echo "File $uniq_urls_file containing just the unique .nz sites (domains and subdomains) to be used as seed urls has now been created."
-num_uniq_urls=`wc -l $uniq_urls_file`
-total_urls=`wc -l $outfile`
-echo ""
-echo ""
-echo "Summary:"
-echo "There were $num_uniq_urls unique sites"
-echo "out of a total of $total_urls urls in $outfile."
-echo ""
-echo ""
+# echo "File $uniq_urls_file containing just the unique .nz sites (domains and subdomains) to be used as seed urls has now been created."
+# num_uniq_urls=`wc -l $uniq_urls_file`
+# total_urls=`wc -l $outfile`
+# echo ""
+# echo ""
+# echo "Summary:"
+# echo "There were $num_uniq_urls unique sites"
+# echo "out of a total of $total_urls urls in $outfile."
+# echo ""
+# echo ""
 
 echo ""
