source: gs3-extensions/maori-lang-detection/bin/script/create-uniq-WET-urls-file.sh@ 33413

#!/bin/bash

# Need to run this script from the top level folder of this extension
# This script needs to be provided with the nz-only-TLDs-<date>.txt file produced by running get_commoncrawl_nz_urls.sh
if [ -z $1 ]; then
    echo ""
    echo "Usage: $0 nz-only-TLDs-<date>.txt"
    echo "   where the nz-only-TLDs-<date>.txt file is produced by running get_commoncrawl_nz_urls.sh."
    echo ""
    exit 1
fi

uniq_urls_file=uniq-tld-nz-WET-urls-`date +%F`.txt
echo "Creating file $uniq_urls_file containing just the unique WET gz urls for the .nz TLD..."

# cat is unnecessary: https://stackoverflow.com/questions/1915636/is-there-a-way-to-uniq-by-column/1915750
#    "You can dump the cat! Rather than piping into <the subsequent process>, just let <the subsequent process> read the file using <. Piping through cat is a common unnecessary complication used by novices. For large amounts of data there's a performance effect to be had."

#<$1 cut -d ' ' -f18 | sed -r 's@\.?"(,|})\s*$@@' | sed -r 's@^\s*"@@' > $uniq_urls_file
#sort -u $uniq_urls_file > tmp.txt

# The first cut grabs the filename field of the json, containing the relative path to the WET file
# The sed operations remove any starting and terminating double quotes followed by , or }
# Then we get the uniq urls out of all that.

# The above is imperfect because space is not a proper field separator (some json property values
# contain spaces)
# https://unix.stackexchange.com/questions/155952/should-i-use-piping-or-redirection-for-input-to-sort
tmpfile=tmp_WET_urls.txt

<$1 sed -r 's@^.*"filename":\s*"@@' | sed -r 's@\.?"(,|}).*$@@' > $tmpfile
sort -u $tmpfile > sorted_WET_urls.txt

rm $tmpfile

# remove all lines containing paths like /crawldiagnostics/ and /robotstxt/
# https://stackoverflow.com/questions/17049962/delete-line-containing-one-of-multiple-strings

# And convert links like
#   https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2019-30/segments/1563195523840.34/warc/CC-MAIN-20190715175205-20190715200159-00000.warc.gz
# to:
#   https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2019-30/segments/1563195523840.34/wet/CC-MAIN-20190715175205-20190715200159-00000.warc.wet.gz

awk '!/crawldiagnostics|robotstxt/' sorted_WET_urls.txt | sed -r 's@\/warc\/CC-MAIN-@/wet/CC-MAIN-@' | sed -r 's@\.warc\[email protected]@' > $uniq_urls_file

rm sorted_WET_urls.txt

# All urls need to be prefixed with:
#ccprefix=https://commoncrawl.s3.amazonaws.com/


# From each line of json of the input file, we want the filename value filed containing
# the relative path to the gzipped WET file.
# The first sed grabs everything after the filename field in each line/json
# The second sed operation remove any terminating double quotes followed by , or }
# Then we sort -u to get the uniq urls out of all that.

echo "File $uniq_urls_file containing just the unique commoncrawl WET urls of the .nz TLD has now been created."
num_uniq_urls=`wc -l $uniq_urls_file`
total_urls=`wc -l $1`
echo ""
echo ""
echo "Summary:"
echo "There were $num_uniq_urls unique WET urls"
echo "out of a total of $total_urls urls in $1."
echo ""
echo ""

exit 0