Index: /gs3-extensions/maori-lang-detection/bin/script/get_maori_WET_records_for_crawl.sh
===================================================================
--- /gs3-extensions/maori-lang-detection/bin/script/get_maori_WET_records_for_crawl.sh	(revision 33494)
+++ /gs3-extensions/maori-lang-detection/bin/script/get_maori_WET_records_for_crawl.sh	(revision 33494)
@@ -0,0 +1,182 @@
+#!/bin/bash
+
+# convert URL index with Spark on Yarn
+
+# Table format configuration
+FORMAT=${FORMAT:-parquet} # parquet, orc
+NESTED="$NESTED"          # "" (empty) or --useNestedSchema
+COMPRS=${COMPRS:-gzip}    # gzip, snappy, lzo, none
+PARTITION_BY="crawl,subset"
+
+# Output directory
+CRAWL_ID=$1
+OUTPUT_PARENTDIR=hdfs:///user/vagrant/${CRAWL_ID}
+     # or just OUTPUT_PARENTDIR=/user/vagrant/${CRAWL_ID}, since /user/vagrant is on hdfs:
+
+# https://stackoverflow.com/questions/26513861/checking-if-directory-in-hdfs-already-exists-or-not
+hdfs dfs -test -d $OUTPUT_PARENTDIR
+if [ $? == 0 ]; then
+    echo "Directory $OUTPUT_PARENTDIR already exists."        
+else
+    echo "Creating directory $OUTPUT_PARENTDIR..."    
+    hdfs dfs -mkdir $OUTPUT_PARENTDIR
+fi
+
+
+# Spark configuration
+SPARK_HOME="$SPARK_HOME"
+# EXECUTOR_MEM=44g
+# EXECUTOR_CORES=12
+# NUM_EXECUTORS=4
+# DRIVER_MEM=4g
+
+EXECUTOR_MEM=3g
+EXECUTOR_CORES=2
+NUM_EXECUTORS=2
+DRIVER_MEM=3g
+
+
+SPARK_ON_YARN="--master yarn"
+SPARK_EXTRA_OPTS=""
+
+# source specific configuration file
+## test -e $(dirname $0)/convert_url_index_conf.sh && . $(dirname $0)/convert_url_index_conf.sh
+
+
+_APPJAR=$PWD/target/cc-spark-0.2-SNAPSHOT-jar-with-dependencies.jar
+
+##export LIBJARS=/home/vagrant/lib/hadoop-aws-2.7.6.jar
+#export LIBJARS=/home/vagrant/lib/*
+#export HADOOP_CLASSPATH=`echo ${LIBJARS} | sed s/,/:/g`
+
+set -e
+set -x
+
+OUTPUTDIR="hdfs:///user/vagrant/${CRAWL_ID}/cc-mri-csv"
+
+#   --conf spark.hadoop.fs.s3a.access.key=ACCESSKEY \
+#   --conf spark.hadoop.fs.s3a.secret.key=SECRETKEY \
+#    --conf spark.hadoop.fs.s3a.access.key=AKIA2EVQBWSTBJ2M4BLM \
+#    --conf spark.hadoop.fs.s3a.secret.key=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 \
+
+
+#    $SPARK_ON_YARN \
+# /home/vagrant/.m2/repository/org/apache/hadoop/hadoop-aws/2.7.6/hadoop-aws-2.7.6.jar
+#    --jars file:/home/vagrant/aws-java-sdk-1.7.4.jar,file:/home/vagrant/lib/hadoop-aws-2.7.6.jar \
+#    --driver-class-path=/home/vagrant/lib/aws-java-sdk-1.7.4.jar:/home/vagrant/lib/hadoop-aws-2.7.6.jar \
+
+# https://www.patricia-anong.com/blog/2017/11/1/extend-vmdk-on-virtualbox
+
+$SPARK_HOME/bin/spark-submit \
+    --jars file:/home/vagrant/.m2/repository/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar,file:/home/vagrant/.m2/repository/org/apache/hadoop/hadoop-aws/2.7.6/hadoop-aws-2.7.6.jar \
+    --driver-class-path=/home/vagrant/.m2/repository/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar:/home/vagrant/.m2/repository/org/apache/hadoop/hadoop-aws/2.7.6/hadoop-aws-2.7.6.jar \
+    --conf spark.hadoop.parquet.enable.dictionary=true \
+    --conf spark.hadoop.parquet.enable.summary-metadata=false \
+    --conf spark.sql.hive.metastorePartitionPruning=true \
+    --conf spark.sql.parquet.filterPushdown=true \
+    --conf spark.sql.parquet.mergeSchema=true \
+    --class org.commoncrawl.spark.examples.CCIndexExport $_APPJAR \
+    --query "SELECT url, warc_filename, warc_record_offset, warc_record_length   
+            FROM ccindex                                                        
+            WHERE crawl = '${CRAWL_ID}' AND subset = 'warc' AND content_languages = 'mri'" \
+    --outputFormat csv \
+    --numOutputPartitions 10 \
+    --outputCompression gzip \
+    s3a://commoncrawl/cc-index/table/cc-main/warc/ \
+    $OUTPUTDIR
+
+
+# The above generates ZIPPED part*.csv files in $OUTPUTDIR (folder cc-mri-csv).
+# First create a folder and unzip into it:
+hdfs dfs -test -d $OUTPUT_PARENTDIR/cc-mri-unzipped-csv
+if [ $? == 0 ]; then
+    echo "Directory cc-mri-unzipped-csv already exists for crawl ${CRAWL_ID}."
+else
+    echo "Creating directory $OUTPUT_PARENTDIR/cc-mri-unzipped-csv..."
+    hdfs dfs -mkdir $OUTPUT_PARENTDIR/cc-mri-unzipped-csv
+fi
+
+echo "Unzipping ${OUTPUTDIR}/part files into $OUTPUT_PARENTDIR/cc-mri-unzipped-csv/cc-mri.csv"
+hdfs dfs -cat $OUTPUTDIR/part* | gzip -d | hdfs dfs -put - $OUTPUT_PARENTDIR/cc-mri-unzipped-csv/cc-mri.csv
+
+
+# Now onto phase 2, which uses the index of MRI warc URLs and offsets,
+# stored in the now unzipped .csv file,
+# to get all the WARC records it specifies at the specified warc offsets.
+# This will get us all the MRI pages of the commoncrawl for CRAWL_ID
+
+# Change OUTPUTDIR
+OUTPUTDIR="hdfs:///user/vagrant/${CRAWL_ID}/warc"
+
+# $SPARK_HOME/bin/spark-submit \
+#     $SPARK_ON_YARN \
+#     --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+#     --conf spark.core.connection.ack.wait.timeout=600s \
+#     --conf spark.network.timeout=120s \
+#     --conf spark.task.maxFailures=20 \
+#     --conf spark.shuffle.io.maxRetries=20 \
+#     --conf spark.shuffle.io.retryWait=60s \
+#     --conf spark.driver.memory=$DRIVER_MEM \
+#     --conf spark.executor.memory=$EXECUTOR_MEM \
+#     $SPARK_EXTRA_OPTS \
+#     --num-executors $NUM_EXECUTORS \
+#     --executor-cores $EXECUTOR_CORES \
+#     --executor-memory $EXECUTOR_MEM \
+#     --conf spark.hadoop.parquet.enable.dictionary=true \
+#     --conf spark.sql.parquet.filterPushdown=true \
+#     --conf spark.sql.parquet.mergeSchema=false \
+#     --conf spark.sql.hive.metastorePartitionPruning=true \
+#     --conf spark.hadoop.parquet.enable.summary-metadata=false \
+#     --class org.commoncrawl.spark.CCIndex2Table $_APPJAR \
+#     --outputCompression=$COMPRS \
+#     --outputFormat=$FORMAT $NESTED \
+#     --partitionBy=$PARTITION_BY \
+#     "$DATA" "$OUTPUTDIR"
+
+#   --conf spark.hadoop.fs.s3a.access.key=ACCESSKEY \
+#   --conf spark.hadoop.fs.s3a.secret.key=SECRETKEY \
+
+
+#    $SPARK_ON_YARN \
+$SPARK_HOME/bin/spark-submit \
+   --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
+   --class org.commoncrawl.spark.examples.CCIndexWarcExport $_APPJAR \
+   --csv $OUTPUT_PARENTDIR/cc-mri-unzipped-csv/cc-mri.csv \
+   --numOutputPartitions 10 \
+   --numRecordsPerWarcFile 5000 \
+   --warcPrefix MAORI-${CRAWL_ID} \
+   s3a://commoncrawl/cc-index/table/cc-main/warc/ \
+   $OUTPUTDIR
+
+
+# Phase 3: convert warc files to wet files and tar them up into the mounted shared area
+
+hdfs dfs -test -f $OUTPUTDIR/_SUCCESS
+if [ $? == 0 ]; then
+    # ia-hadoop-tools converts warc files into wet (and wat) files but expects a particular folder structure
+    # Create the expected folder structure: a "wet" and a "wat" folder should exist
+    # at the same level as the "warc" folder ($OUTPUTDIR) that contains the warc gz files.
+    
+    echo "Creating wet (text) and wat (metadata) directories to put converted warc files into..."
+    hdfs dfs -mkdir $OUTPUT_PARENTDIR/wet
+    hdfs dfs -mkdir $OUTPUT_PARENTDIR/wat
+    # move the _SUCCESS file from /warc to its parent folder
+    hdfs dfs -mv $OUTPUTDIR/_SUCCESS $OUTPUT_PARENTDIR/.
+
+    # Move from the "/home/vagrant/cc-index-table" github project to the "ia-hadoop-tools" github project:
+    cd /home/vagrant/ia-hadoop-tools
+    # PWD is now ia-hadoop-tools folder
+    # To run the following, guava.jar should be on the hadoop classpath
+    # This can be achieved with:
+    # vagrant@node1:~/ia-hadoop-tools$ sudo cp /usr/share/java/guava.jar /usr/local/hadoop/share/hadoop/common/.
+    $HADOOP_MAPRED_HOME/bin/hadoop jar $PWD/target/ia-hadoop-tools-jar-with-dependencies.jar WEATGenerator -strictMode -skipExisting batch-id-xyz $OUTPUTDIR/*.warc.gz
+
+    # Now copy the zipped wet-files into the mounted folder, so we can scp the files from there to where WETProcessor.java is
+    echo "Copying and tarring up the wet files"
+    mkdir /vagrant/${CRAWL_ID}-wet-files
+    hdfs dfs -get $OUTPUT_PARENTDIR/wet/*.warc.wet.gz /vagrant/${CRAWL_ID}-wet-files/.
+    cd /vagrant
+    #tar -cvf ${CRAWL_ID}.tar /vagrant/${CRAWL_ID}-wet-files
+    #rm -rf /vagrant/${CRAWL_ID}-wet-files
+fi
+
