source: gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/setup.sh@ 33538

#!/bin/bash

# set value to "copy_extra_jars" if you want certain additional jars copied over,
# a step that may be optional or may be necessary
copy_extra_jars=$1

echo "Checking out hdfs-cc-work from GS svn's gs3-extensions/maori-lang-detection"
svn co http://svn.greenstone.org/gs3-extensions/maori-lang-detection/hdfs-cc-work

echo "Moving the tarred 3 git projects, cc-index-table (modified), ia-hadoop-tools (modified) and ia-web-commons into the current folder and extracting..."
mv hdfs-cc-work/gitprojects/cc-index-table.tar .
mv hdfs-cc-work/gitprojects/ia-hadoop-tools.tar .
mv hdfs-cc-work/gitprojects/ia-web-commons.tar .

tar -xvf cc-index-table.tar
tar -xvf ia-hadoop-tools.tar
tar -xvf ia-web-commons.tar


echo "Copying scripts into cc-index-table and making them executable"
# the script that does the actual work
cp hdfs-cc-work/scripts/get_maori_WET_records_for_crawl.sh cc-index-table/src/script/.
chmod u+x cc-index-table/src/script/get_maori_WET_records_for_crawl.sh
# top level script belonging in top level of cc-index-table
cp hdfs-cc-work/scripts/get_Maori_WET_records_from_CCSep2018_on.sh cc-index-table/.
chmod u+x cc-index-table/get_Maori_WET_records_from_CCSep2018_on.sh


echo "Compiling the modified cc-index-table"
cd cc-index-table
mvn package

# ia-web-commons has to be compiled up before ia-hadoop-tools
echo "Compiling ia-web-commons"
cd ../ia-web-commons
mvn install

echo "Compiling the modified ia-hadoop-tools"
cd ../ia-hadoop-tools
mvn package


echo "Putting jar files in place"

echo "1. For RUNNING warc to wet conversion (ia-hadoop-tools): Sudo copying hdfs-cc-work/jars/guava.jar into hadoop classpath $hadoop_common_dir"
sudo cp hdfs-cc-work/jars/guava.jar $hadoop_common_dir/.

# Some discussion about == and = allowed in if test for equality
# https://stackoverflow.com/questions/4277665/how-do-i-compare-two-string-variables-in-an-if-statement-in-bash
if [ "x$copy_extra_jars" == "xcopy_extra_jars" ]; then
    hadoop_common_dir=/usr/local/hadoop/share/hadoop/common
    echo "Copying extra jars"
    echo "2. For cc-index-table: Sudo copying hadoop-aws-2.7.6.jar and aws-java-sdk-1.11.616.jar"
    echo "from hdfs-cc-work/jars into hadoop classpath $hadoop_common_dir"
    sudo cp hadoop-aws-2.7.6.jar $hadoop_common_dir/.
    sudo cp aws-java-sdk-1.11.616.jar $hadoop_common_dir/.
    
    dest_dir=/home/vagrant/.m2/repository/com/amazonaws/aws-java-sdk/1.7.4
    if [ ! -e "$dest_dir/aws-java-sdk-1.7.4.jar" ]; then
    echo "3. For cc-index-table: Copying aws-java-sdk-1.7.4.jar into $dest_dir/"
    mkdir -p $dest_dir
    cp hdfs-cc-work/jars/aws-java-sdk-1.7.4.jar $dest_dir/.
    fi
    
    dest_dir=/home/vagrant/.m2/repository/org/apache/hadoop/hadoop-aws/2.7.6
    if [ ! -e "$dest_dir/hadoop-aws-2.7.6.jar" ]; then
    echo "4. For cc-index-table: Copying hadoop-aws-2.7.6.jar into $dest_dir/"
    mkdir -p $dest_dir
    cp hdfs-cc-work/jars/aws-java-sdk-1.7.4.jar $dest_dir/.
    fi
fi

echo "Done compiling and automated parts of setting up."
echo "NEXT STEP:"
echo "Ensure you have sudo edited $SPARK_HOME/conf/spark-defaults.conf"
echo "  (/usr/local/spark-2.3.0-bin-hadoop2.7/conf/spark-defaults.conf)"
echo "to contain the following 3 lines with YOUR Amazon AWS IAM Role access and secret keys:"
echo "   spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem"
echo "   spark.hadoop.fs.s3a.access.key=YOUR_AWS_IAM-ROLE_ACCESSKEY_HERE"
echo "   spark.hadoop.fs.s3a.secret.key=YOUR_AWS_IAM-ROLE_SECRETKEY_HERE"
echo "Consult GS_README.TXT section B (and C) for instructions on setting up an AWS IAM role."
echo "Only when that's done will you be ready to run the following script."
echo ""
echo "THEN:"
echo "To get MRI warc to wet for a particular crawl timestamp, cd into cc-index-table and RUN:"
echo "./get_maori_WET_records_for_crawl.sh CC-MAIN-<YYYY-##>"
echo "   where YYYY is the year and ## is the common crawl number for that year."