source: gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh@ 33563

#!/bin/bash
echo "Hello world!"

sitesDir=to_crawl/sites
echo "SITESDIR: $sitesDir"

NUTCH_HOME=apache-nutch-2.3.1
NUTCH_CONF_DIR=$NUTCH_HOME/conf
NUTCH_URLFILTER_TEMPLATE=$NUTCH_CONF_DIR/regex-urlfilter.GS_TEMPLATE
NUTCH_URLFILTER_FILE=$NUTCH_CONF_DIR/regex-urlfilter.txt

CRAWL_COMMAND=$NUTCH_HOME/runtime/local/bin/crawl

CRAWL_ITERATIONS=10

function prepareSite() {
    siteDir=$1
    crawlId=$2

    #echo "processing site $siteDir"

    #echo "processing site $siteDir with crawlId: $crawlId"

    echo "Copying over template $NUTCH_URLFILTER_TEMPLATE to live version of file"
    cp $NUTCH_URLFILTER_TEMPLATE $NUTCH_URLFILTER_FILE

    echo "Appending contents of regex-urlfilter file for site $siteDir to url-filter file:"
    cat ${siteDir}regex-urlfilter.txt >> $NUTCH_URLFILTER_FILE

    #echo "Contents of seedURLs.txt file for site:"
    #cat ${siteDir}seedURLs.txt                       
    
    # $siteDir parameter is the folder containing seedURLs.txt
    crawl_cmd="./$CRAWL_COMMAND $siteDir $crawlId $CRAWL_ITERATIONS"

    echo "Going to run nutch crawl command:"
    echo "  $crawl_cmd"

    
}


# https://stackoverflow.com/questions/4000613/perform-an-action-in-every-sub-directory-using-bash
for siteDir in $sitesDir/*/; do
    #echo "$siteDir"
    # to get crawl_id like 00001 from $siteDir like to_crawl/sites/00001/
    # Remove the $sitesDir prefix of to_crawl/sites followed by /,
    # Next remove the / suffix that remains
    crawlId=${siteDir#"$sitesDir/"}
    crawlId=${crawlId%/}

    #echo "crawlId: $crawlId"
    prepareSite $siteDir $crawlId
    break
done