source: gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh@ 33564

#!/bin/bash

sitesDir=to_crawl/sites
echo "SITES DIR (INPUT): $sitesDir"
outputDir=crawled
mkdir -p $outputDir
echo "OUTPUT DIR: $outputDir"


NUTCH_HOME=apache-nutch-2.3.1
NUTCH_CONF_DIR=$NUTCH_HOME/runtime/local/conf
NUTCH_URLFILTER_TEMPLATE=$NUTCH_CONF_DIR/regex-urlfilter.GS_TEMPLATE
NUTCH_URLFILTER_FILE=$NUTCH_CONF_DIR/regex-urlfilter.txt

CRAWL_COMMAND=$NUTCH_HOME/runtime/local/bin/crawl
NUTCH_COMMAND=$NUTCH_HOME/runtime/local/bin/nutch

CRAWL_ITERATIONS=10

function crawlSite() {
    siteDir=$1
    crawlId=$2

    echo "processing site $siteDir with crawlId: $crawlId"

    echo "Copying over template $NUTCH_URLFILTER_TEMPLATE to live version of file"
    cp $NUTCH_URLFILTER_TEMPLATE $NUTCH_URLFILTER_FILE

    echo "Appending contents of regex-urlfilter file for site $siteDir to url-filter file:"
    cat ${siteDir}regex-urlfilter.txt >> $NUTCH_URLFILTER_FILE

    #echo "Contents of seedURLs.txt file for site:"
    #cat ${siteDir}seedURLs.txt                       
    
    # $siteDir parameter is the folder containing seedURLs.txt
    # https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout
    crawl_cmd="./$CRAWL_COMMAND $siteDir $crawlId $CRAWL_ITERATIONS"

    # Since we're going to crawl from scratch, create log.out file
    echo "Going to run nutch crawl command (and copy output to ${siteDir}log.out):" 2>&1 | tee ${siteDir}log.out
    # append to log.out file hereafter
    echo "  $crawl_cmd" 2>&1 | tee -a ${siteDir}log.out
    echo "--------------------------------------------------" 2>&1 | tee -a ${siteDir}log.out

    # append output of $crawl_cmd to log.out
    $crawl_cmd 2>&1 | tee -a ${siteDir}log.out
    result=$?

    if [ "x$result" = "x0" ]; then
    # nutch finished crawling successfully.
    
    # But check if the site was crawled thoroughly within $CRAWL_ITERATIONS
    # If not, create file UNFINISHED to indicate a more thorough crawl needed
    tail -10 ${siteDir}log.out | grep "no more URLs to fetch now" > /dev/null
    result=$?
    if [ "x$result" != "x0" ]; then
        echo "A crawl of $CRAWL_ITERATIONS iterations was insufficient for crawlId $crawlId" 2>&1 | tee ${siteDir}UNFINISHED
        echo "" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "To re-run crawl of site with crawlId $crawlId with a larger number of iterations:" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "1. delete $outputDir/$crawlId" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "2. copy the regex-urlfilter file:" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "   cp $NUTCH_URLFILTER_TEMPLATE $NUTCH_URLFILTER_FILE" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "3. Adjust # crawl iterations in old crawl command:\n$crawl_cmd" 2>&1 | tee -a ${siteDir}UNFINISHED
    fi
    
    # outputDir/$crawlId should not yet exist
        ./$NUTCH_COMMAND readdb -dump $outputDir/$crawlId -text -crawlId $crawlId
        ./$NUTCH_COMMAND readdb -stats -crawlId $crawlId > $outputDir/$crawlId/stats
        cat $outputDir/$crawlId/part-r-00000 > $outputDir/$crawlId/dump.txt
    else
    # appending to log.out
        echo "CRAWL FAILED." 2>&1 | tee -a ${siteDir}log.out
    fi
    
}


# https://stackoverflow.com/questions/4000613/perform-an-action-in-every-sub-directory-using-bash
for siteDir in $sitesDir/*/; do

    # to get crawl_id like 00001 from $siteDir like to_crawl/sites/00001/
    # Remove the $sitesDir prefix of to_crawl/sites followed by /,
    # Next remove the / suffix that remains
    crawlId=${siteDir#"$sitesDir/"}
    crawlId=${crawlId%/}

    echo "Processing crawlId: $crawlId"
    
    if [ -d "$outputDir/$crawlId" ]; then
    # Skip site already processed. *Append* this msg to log.out
    echo "" 2>&1 | tee -a ${siteDir}log.out
    echo "**** $siteDir already processed. Skipping...." 2>&1 | tee -a ${siteDir}log.out
    echo "Delete $outputDir/$crawlId if you want to reprocess it." 2>&1 | tee -a ${siteDir}log.out  

    else    
    crawlSite $siteDir $crawlId

    fi
    echo "--------------------------------------------------"
    
    break
done