source: gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh@ 33573

#!/bin/bash

sitesDir=to_crawl/sites
echo "SITES DIR (INPUT): $sitesDir"
outputDir=crawled
mkdir -p $outputDir
echo "OUTPUT DIR: $outputDir"


NUTCH_HOME=apache-nutch-2.3.1
NUTCH_CONF_DIR=$NUTCH_HOME/runtime/local/conf
NUTCH_URLFILTER_TEMPLATE=$NUTCH_CONF_DIR/regex-urlfilter.GS_TEMPLATE
NUTCH_URLFILTER_FILE=$NUTCH_CONF_DIR/regex-urlfilter.txt

CRAWL_COMMAND=$NUTCH_HOME/runtime/local/bin/crawl
NUTCH_COMMAND=$NUTCH_HOME/runtime/local/bin/nutch

CRAWL_ITERATIONS=10

function crawlSite() {
    siteDir=$1
    crawlId=$2

    echo "processing site $siteDir"

    #echo "processing site $siteDir with crawlId: $crawlId"

    echo "Copying over template $NUTCH_URLFILTER_TEMPLATE to live version of file"
    cp $NUTCH_URLFILTER_TEMPLATE $NUTCH_URLFILTER_FILE

    echo "Appending contents of regex-urlfilter file for site $siteDir to url-filter file:"
    cat ${siteDir}regex-urlfilter.txt >> $NUTCH_URLFILTER_FILE

    #echo "Contents of seedURLs.txt file for site:"
    #cat ${siteDir}seedURLs.txt                       
    
    # $siteDir parameter is the folder containing seedURLs.txt
    # https://stackoverflow.com/questions/418896/how-to-redirect-output-to-a-file-and-stdout
    crawl_cmd="./$CRAWL_COMMAND $siteDir $crawlId $CRAWL_ITERATIONS"

    # Since we're going to crawl from scratch, create log.out file
    echo "Going to run nutch crawl command (and copy output to ${siteDir}log.out):" 2>&1 | tee ${siteDir}log.out
    # append to log.out file hereafter
    echo "  $crawl_cmd" 2>&1 | tee -a ${siteDir}log.out
    echo "--------------------------------------------------" 2>&1 | tee -a ${siteDir}log.out

    # append output of $crawl_cmd to log.out
    $crawl_cmd 2>&1 | tee -a ${siteDir}log.out
    result=$?

    if [ "x$result" = "x0" ]; then
    # nutch finished crawling successfully.
    
    # But check if the site was crawled thoroughly within $CRAWL_ITERATIONS
    # If not, create file UNFINISHED to indicate a more thorough crawl needed
    tail -10 ${siteDir}log.out | grep "no more URLs to fetch now" > /dev/null
    result=$?
    if [ "x$result" != "x0" ]; then
        echo "A crawl of $CRAWL_ITERATIONS iterations was insufficient for crawlId $crawlId" 2>&1 | tee ${siteDir}UNFINISHED
        echo "" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "To re-run crawl of site with crawlId $crawlId with a larger number of iterations:" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "1. delete $outputDir/$crawlId" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "2. copy the regex-urlfilter file:" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "   cp $NUTCH_URLFILTER_TEMPLATE $NUTCH_URLFILTER_FILE" 2>&1 | tee -a ${siteDir}UNFINISHED
        echo "3. Adjust # crawl iterations in old crawl command:\n$crawl_cmd" 2>&1 | tee -a ${siteDir}UNFINISHED
    fi
    
    # outputDir/$crawlId should not yet exist
        ./$NUTCH_COMMAND readdb -dump $outputDir/$crawlId -text -crawlId $crawlId
        ./$NUTCH_COMMAND readdb -stats -crawlId $crawlId > $outputDir/$crawlId/stats
        cat $outputDir/$crawlId/part-r-00000 > $outputDir/$crawlId/dump.txt
    else
    # appending to log.out
        echo "CRAWL FAILED." 2>&1 | tee -a ${siteDir}log.out
    fi


    # move the peripheral crawl products (the log.out and optional UNFINISHED file)
    # from the input to the output folder. This way we can re-run the crawl and
    # these files will still have been preserved as long as the output folder
    # isn't deleted
    mv ${siteDir}log.out $outputDir/$crawlId/log.out
    if [ -e "${siteDir}UNFINISHED" ]; then
    mv ${siteDir}UNFINISHED $outputDir/$crawlId/UNFINISHED
    fi

    # finally append the current crawlId to siteIDsCrawled.txt
    echo $crawlId >> $outputDir/siteIDsCrawled.txt
}

function prepareForCrawl() {
    siteDir=$1
    crawlId=$2

    echo "Processing siteDir $siteDir with crawlId: $crawlId"
    
    if [ -d "$outputDir/$crawlId" ]; then
    # Skip site already processed. *Append* this msg to log.out
    echo "" 2>&1 | tee -a ${siteDir}log.out
    echo "**** $siteDir already processed. Skipping...." 2>&1 | tee -a ${siteDir}log.out
    echo "Delete $outputDir/$crawlId if you want to reprocess it." 2>&1 | tee -a ${siteDir}log.out  

    else    
    crawlSite $siteDir $crawlId

    fi
    echo "--------------------------------------------------"
    

    
}

function crawlAll() {

    # https://stackoverflow.com/questions/4000613/perform-an-action-in-every-sub-directory-using-bash
    for siteDir in $sitesDir/*/; do
    
    # to get crawl_id like 00001 from $siteDir like to_crawl/sites/00001/
    # Remove the $sitesDir prefix of to_crawl/sites followed by /,
    # Next remove the / suffix that remains
    crawlId=${siteDir#"$sitesDir/"}
    crawlId=${crawlId%/}
    
    prepareForCrawl $siteDir $crawlId
    #break
    done
}

function crawlRange() {
    
    startCrawlId=$1
    endCrawlId=$2

    # https://unix.stackexchange.com/questions/232384/argument-string-to-integer-in-bash/301285

    # sadly, the numeric value is in octal in both the following
    #end=$(($endCrawlId+0))    
    #printf -v end '%d\n' $endCrawlId 2>/dev/null
    # removes a single 0 prefix
    #end=${endCrawlId##+(0)}
    #${old##+(0)}
    # https://stackoverflow.com/questions/19861394/stripping-leading-zeros-using-bash-substring-removal-only/51196007
    start=${startCrawlId#"${startCrawlId%%[!0]*}"}
    end=${endCrawlId#"${endCrawlId%%[!0]*}"}

    echo "Start: $start ($startCrawlId), End: $end ($endCrawlId)"

    # Generate a range of numbers between start and end
    #https://stackoverflow.com/questions/966020/how-to-produce-a-range-with-step-n-in-bash-generate-a-sequence-of-numbers-with/966026
    for (( COUNTER=$start; $COUNTER<=$end; COUNTER=$COUNTER+1 )); do
    # Now pad back with zeroes to get crawlId
    # https://stackoverflow.com/questions/1117134/padding-zeros-in-a-string 
    crawlId=`printf %05d $COUNTER`
    #echo $COUNTER - $crawlId
    # $siteDir needs / at end to work with existing code
    siteDir=$sitesDir/$crawlId/
    #echo "siteDir $siteDir (crawlId $crawlId)"
    prepareForCrawl $siteDir $crawlId
    done
    
}

function printUsage() {
    echo "Run as:"
    echo "  $0 -all|<ids>"
    echo "    where an id is a folder name in to_crawl/sites"
    echo "    and ids can be a comma or space separated list of"
    echo "    individual ids and/or ranges"
    echo "  Examples:"
    echo "    $0 00008-00022,00025,00026,00028-00034 00050"
    echo "    $0 -all"
    echo "    $0 00312"
    
}

function parseArgs() {
    
    # for i in $*; do
    #   echo "Pinky"
    #   echo $i
    # done

    args="$1"
    #echo "Got arg string: $args"

    if [ "x$args" = "x" ]; then 
    printUsage
    exit 0
    fi
    
    # works - split args on comma or space
    # https://stackoverflow.com/questions/918886/how-do-i-split-a-string-on-a-delimiter-in-bash
    IFS=', ' read -ra IDS <<< "$args"
    for id in "${IDS[@]}"; do
    echo "id: |$id|"
    if [ "x$id" = "x-all" ]; then
        echo "crawlAll"
        crawlAll
    elif [[ $id == *"-"* ]]; then
        # https://stackoverflow.com/questions/229551/how-to-check-if-a-string-contains-a-substring-in-bash
        echo "$id is a range"
        startCrawlId=${id%%-*}
        endCrawlId=${id##*-}
        echo "crawlRange $startCrawlId $endCrawlId"
        crawlRange $startCrawlId $endCrawlId    
        
    else
        echo "$id is singleton"
        crawlId=$id
        # $siteDir needs / at end to work with existing code
        siteDir=$sitesDir/$crawlId/
        echo "prepareForCrawl $siteDir $crawlId"
        prepareForCrawl $siteDir $crawlId       
    fi
    done
}



# Passing as string instead of individual arguments
# Because one can only have 9 individual args without using shift, see
# https://www.unix.com/shell-programming-and-scripting/57225-there-limit-no-arguments-shell-script.html

parseArgs "$*"

# old. testing
#crawlRange "00010" "00015"
#parseArgs "00008-00022,00025,00026,00028-00034"