Changeset 33566 for gs3-extensions

Timestamp:

2019-10-14T22:07:45+13:00 (5 years ago)

Author:

ak19

Message:

batchcrawl.sh script now supports taking a comma or space separated list of crawl folder ids of the form 00000%d (e.g. 00311) and/or ranges of such crawl folder ids. Script is now inactive. Next commit should support the all parameter to crawl all subfolders (and perhaps a range with no final termination value).

File:

• gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh (modified) (3 diffs)

gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh

@@ -22 +22 @@
     crawlId=$2
 
-    echo "processing site $siteDir with crawlId: $crawlId"
+    echo "processing site $siteDir"
+
+    #echo "processing site $siteDir with crawlId: $crawlId"
 
     echo "Copying over template $NUTCH_URLFILTER_TEMPLATE to live version of file"
@@ -75 +77 @@
 }
 
-
-# https://stackoverflow.com/questions/4000613/perform-an-action-in-every-sub-directory-using-bash
-for siteDir in $sitesDir/*/; do
-
-    # to get crawl_id like 00001 from $siteDir like to_crawl/sites/00001/
-    # Remove the $sitesDir prefix of to_crawl/sites followed by /,
-    # Next remove the / suffix that remains
-    crawlId=${siteDir#"$sitesDir/"}
-    crawlId=${crawlId%/}
+function prepareForCrawl() {
+    siteDir=$1
+    crawlId=$2
 
     echo "Processing crawlId: $crawlId"
@@ -99 +95 @@
     echo "--------------------------------------------------"
     
-    break
-done
+
+    
+}
+
+function crawlAll() {
+
+    # https://stackoverflow.com/questions/4000613/perform-an-action-in-every-sub-directory-using-bash
+    for siteDir in $sitesDir/*/; do
+    
+    # to get crawl_id like 00001 from $siteDir like to_crawl/sites/00001/
+    # Remove the $sitesDir prefix of to_crawl/sites followed by /,
+    # Next remove the / suffix that remains
+    crawlId=${siteDir#"$sitesDir/"}
+    crawlId=${crawlId%/}
+    
+    prepareForCrawl $siteDir $crawlId
+    break
+    done
+}
+
+function crawlRange() {
+    
+    startCrawlId=$1
+    endCrawlId=$2
+
+    # https://unix.stackexchange.com/questions/232384/argument-string-to-integer-in-bash/301285
+
+    # sadly, the numeric value is in octal in both the following
+    #end=$(($endCrawlId+0))    
+    #printf -v end '%d\n' $endCrawlId 2>/dev/null
+    # removes a single 0 prefix
+    #end=${endCrawlId##+(0)}
+    #${old##+(0)}
+    # https://stackoverflow.com/questions/19861394/stripping-leading-zeros-using-bash-substring-removal-only/51196007
+    start=${startCrawlId#"${startCrawlId%%[!0]*}"}
+    end=${endCrawlId#"${endCrawlId%%[!0]*}"}
+
+    echo "Start: $start ($startCrawlId), End: $end ($endCrawlId)"
+
+    # Generate a range of numbers between start and end
+    #https://stackoverflow.com/questions/966020/how-to-produce-a-range-with-step-n-in-bash-generate-a-sequence-of-numbers-with/966026
+    for (( COUNTER=$start; $COUNTER<=$end; COUNTER=$COUNTER+1 )); do
+    # Now pad back with zeroes to get crawlId
+    # https://stackoverflow.com/questions/1117134/padding-zeros-in-a-string 
+    crawlId=`printf %05d $COUNTER`
+    #echo $COUNTER - $crawlId
+    siteDir=$sitesDir/$crawlId
+    echo "siteDir $siteDir (crawlId $crawlId)"
+    done
+    
+}
+
+function parseArgs() {
+    # for i in $*; do
+    #   echo "Pinky"
+    #   echo $i
+    # done
+
+    args="$1"
+    echo "Got arg string: $args"
+
+    # works - split args on comma
+    # https://stackoverflow.com/questions/918886/how-do-i-split-a-string-on-a-delimiter-in-bash
+    IFS=', ' read -ra IDS <<< "$args"
+    for id in "${IDS[@]}"; do
+
+    if [[ $id == *"-"* ]]; then
+        echo "$id is a range"
+        startCrawlId=${id%%-*}
+        endCrawlId=${id##*-}
+        echo "crawlRange $startCrawlId $endCrawlId"
+        crawlRange $startCrawlId $endCrawlId
+    else
+        echo "$id is singleton"
+        crawlId=$id
+        siteDir=$sitesDir/$crawlId
+        echo "prepareForCrawl $siteDir $crawlId"
+        #prepareForCrawl $siteDir $crawlId      
+    fi
+    done
+
+
+
+}
+
+#crawlRange "00010" "00015"
+#parseArgs "00008-00022,00025,00026,00028-00034"
+parseArgs "$*"
+# can only have 9 args without using shift, see
+# https://stackoverflow.com/questions/229551/how-to-check-if-a-string-contains-a-substring-in-bash
+#parseArgs horse cow dog cat