Changeset 33567

Timestamp:

2019-10-14T22:40:22+13:00 (5 years ago)

Author:

ak19

Message:

batchcrawl.sh now supports -all flag (and prints usage on 0 args). The script is active again with some minor testing done. Site 00001 (a page off ​http://00.gs) downloads successfully again, this time as a single site argument to the script. Removed the break statement from the crawlAll() function.

File:

• gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh (modified) (4 diffs)

gs3-extensions/maori-lang-detection/hdfs-cc-work/scripts/batchcrawl.sh

@@ -81 +81 @@
     crawlId=$2
 
-    echo "Processing crawlId: $crawlId"
+    echo "Processing siteDir $siteDir with crawlId: $crawlId"
     
     if [ -d "$outputDir/$crawlId" ]; then
@@ -111 +111 @@
     
     prepareForCrawl $siteDir $crawlId
-    break
+    #break
     done
 }
@@ -141 +141 @@
     crawlId=`printf %05d $COUNTER`
     #echo $COUNTER - $crawlId
-    siteDir=$sitesDir/$crawlId
-    echo "siteDir $siteDir (crawlId $crawlId)"
+    # $siteDir needs / at end to work with existing code
+    siteDir=$sitesDir/$crawlId/
+    #echo "siteDir $siteDir (crawlId $crawlId)"
+    prepareForCrawl $siteDir $crawlId
     done
     
 }
 
+function printUsage() {
+    echo "Run as:"
+    echo "  $0 -all|<ids>"
+    echo "    where an id is a folder name in to_crawl/sites"
+    echo "    and ids can be a comma separated list of"
+    echo "    individual ids and/or ranges"
+    echo "  Examples:"
+    echo "    $0 00008-00022,00025,00026,00028-00034"
+    echo "    $0 -all"
+    echo "    $0 00312"
+    
+}
+
 function parseArgs() {
+    
     # for i in $*; do
     #   echo "Pinky"
@@ -154 +170 @@
 
     args="$1"
-    echo "Got arg string: $args"
-
+    #echo "Got arg string: $args"
+
+    if [ "x$args" = "x" ]; then 
+    printUsage
+    exit 0
+    fi
+    
     # works - split args on comma
     # https://stackoverflow.com/questions/918886/how-do-i-split-a-string-on-a-delimiter-in-bash
     IFS=', ' read -ra IDS <<< "$args"
     for id in "${IDS[@]}"; do
-
-    if [[ $id == *"-"* ]]; then
+    echo "id: |$id|"
+    if [ "x$id" = "x-all" ]; then
+        echo "crawlAll"
+        crawlAll
+    elif [[ $id == *"-"* ]]; then
+        # https://stackoverflow.com/questions/229551/how-to-check-if-a-string-contains-a-substring-in-bash
         echo "$id is a range"
         startCrawlId=${id%%-*}
         endCrawlId=${id##*-}
         echo "crawlRange $startCrawlId $endCrawlId"
-        crawlRange $startCrawlId $endCrawlId
+        crawlRange $startCrawlId $endCrawlId    
+        
     else
         echo "$id is singleton"
         crawlId=$id
-        siteDir=$sitesDir/$crawlId
+        # $siteDir needs / at end to work with existing code
+        siteDir=$sitesDir/$crawlId/
         echo "prepareForCrawl $siteDir $crawlId"
-        #prepareForCrawl $siteDir $crawlId      
+        prepareForCrawl $siteDir $crawlId       
     fi
     done
-
-
-
-}
-
+}
+
+
+
+# Passing as string instead of individual arguments
+# Because one can only have 9 individual args without using shift, see
+# https://www.unix.com/shell-programming-and-scripting/57225-there-limit-no-arguments-shell-script.html
+
+parseArgs "$*"
+
+# old. testing
 #crawlRange "00010" "00015"
 #parseArgs "00008-00022,00025,00026,00028-00034"
-parseArgs "$*"
-# can only have 9 args without using shift, see
-# https://stackoverflow.com/questions/229551/how-to-check-if-a-string-contains-a-substring-in-bash
-#parseArgs horse cow dog cat