source: main/trunk/model-sites-dev/commoncrawl/moveDumpTxtFilesIntoImport.sh@ 36359

#!/bin/bash

# EDIT to set crawleddir value, to the folder where crawls are extracted, then run.
# This will also take care of complete Nutch crawled directories
# that include more than dump.txt within siteID folders.
# (like a folder consisting of the combination of all crawledNode#.tar.gz files
# at http://trac.greenstone.org/browser/other-projects/maori-lang-detection)
# In such cases, this script will copy over just the siteID folder along with dump.txt

# The commented out version will copy across siteID/dump.txt as siteID.txt
# instead (without enclosing folder).


# https://superuser.com/questions/44787/looping-through-subdirectories-and-running-a-command-in-each
# https://stackoverflow.com/questions/15148796/get-string-after-character

## producing files called siteID.txt, instead of siteID/dump.txt files
# for dir in txtdumps/*; do
#     #do (cd "$dir" && cp dump.txt );
#     filename=${dir#*/}
#     echo $filename
#     mv "$dir/dump.txt" "import/$filename.txt";
# done



# https://stackoverflow.com/questions/23162299/how-to-get-the-last-part-of-dirname-in-bash
crawleddir=
for dir in $crawleddir/*; do
    if [[ -d "$dir" ]]; then
    foldername="${dir##*/}"

    mkdir import/$foldername
    if [[ ! -f "$dir/dump.txt" ]]; then
        echo "There was no dump.txt in $foldername"
    else
        #echo $foldername
        cp "$dir/dump.txt" import/$foldername/.
    fi
    fi
done