Changeset 31152 for other-projects/hathitrust/wcsa
- Timestamp:
- 2016-11-23T23:11:16+13:00 (7 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-FOREACH-HDFS-PUT.sh
r31128 r31152 1 1 #!/bin/bash 2 2 3 if [ "$#" != "1" ] ; then4 echo "Usage: $0 top-level-dir "3 if [ "$#" == "0" ] ; then 4 echo "Usage: $0 top-level-dir1 [top-level-dir2 ...]" 5 5 exit 6 6 fi 7 7 8 toplevel_dir=$1 8 9 for t in $* ; do 10 echo "" 11 echo "" 12 echo "++++" 13 echo "++++" 14 echo "Top-level dir: $t" 15 echo "++++" 16 echo "++++" 9 17 10 18 11 for d in /data/features/ef-full/$toplevel_dir/*/* ; do12 sub_dir=${d##/data/features/ef-full/}19 for d in /data/features/ef-full/$t/pairtree_root/*/* ; do 20 sub_dir=${d##/data/features/ef-full/} 13 21 14 echo "Processing: $sub_dir" 15 hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir" 22 echo "" 23 echo "****" 24 echo "Processing: $sub_dir" 25 echo "****" 16 26 17 into_dir=${sub_dir%/*} 18 hdfs dfs -put $d "full-ef-json-files/$into_dir/." 27 hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir" 28 29 into_dir=${sub_dir%/*} 30 hdfs dfs -put $d "full-ef-json-files/$into_dir/." 31 done 19 32 done 20 33 34 echo "====" 35 echo "====" 36 echo "====" 37 echo "Finished: processed top-level(s) $*" 38 echo "====" 39 echo "====" 40 echo "===="
Note:
See TracChangeset
for help on using the changeset viewer.