Last change
on this file was 31151, checked in by davidb, 7 years ago |
More nuanced version to help finish off the 'big put'
|
-
Property svn:executable
set to
*
|
File size:
706 bytes
|
Line | |
---|
1 | #!/bin/bash
|
---|
2 |
|
---|
3 | if [ "$#" == "0" ] ; then
|
---|
4 | echo "Usage: $0 top-level-dir1 depth-level1-dir1 [depth-level1-dir2 ...]"
|
---|
5 | exit
|
---|
6 | fi
|
---|
7 |
|
---|
8 |
|
---|
9 | top_level=$1
|
---|
10 | shift
|
---|
11 |
|
---|
12 | echo "++++"
|
---|
13 | echo "++++"
|
---|
14 | echo "Top-level dir: $top_level"
|
---|
15 | echo "++++"
|
---|
16 | echo "++++"
|
---|
17 |
|
---|
18 | for d1 in $* ; do
|
---|
19 |
|
---|
20 |
|
---|
21 | for d in /data/features/ef-full/${top_level}/pairtree_root/${d1}/* ; do
|
---|
22 | sub_dir=${d##/data/features/ef-full/}
|
---|
23 |
|
---|
24 | echo ""
|
---|
25 | echo "****"
|
---|
26 | echo "Processing: $sub_dir"
|
---|
27 | echo "****"
|
---|
28 |
|
---|
29 | hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
|
---|
30 |
|
---|
31 | into_dir=${sub_dir%/*}
|
---|
32 | hdfs dfs -put $d "full-ef-json-files/$into_dir/."
|
---|
33 | done
|
---|
34 | done
|
---|
35 |
|
---|
36 | echo "===="
|
---|
37 | echo "===="
|
---|
38 | echo "===="
|
---|
39 | echo "Finished: processed top-level(s) $*"
|
---|
40 | echo "===="
|
---|
41 | echo "===="
|
---|
42 | echo "===="
|
---|
Note:
See
TracBrowser
for help on using the repository browser.