source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-TL-TARGET-DEPTH1-FOREACH-DEPTH2-HDFS-PUT.sh@ 31161

Last change on this file since 31161 was 31151, checked in by davidb, 5 years ago

More nuanced version to help finish off the 'big put'

  • Property svn:executable set to *
File size: 706 bytes
Line 
1#!/bin/bash
2
3if [ "$#" == "0" ] ; then
4 echo "Usage: $0 top-level-dir1 depth-level1-dir1 [depth-level1-dir2 ...]"
5 exit
6fi
7
8
9top_level=$1
10shift
11
12echo "++++"
13echo "++++"
14echo "Top-level dir: $top_level"
15echo "++++"
16echo "++++"
17
18for d1 in $* ; do
19
20
21 for d in /data/features/ef-full/${top_level}/pairtree_root/${d1}/* ; do
22 sub_dir=${d##/data/features/ef-full/}
23
24 echo ""
25 echo "****"
26 echo "Processing: $sub_dir"
27 echo "****"
28
29 hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
30
31 into_dir=${sub_dir%/*}
32 hdfs dfs -put $d "full-ef-json-files/$into_dir/."
33 done
34done
35
36echo "===="
37echo "===="
38echo "===="
39echo "Finished: processed top-level(s) $*"
40echo "===="
41echo "===="
42echo "===="
Note: See TracBrowser for help on using the repository browser.