source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-FOREACH-HDFS-PUT.sh@ 31152

Last change on this file since 31152 was 31152, checked in by davidb, 5 years ago

Development of script

  • Property svn:executable set to *
File size: 685 bytes
Line 
1#!/bin/bash
2
3if [ "$#" == "0" ] ; then
4 echo "Usage: $0 top-level-dir1 [top-level-dir2 ...]"
5 exit
6fi
7
8
9for t in $* ; do
10 echo ""
11 echo ""
12 echo "++++"
13 echo "++++"
14 echo "Top-level dir: $t"
15 echo "++++"
16 echo "++++"
17
18
19 for d in /data/features/ef-full/$t/pairtree_root/*/* ; do
20 sub_dir=${d##/data/features/ef-full/}
21
22 echo ""
23 echo "****"
24 echo "Processing: $sub_dir"
25 echo "****"
26
27 hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
28
29 into_dir=${sub_dir%/*}
30 hdfs dfs -put $d "full-ef-json-files/$into_dir/."
31 done
32done
33
34echo "===="
35echo "===="
36echo "===="
37echo "Finished: processed top-level(s) $*"
38echo "===="
39echo "===="
40echo "===="
Note: See TracBrowser for help on using the repository browser.