Changeset 31152

Show
Ignore:
Timestamp:
23.11.2016 23:11:16 (3 years ago)
Author:
davidb
Message:

Development of script

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-FOREACH-HDFS-PUT.sh

    r31128 r31152  
    11#!/bin/bash 
    22 
    3 if [ "$#" != "1" ] ; then 
    4   echo "Usage: $0 top-level-dir" 
     3if [ "$#" == "0" ] ; then 
     4  echo "Usage: $0 top-level-dir1 [top-level-dir2 ...]" 
    55  exit 
    66fi 
    77 
    8 toplevel_dir=$1 
     8 
     9for t in $* ; do 
     10    echo "" 
     11    echo "" 
     12    echo "++++" 
     13    echo "++++" 
     14    echo "Top-level dir: $t" 
     15    echo "++++" 
     16    echo "++++" 
    917 
    1018 
    11 for d in /data/features/ef-full/$toplevel_dir/*/* ; do 
    12   sub_dir=${d##/data/features/ef-full/} 
     19  for d in /data/features/ef-full/$t/pairtree_root/*/* ; do 
     20    sub_dir=${d##/data/features/ef-full/} 
    1321 
    14   echo "Processing: $sub_dir" 
    15   hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir" 
     22    echo "" 
     23    echo "****" 
     24    echo "Processing: $sub_dir" 
     25    echo "****" 
    1626 
    17   into_dir=${sub_dir%/*} 
    18   hdfs dfs -put $d "full-ef-json-files/$into_dir/." 
     27    hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir" 
     28 
     29    into_dir=${sub_dir%/*} 
     30    hdfs dfs -put $d "full-ef-json-files/$into_dir/." 
     31  done 
    1932done 
    2033 
     34echo "====" 
     35echo "====" 
     36echo "====" 
     37echo "Finished: processed top-level(s) $*" 
     38echo "====" 
     39echo "====" 
     40echo "===="