Changeset 31152


Ignore:
Timestamp:
2016-11-23T23:11:16+13:00 (5 years ago)
Author:
davidb
Message:

Development of script

File:
1 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-FOREACH-HDFS-PUT.sh

    r31128 r31152  
    11#!/bin/bash
    22
    3 if [ "$#" != "1" ] ; then
    4   echo "Usage: $0 top-level-dir"
     3if [ "$#" == "0" ] ; then
     4  echo "Usage: $0 top-level-dir1 [top-level-dir2 ...]"
    55  exit
    66fi
    77
    8 toplevel_dir=$1
     8
     9for t in $* ; do
     10    echo ""
     11    echo ""
     12    echo "++++"
     13    echo "++++"
     14    echo "Top-level dir: $t"
     15    echo "++++"
     16    echo "++++"
    917
    1018
    11 for d in /data/features/ef-full/$toplevel_dir/*/* ; do
    12   sub_dir=${d##/data/features/ef-full/}
     19  for d in /data/features/ef-full/$t/pairtree_root/*/* ; do
     20    sub_dir=${d##/data/features/ef-full/}
    1321
    14   echo "Processing: $sub_dir"
    15   hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
     22    echo ""
     23    echo "****"
     24    echo "Processing: $sub_dir"
     25    echo "****"
    1626
    17   into_dir=${sub_dir%/*}
    18   hdfs dfs -put $d "full-ef-json-files/$into_dir/."
     27    hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
     28
     29    into_dir=${sub_dir%/*}
     30    hdfs dfs -put $d "full-ef-json-files/$into_dir/."
     31  done
    1932done
    2033
     34echo "===="
     35echo "===="
     36echo "===="
     37echo "Finished: processed top-level(s) $*"
     38echo "===="
     39echo "===="
     40echo "===="
Note: See TracChangeset for help on using the changeset viewer.