Changeset 31152 for other-projects/hathitrust/wcsa

Timestamp:

2016-11-23T23:11:16+13:00 (7 years ago)

Author:

davidb

Message:

Development of script

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-FOREACH-HDFS-PUT.sh (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/PAIRTREE-FOREACH-HDFS-PUT.sh

@@ -1 +1 @@
 #!/bin/bash
 
-if [ "$#" != "1" ] ; then
-  echo "Usage: $0 top-level-dir"
+if [ "$#" == "0" ] ; then
+  echo "Usage: $0 top-level-dir1 [top-level-dir2 ...]"
   exit
 fi
 
-toplevel_dir=$1
+
+for t in $* ; do
+    echo ""
+    echo ""
+    echo "++++"
+    echo "++++"
+    echo "Top-level dir: $t"
+    echo "++++"
+    echo "++++"
 
 
-for d in /data/features/ef-full/$toplevel_dir/*/* ; do
-  sub_dir=${d##/data/features/ef-full/}
+  for d in /data/features/ef-full/$t/pairtree_root/*/* ; do
+    sub_dir=${d##/data/features/ef-full/}
 
-  echo "Processing: $sub_dir"
-  hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
+    echo ""
+    echo "****"
+    echo "Processing: $sub_dir"
+    echo "****"
 
-  into_dir=${sub_dir%/*}
-  hdfs dfs -put $d "full-ef-json-files/$into_dir/."
+    hdfs dfs -mkdir -p "full-ef-json-files/$sub_dir"
+
+    into_dir=${sub_dir%/*}
+    hdfs dfs -put $d "full-ef-json-files/$into_dir/."
+  done
 done
 
+echo "===="
+echo "===="
+echo "===="
+echo "Finished: processed top-level(s) $*"
+echo "===="
+echo "===="
+echo "===="