Changeset 30929

Timestamp:

2016-10-26T09:47:36+13:00 (6 years ago)

Author:

davidb

Message:

Tweaks made while testing the script

Location:

other-projects/hathitrust/solr-extracted-features/trunk

Files:

• RUN-PD-CLUSTER.bash (modified) (2 diffs)
• _RUN.bash (modified) (1 diff)

other-projects/hathitrust/solr-extracted-features/trunk/RUN-PD-CLUSTER.bash

@@ -2 +2 @@
 
 json_filelist=${1:-pd-file-listing-step10000.txt}
+shift
 
 input_dir="hdfs://master:9000/user/htrc/pd-ef-json-files"
@@ -11 +12 @@
 master_opt="--master spark://10.10.0.52:7077"
 
-./_RUN.bash
+. ./_RUN.bash

other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

@@ -24 +24 @@
 fi
 
+echo
+echo "****"
+echo "* Checking for Spark and Hadoop daemons"
+echo "****"
+jps | sed 's/^/* /g'
+echo "****"
+echo "* Done"
+echo "****"
+echo
+
 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
 
-$base_cmd --json-filelist="$json_filelist" "$input_dir" "$output_dir" $*
+cmd="$base_cmd --json-filelist=\"$json_filelist\" $input_dir $output_dir $*"
+
+echo "****"
+echo "* Lauching:"
+echo "*   $cmd"
+echo "****"
+echo "* Monitor progress through:"
+echo "*   http://10.10.0.52:8080/"
+echo "****"
+echo
+
+$cmd
 
 #    spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-json-files $*