Changeset 30934 for other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

Timestamp:

2016-10-26T11:05:28+13:00 (7 years ago)

Author:

davidb

Message:

Providing json-filelist now a compulsory argument, rather than an option

File:

• other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash (modified) (1 diff)

other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

@@ -24 +24 @@
 fi
 
-echo
-echo "****"
-echo "* Checking for Spark and Hadoop daemons"
-echo "****"
-jps | sed 's/^/* /g'
-echo "****"
-echo "* Done"
-echo "****"
-echo
+run_jps=0
+run_jps_daemons=""
+run_jps_daemons_suffix="daemon"
+
+if [ "x${input_dir##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then
+    # Evidence of running command over HDFS
+    run_jps=1
+    run_jps_daemons="Spark"
+fi
+
+if [ "x${master_op##--master spark://*}" = "x" ] ; then
+    # Evidence of running command submitted to Spark cluster
+    run_jps=1
+    if [ "x$run_jps_daemons" != "x" ] ; then
+        run_jps_daemons="$run_jps_daemons and Hadoop"
+    run_jps_daemons_suffix="daemons"
+    else
+        run_jps_daemons="Hadoop"
+    fi
+fi
+
+if [ "$run_jps" = "1" ] ; then
+  echo
+  echo "****"
+  echo "* Checking for $run_jps_daemons $run_jps_daemons_suffix"
+  echo "****"
+  jps | sed 's/^/* /g'
+  echo "****"
+  echo "* Done"
+  echo "****"
+  echo
+
+fi
 
 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
 
-cmd="$base_cmd --json-filelist=\"$json_filelist\" $input_dir $output_dir $*"
+cmd="$base_cmd --verbosity 1 $json_filelist $input_dir $output_dir $*"
 
 echo "****"