Changeset 30939 for other-projects/hathitrust/solr-extracted-features

Timestamp:

2016-10-26T13:57:19+13:00 (7 years ago)

Author:

davidb

Message:

Minor tweaks

File:

• other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash (modified) (5 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

@@ -27 +27 @@
 run_jps_daemons=""
 run_jps_daemons_suffix="daemon"
+using_hdfs=0
 
 if [ "x${input_dir##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then
@@ -32 +33 @@
     run_jps=1
     run_jps_daemons="Spark"
+    using_hdfs=1
 fi
 
@@ -50 +52 @@
   echo "* Checking for $run_jps_daemons $run_jps_daemons_suffix"
   echo "****"
-  jps | sed 's/^/* /g'
+  jps | egrep -v " Jps$" |  sed 's/^/* /g'
   echo "****"
   echo "* Done"
@@ -59 +61 @@
 fi
 
+if [ "$using_hdfs" = "1" ] ; then
+  hadoop fs -test -d "$output_dir"
+
+  if [ $? != 0 ] ; then
+      echo "Creating directory:"
+      echo "  $output_dir"
+  fi
+fi
+    
 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
@@ -74 +85 @@
 fi
 echo
-sleep 1
+sleep 2
 
 $cmd
 
-#    spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-json-files $* 
-
-# spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target\htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step1000.txt json-files solr-files $*