Changeset 32107 for other-projects

Timestamp:

2018-01-16T23:17:42+13:00 (6 years ago)

Author:

davidb

Message:

Rekindling the ability to run a JSON-filelist Spark run via YARN

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• JSONLIST-YARN-INGEST.sh (added)
• scripts/_RUN.sh (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/scripts/_RUN.sh

@@ -3 +3 @@
 # To work, the follow bash variables need to have been set:
 #
-#  json_filelist input_dir output_dir
+#  seq_file (output_dir optional)
+#
+# Or:
+#
+#  json_filelist input_dir (output_dir optional)
 #
 # Typically done through running a wrapper script, such as:
 #
-#  RUN-PD-CLUSTER.bash
+#  FULL-RUN-YARN-SPARK.sh
 
-if [ "x$json_filelist" = "x" ] ; then
-    echo "_RUN.bash: Failed to set 'json_filelist'" 1>&2
-    exit
+
+show_usage=1
+class_mode=""
+if [ "x$seq_file" != "x" ] ; then
+    show_usage=0
+    class_mode="seq"
+else
+
+    
+    if [ "x$json_filelist" != "x" ] ; then
+    class_mode="json"
+    fi
+
+    if [ "x$input_dir" != "x" ] ; then
+    if [ $show_usage = "json" ] ; then
+        show_usage=0
+    fi
+    fi
+
 fi
 
-if [ "x$input_dir" = "x" ] ; then
-    echo "_RUN.bash: Failed to set 'input_dir'" 1>&2
-    exit
+if [ $show_usage = "1" ] ; then
+    echo "_RUN.bash: Failed to set 'seq_file' or 'input_dir json_filelist" 1>&2
+    exit 1
 fi
+
 
 #if [ "x$output_dir" = "x" ] ; then
@@ -29 +50 @@
 using_hdfs=0
 
-if [ "x${input_dir##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then
+if [ "$class_mode" = "seq" ] ; then
+  if [ "x${seq_file##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then
     # Evidence of running command over HDFS
     run_jps=1
     run_jps_daemons="Spark"
     using_hdfs=1
+  fi
+fi
+
+if [ "$class_mode" = "json" ] ; then
+  if [ "x${input_dir##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then
+    # Evidence of running command over HDFS
+    run_jps=1
+    run_jps_daemons="Spark"
+    using_hdfs=1
+  fi
 fi
 
@@ -83 +115 @@
 cmd="spark-submit --class $classmain $master_opt $self_contained_jar"
 
-if [ "x$solr_url" != "x" ] ; then
-    cmd="$cmd --solr-url $solr_url"
-fi
+if [ "$classmain" = "org.hathitrust.extractedfeatures.ProcessForSolrIngest" ] || [ "$classmain" = "org.hathitrust.extractedfeatures.ProcessForSolrIngestJSONFilelist" ] ; then
+  if [ "x$solr_base_url" != "x" ] ; then
+      cmd="$cmd --solr-base-url $solr_base_url"
+  fi
 
-if [ "x$output_dir" != "x" ] ; then
+  if [ "x$output_dir" != "x" ] ; then
     cmd="$cmd --output-dir $output_dir"
+  fi
 fi
 
 
-cmd="$cmd --properties ef-solr.properties $input_dir $json_filelist $*"
+if [ "$class_mode" = "seq" ] ; then
+    cmd="$cmd --properties ef-solr.properties $seq_file $*"
+    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $seq_file $*"
+else
+    cmd="$cmd --properties ef-solr.properties $json_filelist $input_dir $*"
+    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $json_filelist $input_dir $*"
+
+fi
 
 echo "****"