Changeset 30926

Timestamp:

2016-10-26T09:09:03+13:00 (7 years ago)

Author:

davidb

Message:

Restructuring of RUN scripts to be more flexible

Location:

other-projects/hathitrust/solr-extracted-features/trunk

Files:

• RUN-PD-CLUSTER.bash (added)
• RUN-PD-LOCAL.bash (added)
• _RUN.bash (moved) (moved from other-projects/hathitrust/solr-extracted-features/trunk/RUN.bash ) (1 diff)

other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

@@ -1 +1 @@
 #!/bin/bash
 
-#input_dir=pd-ef-json-files
-input_dir="hdfs://10.10.0.52:9000/user/htrc/pd-ef-json-files"
-output_dir=pd-solr-json-files
+# To work, the follow bash variables need to have been set:
+#
+#  json_filelist input_dir output_dir
+#
+# Typically done through running a wrapper script, such as:
+#
+#  RUN-PD-CLUSTER.bash
 
-#master_opt="--master local[4]"
-master_opt="--master spark://10.10.0.52:7077"
+if [ "x$json_filelist" = "x" ] ; then
+    echo "_RUN.bash: Failed to set 'json_filelist'" 1>2
+    exit
+fi
+
+if [ "x$input_dir" = "x" ] ; then
+    echo "_RUN.bash: Failed to set 'input_dir'" 1>2
+    exit
+fi
+
+if [ "x$output_dir" = "x" ] ; then
+    echo "_RUN.bash: Failed to set 'output_dir'" 1>2
+    exit
+fi
 
 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
 
-if [ $# -ge 1 ] ; then
-    file_listing=$1
-    shift
-    $base_cmd --json-filelist="$file_listing" $input_dir $output_dir $*
-else
-    echo "****"
-    echo "* Processing all files in: $input_dir"
-    echo "****"
-    $base_cmd $input_dir/*.json.bz2 $output_dir $*
-fi
+$base_cmd --json-filelist="$json_filelist" "$input_dir" "$output_dir" $*
 
 #    spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-json-files $*