Changeset 30923

Timestamp:

2016-10-25T23:28:22+13:00 (6 years ago)

Author:

davidb

Message:

Rough cut version that reads in each JSON file over HDFS

File:

• other-projects/hathitrust/solr-extracted-features/trunk/RUN.bash (modified) (2 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/RUN.bash

@@ -1 +1 @@
 #!/bin/bash
 
-input_dir=pd-ef-json-files
+#input_dir=pd-ef-json-files
+input_dir="hdfs://10.10.0.52:9000/user/htrc/pd-ef-json-files"
 output_dir=pd-solr-json-files
 
-master_opt="--master local[4]"
+#master_opt="--master local[4]"
+master_opt="--master spark://10.10.0.52:7077"
+
 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
 
 if [ $# -ge 1 ] ; then
-    file_listing=shift $*
+    file_listing=$1
+    shift
     $base_cmd --json-filelist="$file_listing" $input_dir $output_dir $*
 else
@@ -15 +19 @@
     echo "* Processing all files in: $input_dir"
     echo "****"
-    $base_cmd $input_dir $output_dir $*
+    $base_cmd $input_dir/*.json.bz2 $output_dir $*
 fi