source: other-projects/hathitrust/solr-extracted-features/trunk/RUN.bash@ 30918

Last change on this file since 30918 was 30918, checked in by davidb, 4 years ago

More flexible command-line args

  • Property svn:executable set to *
File size: 931 bytes
Line 
1#!/bin/bash
2
3input_dir=pd-ef-json-files
4output_dir=pd-solr-json-files
5
6master_opt="--master local[4]"
7self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
8base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
9
10if [ $# -ge 1 ] ; then
11 file_listing=shift $*
12 $base_cmd --json-filelist="$file_listing" $input_dir $output_dir $*
13else
14 echo "****"
15 echo "* Processing all files in: $input_dir"
16 echo "****"
17 $base_cmd $input_dir $output_dir $*
18fi
19
20# spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-json-files $*
21
22# spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target\htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step1000.txt json-files solr-files $*
Note: See TracBrowser for help on using the repository browser.