Show
Ignore:
Timestamp:
26.10.2016 13:57:19 (3 years ago)
Author:
davidb
Message:

Minor tweaks

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

    r30936 r30939  
    2727run_jps_daemons="" 
    2828run_jps_daemons_suffix="daemon" 
     29using_hdfs=0 
    2930 
    3031if [ "x${input_dir##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then 
     
    3233    run_jps=1 
    3334    run_jps_daemons="Spark" 
     35    using_hdfs=1 
    3436fi 
    3537 
     
    5052  echo "* Checking for $run_jps_daemons $run_jps_daemons_suffix" 
    5153  echo "****" 
    52   jps | sed 's/^/* /g' 
     54  jps | egrep -v " Jps$" |  sed 's/^/* /g' 
    5355  echo "****" 
    5456  echo "* Done" 
     
    5961fi 
    6062 
     63if [ "$using_hdfs" = "1" ] ; then 
     64  hadoop fs -test -d "$output_dir" 
     65 
     66  if [ $? != 0 ] ; then 
     67      echo "Creating directory:" 
     68      echo "  $output_dir" 
     69  fi 
     70fi 
     71     
    6172self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar 
    6273base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar" 
     
    7485fi 
    7586echo 
    76 sleep 1 
     87sleep 2 
    7788 
    7889$cmd 
    7990 
    80 #    spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-json-files $*  
    81  
    82 # spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target\htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step1000.txt json-files solr-files $*