short_hostname=`hostname -s` #export SPARK_MASTER_HOST=gchead export SPARK_MASTER_HOST=192.168.64.1 export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077 export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9" export ZOOKEEPER_SERVER=gchead:8181 export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983" #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984" export HTRC_EF_PACKAGE_HOME=`pwd` if [ "$short_hostname" = "gsliscluster1" ] ; then echo "" echo "****" fi export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0" export PATH="$JAVA_HOME/bin:$PATH" #export _JAVA_OPTIONS="-Xmx512m" #export _JAVA_OPTIONS="-Xmx1024m" #export _JAVA_OPTIONS="-Xmx2048m" export _JAVA_OPTIONS= #export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" if [ "$short_hostname" = "gsliscluster1" ] ; then echo "* Added in JDK 1.8 into PATH" fi source SETUP/setup-spark.bash source SETUP/setup-zookeeper.bash source SETUP/setup-solr.bash export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH" if [ "$short_hostname" = "gsliscluster1" ] ; then echo "* Added in HTRC EF PACKAGE remote scripting into PATH" fi spark_conf_slaves="$SPARK_HOME/conf/slaves" if [ ! -f "$spark_conf_slaves" ] ; then echo "****" echo "* Populatig $spark_conf_slaves" echo "* With: $SPARK_SLAVE_HOSTS" echo "****" for s in $SPARK_SLAVE_HOSTS ; do echo $s >> "$spark_conf_slaves" done else slaves=`cat "$spark_conf_slaves" | tr '\n' ' '` if [ "$short_hostname" = "gsliscluster1" ] ; then echo "****" echo "* Spark slaves: $slaves" echo "****" fi fi zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg" zookeeper_data_dir="$ZOOKEEPER_HOME/data" if [ ! -f "$zookeeper_config_file" ] ; then echo "****" echo "* Generating $zookeeper_config_file" cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file" if [ ! -d "$zookeeper_data_dir" ] ; then echo "* Creating Zookeeper dataDir:" echo "* $zookeeper_data_dir" mkdir "$zookeeper_data_dir" fi echo "****" fi solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets" if [ ! -d "$solr_configsets/htrc_configs" ] ; then echo "Untarring htrc_configs.tar.gz in Solr configtests directory" tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets" fi if [ "$short_hostname" = "gsliscluster1" ] ; then echo "****" echo "* Solr nodes: $SOLR_NODES" echo "****" fi