[31078] | 1 |
|
---|
[31103] | 2 | short_hostname=`hostname -s`
|
---|
[31078] | 3 |
|
---|
[31098] | 4 | #export SPARK_MASTER_HOST=gchead
|
---|
| 5 | export SPARK_MASTER_HOST=192.168.64.1
|
---|
| 6 |
|
---|
[31082] | 7 | export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
|
---|
| 8 | export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
|
---|
[31078] | 9 |
|
---|
[31233] | 10 | if [ "$short_hostname" = "nema" ] ; then
|
---|
| 11 | export ZOOKEEPER_SERVER=localhost:8181
|
---|
| 12 | export SOLR_NODES="localhost:8983 localhost:8984 localhost:8985 localhost:8986 localhost:8987 localhost:8988 localhost:8989 localhost:8990 localhost:8991 localhost:8992"
|
---|
| 13 | else
|
---|
| 14 | export ZOOKEEPER_SERVER=gchead:8181
|
---|
| 15 | export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
|
---|
| 16 | #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
|
---|
| 17 | fi
|
---|
[31078] | 18 |
|
---|
[31234] | 19 | if [ "$short_hostname" != "nema" ] ; then
|
---|
| 20 | HDFS_HEAD=hdfs://gchead:9000
|
---|
| 21 | fi
|
---|
[31233] | 22 |
|
---|
[31078] | 23 | export HTRC_EF_PACKAGE_HOME=`pwd`
|
---|
| 24 |
|
---|
[31103] | 25 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
| 26 | echo ""
|
---|
| 27 | echo "****"
|
---|
| 28 | fi
|
---|
[31078] | 29 |
|
---|
[31082] | 30 | export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0"
|
---|
| 31 | export PATH="$JAVA_HOME/bin:$PATH"
|
---|
[31098] | 32 | #export _JAVA_OPTIONS="-Xmx512m"
|
---|
| 33 | #export _JAVA_OPTIONS="-Xmx1024m"
|
---|
| 34 | #export _JAVA_OPTIONS="-Xmx2048m"
|
---|
| 35 | export _JAVA_OPTIONS=
|
---|
| 36 | #export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
|
---|
| 37 |
|
---|
[31103] | 38 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
| 39 | echo "* Added in JDK 1.8 into PATH"
|
---|
| 40 | fi
|
---|
[31078] | 41 |
|
---|
[31233] | 42 | if [ "$short_hostname" != "nema" ] ; then
|
---|
| 43 | source SETUP/setup-spark.bash
|
---|
| 44 | fi
|
---|
[31078] | 45 |
|
---|
| 46 | source SETUP/setup-zookeeper.bash
|
---|
| 47 | source SETUP/setup-solr.bash
|
---|
| 48 |
|
---|
| 49 |
|
---|
| 50 | export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH"
|
---|
[31103] | 51 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
| 52 | echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
|
---|
| 53 | fi
|
---|
[31078] | 54 |
|
---|
[31233] | 55 | if [ "$short_hostname" != "nema" ] ; then
|
---|
| 56 | spark_conf_slaves="$SPARK_HOME/conf/slaves"
|
---|
| 57 | if [ ! -f "$spark_conf_slaves" ] ; then
|
---|
[31103] | 58 | echo "****"
|
---|
[31233] | 59 | echo "* Populatig $spark_conf_slaves"
|
---|
| 60 | echo "* With: $SPARK_SLAVE_HOSTS"
|
---|
[31103] | 61 | echo "****"
|
---|
[31233] | 62 | for s in $SPARK_SLAVE_HOSTS ; do
|
---|
| 63 | echo $s >> "$spark_conf_slaves"
|
---|
| 64 | done
|
---|
| 65 | else
|
---|
| 66 | slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
|
---|
| 67 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
| 68 | echo "****"
|
---|
| 69 | echo "* Spark slaves: $slaves"
|
---|
| 70 | echo "****"
|
---|
| 71 | fi
|
---|
[31103] | 72 | fi
|
---|
[31078] | 73 | fi
|
---|
| 74 |
|
---|
[31098] | 75 |
|
---|
| 76 | zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
|
---|
| 77 | zookeeper_data_dir="$ZOOKEEPER_HOME/data"
|
---|
| 78 |
|
---|
| 79 | if [ ! -f "$zookeeper_config_file" ] ; then
|
---|
| 80 | echo "****"
|
---|
| 81 | echo "* Generating $zookeeper_config_file"
|
---|
| 82 | cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
|
---|
| 83 |
|
---|
| 84 | if [ ! -d "$zookeeper_data_dir" ] ; then
|
---|
| 85 | echo "* Creating Zookeeper dataDir:"
|
---|
| 86 | echo "* $zookeeper_data_dir"
|
---|
| 87 | mkdir "$zookeeper_data_dir"
|
---|
| 88 | fi
|
---|
| 89 | echo "****"
|
---|
| 90 | fi
|
---|
| 91 |
|
---|
[31103] | 92 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
| 93 | echo "****"
|
---|
| 94 | echo "* Solr nodes: $SOLR_NODES"
|
---|
| 95 | echo "****"
|
---|
[31234] | 96 | else
|
---|
| 97 | solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
|
---|
| 98 | if [ ! -d "$solr_configsets/htrc_configs" ] ; then
|
---|
| 99 | echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
|
---|
| 100 | tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
|
---|
| 101 | fi
|
---|
[31103] | 102 | fi
|
---|