1 |
|
---|
2 | short_hostname=`hostname -s`
|
---|
3 |
|
---|
4 | #export SPARK_MASTER_HOST=gchead
|
---|
5 | export SPARK_MASTER_HOST=192.168.64.1
|
---|
6 |
|
---|
7 | export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
|
---|
8 | export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
|
---|
9 |
|
---|
10 | if [ "$short_hostname" = "nema" ] ; then
|
---|
11 | export ZOOKEEPER_SERVER=localhost:8181
|
---|
12 | export SOLR_NODES="localhost:8983 localhost:8984 localhost:8985 localhost:8986 localhost:8987 localhost:8988 localhost:8989 localhost:8990 localhost:8991 localhost:8992"
|
---|
13 |
|
---|
14 | elif [ "${short_hostname%[1-2]}" = "solr" ] ; then
|
---|
15 | export ZOOKEEPER_SERVER=solr1:8181
|
---|
16 | export SOLR_NODES="solr1:8983 solr1:8984 solr1:8985 solr1:8986 solr1:8987"
|
---|
17 | export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr2:8987"
|
---|
18 | export SOLR_SHARDS="/disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
|
---|
19 | export SOLR_SHARDS="$SOLR_SHARDS /disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
|
---|
20 | else
|
---|
21 | export ZOOKEEPER_SERVER=gchead:8181
|
---|
22 | # export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
|
---|
23 | # #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
|
---|
24 |
|
---|
25 | export ZOOKEEPER_SERVER=solr1-s:8181
|
---|
26 | export SOLR_NODES="solr1-s:8983 solr1-s:8984 solr1-s:8985 solr1-s:8986 solr1-s:8987"
|
---|
27 | export SOLR_NODES="$SOLR_NODES solr2-s:8983 solr2-s:8984 solr2-s:8985 solr2-s:8986 solr2-s:8987"
|
---|
28 |
|
---|
29 | fi
|
---|
30 |
|
---|
31 | if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
|
---|
32 | HDFS_HEAD=hdfs://gchead:9000
|
---|
33 | fi
|
---|
34 |
|
---|
35 | if [ "${short_hostname%[0-9]}" = "gc" ] ; then
|
---|
36 | export HTRC_EF_PACKAGE_HOME="/hdfsd05/dbbridge/gslis-cluster"
|
---|
37 | else
|
---|
38 | export HTRC_EF_PACKAGE_HOME=`pwd`
|
---|
39 | fi
|
---|
40 |
|
---|
41 | HTRC_EF_NETWORK_HOME=`pwd`
|
---|
42 |
|
---|
43 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
44 | echo ""
|
---|
45 | echo "****"
|
---|
46 | fi
|
---|
47 |
|
---|
48 | if [ "${short_hostname%[1-2]}" = "solr" ] ; then
|
---|
49 | export JAVA_HOME="/usr/lib/jvm/j2sdk1.8-oracle"
|
---|
50 | else
|
---|
51 | export JAVA_HOME="$HTRC_EF_NETWORK_HOME/jdk1.8.0"
|
---|
52 | fi
|
---|
53 |
|
---|
54 | export PATH="$JAVA_HOME/bin:$PATH"
|
---|
55 | #export _JAVA_OPTIONS="-Xmx512m"
|
---|
56 | #export _JAVA_OPTIONS="-Xmx1024m"
|
---|
57 | #export _JAVA_OPTIONS="-Xmx2048m"
|
---|
58 | export _JAVA_OPTIONS=
|
---|
59 | #export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
|
---|
60 |
|
---|
61 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
62 | echo "* Added in JDK 1.8 into PATH"
|
---|
63 | fi
|
---|
64 |
|
---|
65 | if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
|
---|
66 | source SETUP/setup-spark.bash
|
---|
67 | fi
|
---|
68 |
|
---|
69 | source SETUP/setup-zookeeper.bash
|
---|
70 | source SETUP/setup-solr.bash
|
---|
71 |
|
---|
72 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
73 | export MONGODB_CONFIG_HOSTS="gc0 gc1 gc2"
|
---|
74 | export MONGODB_CONFIG_PORT="27019"
|
---|
75 |
|
---|
76 | export MONGODB_ROUTER_HOSTS="gc3 gc4 gc5"
|
---|
77 | export MONGODB_ROUTER_PORT="27017"
|
---|
78 |
|
---|
79 | export MONGODB_REPLSET1_HOSTS="gc6 gc7"
|
---|
80 | export MONGODB_REPLSET2_HOSTS="gc8 gc9"
|
---|
81 | export MONGODB_REPLSET_PORT="27017"
|
---|
82 | ##export MONGODB_REPLSET_ARRAY=("$MONGODB_REPLSET1_HOSTS" "$MONGODB_REPLSET2_HOSTS")
|
---|
83 | export MONGODB_REPLSET_METALIST="MONGODB_REPLSET1_HOSTS MONGODB_REPLSET2_HOSTS"
|
---|
84 | export MONGODB_SHARD_HOSTS="$MONGODB_REPLSET1_HOSTS $MONGODB_REPLSET2_HOSTS"
|
---|
85 |
|
---|
86 | source SETUP/setup-mongodb.bash
|
---|
87 | fi
|
---|
88 |
|
---|
89 | export PATH="$HTRC_EF_NETWORK_HOME/SCRIPTS:$PATH"
|
---|
90 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
91 | echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
|
---|
92 | fi
|
---|
93 |
|
---|
94 | if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
|
---|
95 | spark_conf_slaves="$SPARK_HOME/conf/slaves"
|
---|
96 | if [ ! -f "$spark_conf_slaves" ] ; then
|
---|
97 | echo "****"
|
---|
98 | echo "* Populatig $spark_conf_slaves"
|
---|
99 | echo "* With: $SPARK_SLAVE_HOSTS"
|
---|
100 | echo "****"
|
---|
101 | for s in $SPARK_SLAVE_HOSTS ; do
|
---|
102 | echo $s >> "$spark_conf_slaves"
|
---|
103 | done
|
---|
104 | else
|
---|
105 | slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
|
---|
106 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
107 | echo "****"
|
---|
108 | echo "* Spark slaves: $slaves"
|
---|
109 | echo "****"
|
---|
110 | fi
|
---|
111 | fi
|
---|
112 | fi
|
---|
113 |
|
---|
114 |
|
---|
115 | zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
|
---|
116 | zookeeper_data_dir="$ZOOKEEPER_HOME/data"
|
---|
117 |
|
---|
118 | if [ ! -f "$zookeeper_config_file" ] ; then
|
---|
119 | echo "****"
|
---|
120 | echo "* Generating $zookeeper_config_file"
|
---|
121 | cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
|
---|
122 |
|
---|
123 | if [ ! -d "$zookeeper_data_dir" ] ; then
|
---|
124 | echo "* Creating Zookeeper dataDir:"
|
---|
125 | echo "* $zookeeper_data_dir"
|
---|
126 | mkdir "$zookeeper_data_dir"
|
---|
127 | fi
|
---|
128 | echo "****"
|
---|
129 | fi
|
---|
130 |
|
---|
131 | if [ "$short_hostname" = "gsliscluster1" ] ; then
|
---|
132 | echo "****"
|
---|
133 | echo "* Solr nodes: $SOLR_NODES"
|
---|
134 | echo "****"
|
---|
135 | else
|
---|
136 |
|
---|
137 | solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
|
---|
138 | if [ ! -d "$solr_configsets/htrc_configs" ] ; then
|
---|
139 | echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
|
---|
140 | tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
|
---|
141 | fi
|
---|
142 | fi
|
---|