source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash@ 31367

Last change on this file since 31367 was 31367, checked in by davidb, 6 years ago

Changes to work with solr1 and solr2

File size: 4.2 KB
Line 
1
2short_hostname=`hostname -s`
3
4#export SPARK_MASTER_HOST=gchead
5export SPARK_MASTER_HOST=192.168.64.1
6
7export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
8export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
9
10if [ "$short_hostname" = "nema" ] ; then
11 export ZOOKEEPER_SERVER=localhost:8181
12 export SOLR_NODES="localhost:8983 localhost:8984 localhost:8985 localhost:8986 localhost:8987 localhost:8988 localhost:8989 localhost:8990 localhost:8991 localhost:8992"
13
14elif [ "${short_hostname%[1-2]}" = "solr" ] ; then
15 export ZOOKEEPER_SERVER=solr1:8181
16 export SOLR_NODES="solr1:8983 solr1:8984 solr1:8985 solr1:8986 solr1:8987"
17 export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr:8987"
18
19else
20 export ZOOKEEPER_SERVER=gchead:8181
21 export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
22 #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
23fi
24
25if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
26 HDFS_HEAD=hdfs://gchead:9000
27fi
28
29if [ "${short_hostname%[0-9]}" = "gc" ] ; then
30 export HTRC_EF_PACKAGE_HOME="/hdfsd05/dbbridge/gslis-cluster"
31else
32 export HTRC_EF_PACKAGE_HOME=`pwd`
33fi
34
35HTRC_EF_NETWORK_HOME=`pwd`
36
37if [ "$short_hostname" = "gsliscluster1" ] ; then
38 echo ""
39 echo "****"
40fi
41
42if [ "${short_hostname%[1-2]}" = "solr" ] ; then
43 export JAVA_HOME="/usr/lib/jvm/j2sdk1.8-oracle"
44else
45 export JAVA_HOME="$HTRC_EF_NETWORK_HOME/jdk1.8.0"
46fi
47
48export PATH="$JAVA_HOME/bin:$PATH"
49#export _JAVA_OPTIONS="-Xmx512m"
50#export _JAVA_OPTIONS="-Xmx1024m"
51#export _JAVA_OPTIONS="-Xmx2048m"
52export _JAVA_OPTIONS=
53#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
54
55if [ "$short_hostname" = "gsliscluster1" ] ; then
56 echo "* Added in JDK 1.8 into PATH"
57fi
58
59if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
60 source SETUP/setup-spark.bash
61fi
62
63source SETUP/setup-zookeeper.bash
64source SETUP/setup-solr.bash
65
66if [ "$short_hostname" = "gsliscluster1" ] ; then
67 export MONGODB_CONFIG_HOSTS="gc0 gc1 gc2"
68 export MONGODB_CONFIG_PORT="27019"
69
70 export MONGODB_ROUTER_HOSTS="gc3 gc4 gc5"
71 export MONGODB_ROUTER_PORT="27017"
72
73 export MONGODB_REPLSET1_HOSTS="gc6 gc7"
74 export MONGODB_REPLSET2_HOSTS="gc8 gc9"
75 export MONGODB_REPLSET_PORT="27017"
76 ##export MONGODB_REPLSET_ARRAY=("$MONGODB_REPLSET1_HOSTS" "$MONGODB_REPLSET2_HOSTS")
77 export MONGODB_REPLSET_METALIST="MONGODB_REPLSET1_HOSTS MONGODB_REPLSET2_HOSTS"
78 export MONGODB_SHARD_HOSTS="$MONGODB_REPLSET1_HOSTS $MONGODB_REPLSET2_HOSTS"
79
80 source SETUP/setup-mongodb.bash
81fi
82
83export PATH="$HTRC_EF_NETWORK_HOME/SCRIPTS:$PATH"
84if [ "$short_hostname" = "gsliscluster1" ] ; then
85 echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
86fi
87
88if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
89 spark_conf_slaves="$SPARK_HOME/conf/slaves"
90 if [ ! -f "$spark_conf_slaves" ] ; then
91 echo "****"
92 echo "* Populatig $spark_conf_slaves"
93 echo "* With: $SPARK_SLAVE_HOSTS"
94 echo "****"
95 for s in $SPARK_SLAVE_HOSTS ; do
96 echo $s >> "$spark_conf_slaves"
97 done
98 else
99 slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
100 if [ "$short_hostname" = "gsliscluster1" ] ; then
101 echo "****"
102 echo "* Spark slaves: $slaves"
103 echo "****"
104 fi
105 fi
106fi
107
108
109zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
110zookeeper_data_dir="$ZOOKEEPER_HOME/data"
111
112if [ ! -f "$zookeeper_config_file" ] ; then
113 echo "****"
114 echo "* Generating $zookeeper_config_file"
115 cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
116
117 if [ ! -d "$zookeeper_data_dir" ] ; then
118 echo "* Creating Zookeeper dataDir:"
119 echo "* $zookeeper_data_dir"
120 mkdir "$zookeeper_data_dir"
121 fi
122 echo "****"
123fi
124
125if [ "$short_hostname" = "gsliscluster1" ] ; then
126 echo "****"
127 echo "* Solr nodes: $SOLR_NODES"
128 echo "****"
129else
130 solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
131 if [ ! -d "$solr_configsets/htrc_configs" ] ; then
132 echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
133 tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
134 fi
135fi
Note: See TracBrowser for help on using the repository browser.