root/other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash @ 32173

Revision 32173, 6.1 KB (checked in by davidb, 19 months ago)

Shift to using Solr7 setup

Line 
1
2short_hostname=`hostname -s`
3
4#export SPARK_MASTER_HOST=gchead
5export SPARK_MASTER_HOST=192.168.64.1
6
7export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
8export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
9
10if [ "$short_hostname" = "nema" ] ; then
11  export ZOOKEEPER_SERVER=localhost:8181
12  export SOLR_NODES="localhost:8983 localhost:8984 localhost:8985 localhost:8986 localhost:8987 localhost:8988 localhost:8989 localhost:8990 localhost:8991 localhost:8992"
13 
14elif [ "${short_hostname%[1-2]}" = "solr" ] ; then
15  export ZOOKEEPER_SERVER=solr1:8181
16
17  export SOLR_NODES="solr1:8983 solr1:8984 solr1:8985 solr1:8986 solr1:8987"
18  export SOLR_NODES="$SOLR_NODES solr1:8988 solr1:8989 solr1:8990 solr1:8991 solr1:8992"
19  export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr2:8987"
20  export SOLR_NODES="$SOLR_NODES solr2:8988 solr2:8989 solr2:8990 solr2:8991 solr2:8992"
21
22  export SOLR_SHARDS="/disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
23  export SOLR_SHARDS="$SOLR_SHARDS /disk5/solr-full-ef /disk6/solr-full-ef /disk7/solr-full-ef /disk8/solr-full-ef /disk9/solr-full-ef"
24  export SOLR_SHARDS="$SOLR_SHARDS /disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
25  export SOLR_SHARDS="$SOLR_SHARDS /disk5/solr-full-ef /disk6/solr-full-ef /disk7/solr-full-ef /disk8/solr-full-ef /disk9/solr-full-ef"
26
27  #export SOLR_JAVA_MEM="-Xms10g -Xmx15g"
28  # export SOLR_JAVA_MEM="-Xms5g -Xmx7g"
29  export SOLR_JAVA_MEM="-Xmx14g" 
30
31#  export SOLR7_NODES="solr1:9983 solr1:9984 solr1:9985 solr1:9986 solr1:9987"
32#  export SOLR7_NODES="$SOLR_NODES solr1:9988 solr1:9989 solr1:9990 solr1:9991 solr1:9992"
33#  export SOLR7_NODES="$SOLR_NODES solr2:9983 solr2:9984 solr2:9985 solr2:9986 solr2:9987"
34#  export SOLR7_NODES="$SOLR_NODES solr2:9988 solr2:9989 solr2:9990 solr2:9991 solr2:9992"
35
36#  export SOLR7_SHARDS="/disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
37#  export SOLR7_SHARDS="$SOLR_SHARDS /disk5/solr-full-ef /disk6/solr-full-ef /disk7/solr-full-ef /disk8/solr-full-ef /disk9/solr-full-ef"
38#  export SOLR7_SHARDS="$SOLR_SHARDS /disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
39#  export SOLR7_SHARDS="$SOLR_SHARDS /disk5/solr-full-ef /disk6/solr-full-ef /disk7/solr-full-ef /disk8/solr-full-ef /disk9/solr-full-ef"
40
41 
42else
43  export ZOOKEEPER_SERVER=gchead:8181
44#  export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
45#  #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
46
47  export ZOOKEEPER_SERVER=solr1-s:8181
48  export SOLR_NODES="solr1-s:8983 solr1-s:8984 solr1-s:8985 solr1-s:8986 solr1-s:8987"
49  export SOLR_NODES="$SOLR_NODES solr2-s:8983 solr2-s:8984 solr2-s:8985 solr2-s:8986 solr2-s:8987"
50
51fi
52
53if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
54  HDFS_HEAD=hdfs://gchead:9000
55fi
56
57if [ "${short_hostname%[0-9]}" = "gc" ] ; then
58  export HTRC_EF_PACKAGE_HOME="/hdfsd05/dbbridge/gslis-cluster"
59else
60  export HTRC_EF_PACKAGE_HOME=`pwd`
61fi
62
63HTRC_EF_NETWORK_HOME=`pwd`
64
65if [ "$short_hostname" = "gsliscluster1" ] ; then
66  echo ""
67  echo "****"
68fi
69
70if [ "${short_hostname%[1-2]}" = "solr" ] ; then
71  export JAVA_HOME="/usr/lib/jvm/j2sdk1.8-oracle"
72else
73  export JAVA_HOME="$HTRC_EF_NETWORK_HOME/jdk1.8.0"
74fi
75
76export PATH="$JAVA_HOME/bin:$PATH"
77#export _JAVA_OPTIONS="-Xmx512m"
78#export _JAVA_OPTIONS="-Xmx1024m"
79#export _JAVA_OPTIONS="-Xmx2048m"
80export _JAVA_OPTIONS=
81#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
82
83if [ "$short_hostname" = "gsliscluster1" ] ; then
84  echo "* Added in JDK 1.8 into PATH"
85fi
86
87if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then
88  source SETUP/setup-spark.bash
89fi
90
91source SETUP/setup-zookeeper.bash
92#source SETUP/setup-solr.bash
93source SETUP/setup-solr7.bash
94
95if [ "$short_hostname" = "gsliscluster1" ] ; then
96  export MONGODB_CONFIG_HOSTS="gc0 gc1 gc2"
97  export MONGODB_CONFIG_PORT="27019"
98
99  export MONGODB_ROUTER_HOSTS="gc3 gc4 gc5"
100  export MONGODB_ROUTER_PORT="27017"
101
102  export MONGODB_REPLSET1_HOSTS="gc6 gc7"
103  export MONGODB_REPLSET2_HOSTS="gc8 gc9"
104  export MONGODB_REPLSET_PORT="27017"
105  ##export MONGODB_REPLSET_ARRAY=("$MONGODB_REPLSET1_HOSTS" "$MONGODB_REPLSET2_HOSTS")
106  export MONGODB_REPLSET_METALIST="MONGODB_REPLSET1_HOSTS MONGODB_REPLSET2_HOSTS"
107  export MONGODB_SHARD_HOSTS="$MONGODB_REPLSET1_HOSTS $MONGODB_REPLSET2_HOSTS"
108
109  source SETUP/setup-mongodb.bash
110fi
111
112export PATH="$HTRC_EF_NETWORK_HOME/SCRIPTS:$PATH"
113if [ "$short_hostname" = "gsliscluster1" ] ; then
114  echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
115fi
116
117if [ "$short_hostname" != "nema" ] && [ "${short_hostname%[1-2]}" != "solr" ] ; then   
118  spark_conf_slaves="$SPARK_HOME/conf/slaves"
119  if [ ! -f "$spark_conf_slaves" ] ; then
120    echo "****"
121    echo "* Populatig $spark_conf_slaves"
122    echo "* With: $SPARK_SLAVE_HOSTS"
123    echo "****"
124    for s in $SPARK_SLAVE_HOSTS ; do
125      echo $s >> "$spark_conf_slaves"
126    done
127  else
128    slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
129    if [ "$short_hostname" = "gsliscluster1" ] ; then
130      echo "****"
131      echo "* Spark slaves: $slaves"
132      echo "****"
133    fi
134  fi
135fi
136
137
138zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
139zookeeper_data_dir="$ZOOKEEPER_HOME/data"
140
141if [ ! -f "$zookeeper_config_file" ] ; then
142  echo "****"
143  echo "* Generating $zookeeper_config_file"
144  cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
145
146  if [ ! -d "$zookeeper_data_dir" ] ; then
147    echo "* Creating Zookeeper dataDir:"
148    echo "*   $zookeeper_data_dir"
149    mkdir "$zookeeper_data_dir"
150  fi
151  echo "****"
152fi
153
154if [ "$short_hostname" = "gsliscluster1" ] ; then
155  echo "****"
156  echo "* Solr nodes: $SOLR_NODES"
157  echo "****"
158else
159 
160  solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
161  if [ ! -d "$solr_configsets/htrc_configs" ] ; then
162    echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
163    tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
164  fi
165fi
Note: See TracBrowser for help on using the browser.