source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash@ 31103

Last change on this file since 31103 was 31103, checked in by davidb, 5 years ago

Changes made after testing with 20 solr nodes

File size: 2.5 KB
Line 
1
2short_hostname=`hostname -s`
3
4#export SPARK_MASTER_HOST=gchead
5export SPARK_MASTER_HOST=192.168.64.1
6
7export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
8export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
9
10export ZOOKEEPER_SERVER=gchead:8181
11export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
12#export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
13
14export HTRC_EF_PACKAGE_HOME=`pwd`
15
16if [ "$short_hostname" = "gsliscluster1" ] ; then
17 echo ""
18 echo "****"
19fi
20
21export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0"
22export PATH="$JAVA_HOME/bin:$PATH"
23#export _JAVA_OPTIONS="-Xmx512m"
24#export _JAVA_OPTIONS="-Xmx1024m"
25#export _JAVA_OPTIONS="-Xmx2048m"
26export _JAVA_OPTIONS=
27#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
28
29if [ "$short_hostname" = "gsliscluster1" ] ; then
30 echo "* Added in JDK 1.8 into PATH"
31fi
32
33source SETUP/setup-spark.bash
34
35source SETUP/setup-zookeeper.bash
36source SETUP/setup-solr.bash
37
38
39export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH"
40if [ "$short_hostname" = "gsliscluster1" ] ; then
41 echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
42fi
43
44spark_conf_slaves="$SPARK_HOME/conf/slaves"
45if [ ! -f "$spark_conf_slaves" ] ; then
46 echo "****"
47 echo "* Populatig $spark_conf_slaves"
48 echo "* With: $SPARK_SLAVE_HOSTS"
49 echo "****"
50 for s in $SPARK_SLAVE_HOSTS ; do
51 echo $s >> "$spark_conf_slaves"
52 done
53else
54 slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
55 if [ "$short_hostname" = "gsliscluster1" ] ; then
56 echo "****"
57 echo "* Spark slaves: $slaves"
58 echo "****"
59 fi
60fi
61
62
63zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
64zookeeper_data_dir="$ZOOKEEPER_HOME/data"
65
66if [ ! -f "$zookeeper_config_file" ] ; then
67 echo "****"
68 echo "* Generating $zookeeper_config_file"
69 cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
70
71 if [ ! -d "$zookeeper_data_dir" ] ; then
72 echo "* Creating Zookeeper dataDir:"
73 echo "* $zookeeper_data_dir"
74 mkdir "$zookeeper_data_dir"
75 fi
76 echo "****"
77fi
78
79solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
80if [ ! -d "$solr_configsets/htrc_configs" ] ; then
81 echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
82 tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
83fi
84
85if [ "$short_hostname" = "gsliscluster1" ] ; then
86 echo "****"
87 echo "* Solr nodes: $SOLR_NODES"
88 echo "****"
89fi
Note: See TracBrowser for help on using the repository browser.