source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash@ 31234

Last change on this file since 31234 was 31234, checked in by davidb, 4 years ago

More selective control of what to source/setup depending on hostname

File size: 2.9 KB
Line 
1
2short_hostname=`hostname -s`
3
4#export SPARK_MASTER_HOST=gchead
5export SPARK_MASTER_HOST=192.168.64.1
6
7export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
8export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
9
10if [ "$short_hostname" = "nema" ] ; then
11 export ZOOKEEPER_SERVER=localhost:8181
12 export SOLR_NODES="localhost:8983 localhost:8984 localhost:8985 localhost:8986 localhost:8987 localhost:8988 localhost:8989 localhost:8990 localhost:8991 localhost:8992"
13else
14 export ZOOKEEPER_SERVER=gchead:8181
15 export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
16 #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
17fi
18
19if [ "$short_hostname" != "nema" ] ; then
20 HDFS_HEAD=hdfs://gchead:9000
21fi
22
23export HTRC_EF_PACKAGE_HOME=`pwd`
24
25if [ "$short_hostname" = "gsliscluster1" ] ; then
26 echo ""
27 echo "****"
28fi
29
30export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0"
31export PATH="$JAVA_HOME/bin:$PATH"
32#export _JAVA_OPTIONS="-Xmx512m"
33#export _JAVA_OPTIONS="-Xmx1024m"
34#export _JAVA_OPTIONS="-Xmx2048m"
35export _JAVA_OPTIONS=
36#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
37
38if [ "$short_hostname" = "gsliscluster1" ] ; then
39 echo "* Added in JDK 1.8 into PATH"
40fi
41
42if [ "$short_hostname" != "nema" ] ; then
43 source SETUP/setup-spark.bash
44fi
45
46source SETUP/setup-zookeeper.bash
47source SETUP/setup-solr.bash
48
49
50export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH"
51if [ "$short_hostname" = "gsliscluster1" ] ; then
52 echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
53fi
54
55if [ "$short_hostname" != "nema" ] ; then
56 spark_conf_slaves="$SPARK_HOME/conf/slaves"
57 if [ ! -f "$spark_conf_slaves" ] ; then
58 echo "****"
59 echo "* Populatig $spark_conf_slaves"
60 echo "* With: $SPARK_SLAVE_HOSTS"
61 echo "****"
62 for s in $SPARK_SLAVE_HOSTS ; do
63 echo $s >> "$spark_conf_slaves"
64 done
65 else
66 slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
67 if [ "$short_hostname" = "gsliscluster1" ] ; then
68 echo "****"
69 echo "* Spark slaves: $slaves"
70 echo "****"
71 fi
72 fi
73fi
74
75
76zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
77zookeeper_data_dir="$ZOOKEEPER_HOME/data"
78
79if [ ! -f "$zookeeper_config_file" ] ; then
80 echo "****"
81 echo "* Generating $zookeeper_config_file"
82 cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
83
84 if [ ! -d "$zookeeper_data_dir" ] ; then
85 echo "* Creating Zookeeper dataDir:"
86 echo "* $zookeeper_data_dir"
87 mkdir "$zookeeper_data_dir"
88 fi
89 echo "****"
90fi
91
92if [ "$short_hostname" = "gsliscluster1" ] ; then
93 echo "****"
94 echo "* Solr nodes: $SOLR_NODES"
95 echo "****"
96else
97 solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
98 if [ ! -d "$solr_configsets/htrc_configs" ] ; then
99 echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
100 tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
101 fi
102fi
Note: See TracBrowser for help on using the repository browser.