Changeset 31098 for other-projects/hathitrust/wcsa/extracted-features-solr

Timestamp:

2016-11-10T22:42:31+13:00 (7 years ago)

Author:

davidb

Message:

Changes resulting from test runs to get Zookeeper and Solr running on gsliscluster1

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster

Files:

• SCRIPTS/remote-solr-start-all.sh (modified) (1 diff)
• SCRIPTS/remote-solr-stop-all.sh (modified) (1 diff)
• SETUP.bash (modified) (3 diffs)
• SETUP/setup-solr.bash (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

@@ -2 +2 @@
 
 for solr_node in $SOLR_NODES ; do
-  echo $solr_node
-#  ssh $solr_node solr start -c -z $ZOOKEEPER_SERVER -h $solr_node
+  solr_host=${solr_node%:*}
+  echo "Starting solr cloud node on: $solr_host"
+  ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_node
 done

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-stop-all.sh

@@ -2 +2 @@
 
 for solr_node in $SOLR_NODES ; do
-  echo $solr_node
-#  ssh $solr_node solr stop
+  solr_host=${solr_node%:*}
+  echo "Stopping Solr cloud node: $solr_host"
+  ssh $solr_host solr stop
 done

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

@@ -1 +1 @@
 
 
-export SPARK_MASTER_HOST=gchead
+#export SPARK_MASTER_HOST=gchead
+export SPARK_MASTER_HOST=192.168.64.1
+
 export SPARK_MASTER_URL=spark://$SPARK_MASTER_HOST:7077
 export SPARK_SLAVE_HOSTS="gc0 gc1 gc2 gc3 gc4 gc5 gc6 gc7 gc8 gc9"
 
-export ZOOKEEPER_SERVER=gchead:2181
+export ZOOKEEPER_SERVER=gchead:8181
 export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
 
@@ -15 +17 @@
 export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0"
 export PATH="$JAVA_HOME/bin:$PATH"
+#export _JAVA_OPTIONS="-Xmx512m"
+#export _JAVA_OPTIONS="-Xmx1024m"
+#export _JAVA_OPTIONS="-Xmx2048m"
+export _JAVA_OPTIONS=
+#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
+
 echo "* Added in JDK 1.8 into PATH"
 
@@ -42 +50 @@
 fi
 
-echo "****"
-echo ""
+
+zookeeper_config_file="$ZOOKEEPER_HOME/conf/zoo.cfg"
+zookeeper_data_dir="$ZOOKEEPER_HOME/data"
+
+if [ ! -f "$zookeeper_config_file" ] ; then
+  echo "****"
+  echo "* Generating $zookeeper_config_file" 
+  cat CONF/zoo.cfg.in | sed "s%@zookeeper-data-dir@%$zookeeper_data_dir%g" > "$zookeeper_config_file"
+
+  if [ ! -d "$zookeeper_data_dir" ] ; then
+    echo "* Creating Zookeeper dataDir:"
+    echo "*   $zookeeper_data_dir"
+    mkdir "$zookeeper_data_dir"
+  fi
+  echo "****"
+fi
+
+solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
+if [ ! -d "$solr_configsets/htrc_configs" ] ; then
+  echo "Untarring htrc_configs.tar.gz in Solr configtests directory"
+  tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
+fi

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

@@ -5 +5 @@
 echo "* Added in Solr into PATH"
 
-export SOLR_HOME="$HTRC_EF_PACKAGE_HOME/solr/server/solr"
+export SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr"
+export SOLR_HOME="$SOLR_TOP_LEVEL_HOME/server/solr"
 echo "* Set Solr node (home) conf dir to:"
 echo "*   $SOLR_HOME"