Changeset 31103 for other-projects/hathitrust

Timestamp:

2016-11-11T13:22:02+13:00 (7 years ago)

Author:

davidb

Message:

Changes made after testing with 20 solr nodes

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster

Files:

• SCRIPTS/remote-solr-init-full-ef-collection.sh (modified) (1 diff)
• SCRIPTS/remote-solr-start-all.sh (modified) (1 diff)
• SCRIPTS/remote-solr-stop-all.sh (modified) (1 diff)
• SETUP.bash (modified) (6 diffs)
• SETUP/setup-solr.bash (modified) (1 diff)
• SETUP/setup-spark.bash (modified) (1 diff)
• SETUP/setup-zookeeper.bash (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-init-full-ef-collection.sh

@@ -1 +1 @@
 #!/bin/bash
 
-ssh gc0 \
+solr_num_shards=`echo $SOLR_NODES | wc -w`
+solr_node_head=`echo $SOLR_NODES | tr ' ' '\n' | head -n 1`
+solr_host=${solr_node_head%:*}
+
+
+ssh $solr_host \
   solr create \
      -c htrc-full-ef \
      -d htrc_configs \
-     -shards 10 \
-     -replicationFactor 2
+     -shards $solr_num_shards \
+     -replicationFactor 1

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

@@ -3 +3 @@
 for solr_node in $SOLR_NODES ; do
   solr_host=${solr_node%:*}
-  echo "Starting solr cloud node on: $solr_host"
-  ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_node
+  solr_port=${solr_node##*:}
+  echo "Starting solr cloud node on: $solr_host [port $solr_port]"
+  ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port
 done

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-stop-all.sh

@@ -3 +3 @@
 for solr_node in $SOLR_NODES ; do
   solr_host=${solr_node%:*}
+  solr_port=${solr_node##*:}
   echo "Stopping Solr cloud node: $solr_host"
-  ssh $solr_host solr stop
+  ssh $solr_host solr stop -p $solr_port
 done

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

@@ -1 +1 @@
 
+short_hostname=`hostname -s`
 
 #export SPARK_MASTER_HOST=gchead
@@ -9 +10 @@
 export ZOOKEEPER_SERVER=gchead:8181
 export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
+#export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
 
 export HTRC_EF_PACKAGE_HOME=`pwd`
 
-echo ""
-echo "****"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo ""
+  echo "****"
+fi
 
 export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0"
@@ -23 +27 @@
 #export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
 
-echo "* Added in JDK 1.8 into PATH"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "* Added in JDK 1.8 into PATH"
+fi
 
 source SETUP/setup-spark.bash
@@ -32 +38 @@
 
 export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH"
-echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
+fi
 
 spark_conf_slaves="$SPARK_HOME/conf/slaves" 
@@ -45 +53 @@
 else
   slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
-  echo "****"
-  echo "* Solr slaves: $slaves"
-  echo "****"
+  if [ "$short_hostname" = "gsliscluster1" ] ; then
+    echo "****"
+    echo "* Spark slaves: $slaves"
+    echo "****"
+  fi
 fi
 
@@ -72 +82 @@
   tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
 fi
+
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "****"
+  echo "* Solr nodes: $SOLR_NODES"
+  echo "****"
+fi

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

@@ -3 +3 @@
 
 export PATH="$HTRC_EF_PACKAGE_HOME/solr/bin:$PATH"
-echo "* Added in Solr into PATH"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "* Added in Solr into PATH"
+fi
 
 export SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr"
 export SOLR_HOME="$SOLR_TOP_LEVEL_HOME/server/solr"
-echo "* Set Solr node (home) conf dir to:"
-echo "*   $SOLR_HOME"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "* Set Solr node (home) conf dir to:"
+  echo "*   $SOLR_HOME"
+fi
 
-#alias solr-start-node.sh="solr start -c -z zookeeper:2181 -h `hostname`:8983"
-#alias solr-stop-node.sh="solr stop"

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-spark.bash

@@ -2 +2 @@
 export SPARK_HOME="$HTRC_EF_PACKAGE_HOME/spark"
 export PATH="$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH"
-echo "* Added in Spark's bin and sbin into PATH"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "* Added in Spark's bin and sbin into PATH"
+fi
 
-#alias spark-start-all.sh="$SPARK_HOME/sbin/start-all.sh"
-#alias spark-stop-all.sh="$SPARK_HOME/sbin/stop-all.sh"
 

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-zookeeper.bash

@@ -3 +3 @@
 export ZOOKEEPER_HOME="$HTRC_EF_PACKAGE_HOME/zookeeper"
 export PATH="$ZOOKEEPER_HOME/bin:$PATH"
-echo "* Added in Zookeeper into PATH"
+if [ "$short_hostname" = "gsliscluster1" ] ; then
+  echo "* Added in Zookeeper into PATH"
+fi
 
-#alias zookeeper-start.sh="zkServer.sh start"
-#alias zookeeper-stop.sh="zkServer.sh stop"