Show
Ignore:
Timestamp:
11.11.2016 13:22:02 (3 years ago)
Author:
davidb
Message:

Changes made after testing with 20 solr nodes

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster
Files:
7 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-init-full-ef-collection.sh

    r31101 r31103  
    11#!/bin/bash 
    22 
    3 ssh gc0 \ 
     3solr_num_shards=`echo $SOLR_NODES | wc -w` 
     4solr_node_head=`echo $SOLR_NODES | tr ' ' '\n' | head -n 1` 
     5solr_host=${solr_node_head%:*} 
     6 
     7 
     8ssh $solr_host \ 
    49  solr create \ 
    510     -c htrc-full-ef \ 
    611     -d htrc_configs \ 
    7      -shards 10 \ 
    8      -replicationFactor 2 
     12     -shards $solr_num_shards \ 
     13     -replicationFactor 1 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

    r31098 r31103  
    33for solr_node in $SOLR_NODES ; do 
    44  solr_host=${solr_node%:*} 
    5   echo "Starting solr cloud node on: $solr_host" 
    6   ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_node 
     5  solr_port=${solr_node##*:} 
     6  echo "Starting solr cloud node on: $solr_host [port $solr_port]" 
     7  ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port 
    78done 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-stop-all.sh

    r31098 r31103  
    33for solr_node in $SOLR_NODES ; do 
    44  solr_host=${solr_node%:*} 
     5  solr_port=${solr_node##*:} 
    56  echo "Stopping Solr cloud node: $solr_host" 
    6   ssh $solr_host solr stop 
     7  ssh $solr_host solr stop -p $solr_port 
    78done 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

    r31098 r31103  
    11 
     2short_hostname=`hostname -s` 
    23 
    34#export SPARK_MASTER_HOST=gchead 
     
    910export ZOOKEEPER_SERVER=gchead:8181 
    1011export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983" 
     12#export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984" 
    1113 
    1214export HTRC_EF_PACKAGE_HOME=`pwd` 
    1315 
    14 echo "" 
    15 echo "****" 
     16if [ "$short_hostname" = "gsliscluster1" ] ; then 
     17  echo "" 
     18  echo "****" 
     19fi 
    1620 
    1721export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0" 
     
    2327#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" 
    2428 
    25 echo "* Added in JDK 1.8 into PATH" 
     29if [ "$short_hostname" = "gsliscluster1" ] ; then 
     30  echo "* Added in JDK 1.8 into PATH" 
     31fi 
    2632 
    2733source SETUP/setup-spark.bash 
     
    3238 
    3339export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH" 
    34 echo "* Added in HTRC EF PACKAGE remote scripting into PATH" 
     40if [ "$short_hostname" = "gsliscluster1" ] ; then 
     41  echo "* Added in HTRC EF PACKAGE remote scripting into PATH" 
     42fi 
    3543 
    3644spark_conf_slaves="$SPARK_HOME/conf/slaves"  
     
    4553else 
    4654  slaves=`cat "$spark_conf_slaves" | tr '\n' ' '` 
    47   echo "****" 
    48   echo "* Solr slaves: $slaves" 
    49   echo "****" 
     55  if [ "$short_hostname" = "gsliscluster1" ] ; then 
     56    echo "****" 
     57    echo "* Spark slaves: $slaves" 
     58    echo "****" 
     59  fi 
    5060fi 
    5161 
     
    7282  tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets" 
    7383fi 
     84 
     85if [ "$short_hostname" = "gsliscluster1" ] ; then 
     86  echo "****" 
     87  echo "* Solr nodes: $SOLR_NODES" 
     88  echo "****" 
     89fi 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

    r31098 r31103  
    33 
    44export PATH="$HTRC_EF_PACKAGE_HOME/solr/bin:$PATH" 
    5 echo "* Added in Solr into PATH" 
     5if [ "$short_hostname" = "gsliscluster1" ] ; then 
     6  echo "* Added in Solr into PATH" 
     7fi 
    68 
    79export SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr" 
    810export SOLR_HOME="$SOLR_TOP_LEVEL_HOME/server/solr" 
    9 echo "* Set Solr node (home) conf dir to:" 
    10 echo "*   $SOLR_HOME" 
     11if [ "$short_hostname" = "gsliscluster1" ] ; then 
     12  echo "* Set Solr node (home) conf dir to:" 
     13  echo "*   $SOLR_HOME" 
     14fi 
    1115 
    12 #alias solr-start-node.sh="solr start -c -z zookeeper:2181 -h `hostname`:8983" 
    13 #alias solr-stop-node.sh="solr stop" 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-spark.bash

    r31080 r31103  
    22export SPARK_HOME="$HTRC_EF_PACKAGE_HOME/spark" 
    33export PATH="$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH" 
    4 echo "* Added in Spark's bin and sbin into PATH" 
     4if [ "$short_hostname" = "gsliscluster1" ] ; then 
     5  echo "* Added in Spark's bin and sbin into PATH" 
     6fi 
    57 
    6 #alias spark-start-all.sh="$SPARK_HOME/sbin/start-all.sh" 
    7 #alias spark-stop-all.sh="$SPARK_HOME/sbin/stop-all.sh" 
    88 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-zookeeper.bash

    r31080 r31103  
    33export ZOOKEEPER_HOME="$HTRC_EF_PACKAGE_HOME/zookeeper" 
    44export PATH="$ZOOKEEPER_HOME/bin:$PATH" 
    5 echo "* Added in Zookeeper into PATH" 
     5if [ "$short_hostname" = "gsliscluster1" ] ; then 
     6  echo "* Added in Zookeeper into PATH" 
     7fi 
    68 
    7 #alias zookeeper-start.sh="zkServer.sh start" 
    8 #alias zookeeper-stop.sh="zkServer.sh stop" 
    99