Ignore:
Timestamp:
2016-11-11T13:22:02+13:00 (5 years ago)
Author:
davidb
Message:

Changes made after testing with 20 solr nodes

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-init-full-ef-collection.sh

    r31101 r31103  
    11#!/bin/bash
    22
    3 ssh gc0 \
     3solr_num_shards=`echo $SOLR_NODES | wc -w`
     4solr_node_head=`echo $SOLR_NODES | tr ' ' '\n' | head -n 1`
     5solr_host=${solr_node_head%:*}
     6
     7
     8ssh $solr_host \
    49  solr create \
    510     -c htrc-full-ef \
    611     -d htrc_configs \
    7      -shards 10 \
    8      -replicationFactor 2
     12     -shards $solr_num_shards \
     13     -replicationFactor 1
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

    r31098 r31103  
    33for solr_node in $SOLR_NODES ; do
    44  solr_host=${solr_node%:*}
    5   echo "Starting solr cloud node on: $solr_host"
    6   ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_node
     5  solr_port=${solr_node##*:}
     6  echo "Starting solr cloud node on: $solr_host [port $solr_port]"
     7  ssh $solr_host solr start -c -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port
    78done
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-stop-all.sh

    r31098 r31103  
    33for solr_node in $SOLR_NODES ; do
    44  solr_host=${solr_node%:*}
     5  solr_port=${solr_node##*:}
    56  echo "Stopping Solr cloud node: $solr_host"
    6   ssh $solr_host solr stop
     7  ssh $solr_host solr stop -p $solr_port
    78done
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

    r31098 r31103  
    11
     2short_hostname=`hostname -s`
    23
    34#export SPARK_MASTER_HOST=gchead
     
    910export ZOOKEEPER_SERVER=gchead:8181
    1011export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
     12#export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
    1113
    1214export HTRC_EF_PACKAGE_HOME=`pwd`
    1315
    14 echo ""
    15 echo "****"
     16if [ "$short_hostname" = "gsliscluster1" ] ; then
     17  echo ""
     18  echo "****"
     19fi
    1620
    1721export JAVA_HOME="$HTRC_EF_PACKAGE_HOME/jdk1.8.0"
     
    2327#export _JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
    2428
    25 echo "* Added in JDK 1.8 into PATH"
     29if [ "$short_hostname" = "gsliscluster1" ] ; then
     30  echo "* Added in JDK 1.8 into PATH"
     31fi
    2632
    2733source SETUP/setup-spark.bash
     
    3238
    3339export PATH="$HTRC_EF_PACKAGE_HOME/SCRIPTS:$PATH"
    34 echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
     40if [ "$short_hostname" = "gsliscluster1" ] ; then
     41  echo "* Added in HTRC EF PACKAGE remote scripting into PATH"
     42fi
    3543
    3644spark_conf_slaves="$SPARK_HOME/conf/slaves"
     
    4553else
    4654  slaves=`cat "$spark_conf_slaves" | tr '\n' ' '`
    47   echo "****"
    48   echo "* Solr slaves: $slaves"
    49   echo "****"
     55  if [ "$short_hostname" = "gsliscluster1" ] ; then
     56    echo "****"
     57    echo "* Spark slaves: $slaves"
     58    echo "****"
     59  fi
    5060fi
    5161
     
    7282  tar xvzf CONF/htrc_configs.tar.gz -C "$solr_configsets"
    7383fi
     84
     85if [ "$short_hostname" = "gsliscluster1" ] ; then
     86  echo "****"
     87  echo "* Solr nodes: $SOLR_NODES"
     88  echo "****"
     89fi
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

    r31098 r31103  
    33
    44export PATH="$HTRC_EF_PACKAGE_HOME/solr/bin:$PATH"
    5 echo "* Added in Solr into PATH"
     5if [ "$short_hostname" = "gsliscluster1" ] ; then
     6  echo "* Added in Solr into PATH"
     7fi
    68
    79export SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr"
    810export SOLR_HOME="$SOLR_TOP_LEVEL_HOME/server/solr"
    9 echo "* Set Solr node (home) conf dir to:"
    10 echo "*   $SOLR_HOME"
     11if [ "$short_hostname" = "gsliscluster1" ] ; then
     12  echo "* Set Solr node (home) conf dir to:"
     13  echo "*   $SOLR_HOME"
     14fi
    1115
    12 #alias solr-start-node.sh="solr start -c -z zookeeper:2181 -h `hostname`:8983"
    13 #alias solr-stop-node.sh="solr stop"
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-spark.bash

    r31080 r31103  
    22export SPARK_HOME="$HTRC_EF_PACKAGE_HOME/spark"
    33export PATH="$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH"
    4 echo "* Added in Spark's bin and sbin into PATH"
     4if [ "$short_hostname" = "gsliscluster1" ] ; then
     5  echo "* Added in Spark's bin and sbin into PATH"
     6fi
    57
    6 #alias spark-start-all.sh="$SPARK_HOME/sbin/start-all.sh"
    7 #alias spark-stop-all.sh="$SPARK_HOME/sbin/stop-all.sh"
    88
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-zookeeper.bash

    r31080 r31103  
    33export ZOOKEEPER_HOME="$HTRC_EF_PACKAGE_HOME/zookeeper"
    44export PATH="$ZOOKEEPER_HOME/bin:$PATH"
    5 echo "* Added in Zookeeper into PATH"
     5if [ "$short_hostname" = "gsliscluster1" ] ; then
     6  echo "* Added in Zookeeper into PATH"
     7fi
    68
    7 #alias zookeeper-start.sh="zkServer.sh start"
    8 #alias zookeeper-stop.sh="zkServer.sh stop"
    99
Note: See TracChangeset for help on using the changeset viewer.