Changeset 31373

Show
Ignore:
Timestamp:
31.01.2017 00:08:35 (3 years ago)
Author:
davidb
Message:

Changes made to operate on solr1 and solr2 boxes

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster
Files:
4 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-setup-local-disk-all.sh

    r31304 r31373  
    11#!/bin/bash 
    22 
    3 unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq` 
     3if [ "x$SOLR_SHARDS" != "x" ] ; then 
     4    SOLR_NODES_ARRAY=($SOLR_NODES) 
     5    SOLR_SHARDS_ARRAY=($SOLR_SHARDS) 
    46 
    5 #local_dir="/tmp/solr-ef" 
    6 local_dir1="/hdfsd05/dbbridge/solr-ef" 
    7 local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr" 
     7    num_shards=${#SOLR_NODES_ARRAY[*]} 
     8     
     9    i=0 
     10     
     11    while [ $i -lt $num_shards ] ; do 
     12    solr_node=${SOLR_NODES_ARRAY[$i]} 
     13    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]} 
     14     
     15    solr_host=${solr_node%:*} 
     16    solr_port=${solr_node##*:} 
    817 
    9 for solr_host in $unique_hosts ; do 
    10   remote_dir1="$solr_host:$local_dir1" 
    11   remote_dir2="$solr_host:$local_dir2" 
     18    remote_dir="$solr_host:$solr_home_shard_dir" 
    1219 
    13   echo "Running rsync from networked SOLR_HOME to $remote_dir1" 
    14   ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi" 
    15   rsync -pav "$SOLR_HOME/." "$remote_dir1/." 
     20    echo "Running rsync from networked SOLR_HOME to $remote_dir" 
     21    ssh $solr_host "if [ ! -d $solr_home_shard_dir ] ; then mkdir $solr_home_shard_dir ; fi" 
     22    rsync -pav "$SOLR_HOME/." "$remote_dir/." 
    1623 
    17   echo "Running rsync from networked SOLR_HOME to $remote_dir2" 
    18   ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi" 
    19   rsync -pav "$SOLR_HOME/." "$remote_dir2/." 
    20 done 
     24    server_dir="/disk0/solr-server-$solr_host-$solr_port" 
     25    remote_server_dir="$solr_host:$server_dir" 
     26     
     27    echo "Running rsync from networked SOLR_HOME/server to $remote_server_dir" 
     28    ssh $solr_host "if [ ! -d $server_dir ] ; then mkdir $server_dir ; fi" 
     29    rsync -pav "$SOLR_TOP_LEVEL_HOME/server/." "$remote_server_dir/." 
     30     
     31    i=$((i+1)) 
     32    done 
     33else  
     34 
     35  unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq` 
     36 
     37  #local_dir="/tmp/solr-ef" 
     38  local_dir1="/hdfsd05/dbbridge/solr-ef" 
     39  local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr" 
     40 
     41  for solr_host in $unique_hosts ; do 
     42    remote_dir1="$solr_host:$local_dir1" 
     43    remote_dir2="$solr_host:$local_dir2" 
     44 
     45    echo "Running rsync from networked SOLR_HOME to $remote_dir1" 
     46    ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi" 
     47    rsync -pav "$SOLR_HOME/." "$remote_dir1/." 
     48 
     49    echo "Running rsync from networked SOLR_HOME to $remote_dir2" 
     50    ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi" 
     51    rsync -pav "$SOLR_HOME/." "$remote_dir2/." 
     52  done 
     53fi 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

    r31231 r31373  
    11#!/bin/bash 
    22 
    3 run_from_local_disk=1 
    43 
    5 #local_dir="/tmp/solr-ef" 
    6 local_dir="/hdfsd05/dbbridge/solr-ef" 
     4if [ "x$SOLR_SHARDS" != "x" ] ; then 
     5    SOLR_NODES_ARRAY=($SOLR_NODES) 
     6    SOLR_SHARDS_ARRAY=($SOLR_SHARDS) 
     7 
     8    num_shards=${#SOLR_NODES_ARRAY[*]} 
     9     
     10    i=0 
     11     
     12    while [ $i -lt $num_shards ] ; do 
     13    solr_node=${SOLR_NODES_ARRAY[$i]} 
     14    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]} 
     15     
     16    solr_host=${solr_node%:*} 
     17    solr_port=${solr_node##*:} 
     18 
     19    server_dir="/disk0/solr-server-$solr_host-$solr_port" 
     20     
     21    export SOLR_HOME= 
     22    export SOLR_PID_DIR= 
     23    #export SOLR_PID_DIR="$solr_home_shard_dir" 
     24    echo "Starting solr cloud node on: $solr_host:$solr_port solr_home=$solr_home_shard_dir" 
     25    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port -d "$server_dir" -s "$solr_home_shard_dir" 
     26 
     27    i=$((i+1)) 
     28    done 
     29else 
     30 
     31    run_from_local_disk=1 
     32 
     33    #local_dir="/tmp/solr-ef" 
     34    local_dir="/hdfsd05/dbbridge/solr-ef" 
    735 
    836 
    9 if [ $run_from_local_disk = "1" ] ; then 
    10   opt_s="-s $local_dir" 
    11   export SOLR_HOME= 
    12   echo "****" 
    13   echo "* Running solr from local disk: $local_dir" 
    14   echo "****" 
    15 else  
    16   opt_s= 
     37    if [ $run_from_local_disk = "1" ] ; then 
     38    opt_s="-s $local_dir" 
     39    export SOLR_HOME= 
     40    echo "****" 
     41    echo "* Running solr from local disk: $local_dir" 
     42    echo "****" 
     43    else  
     44    opt_s= 
     45    fi 
     46     
     47    for solr_node in $SOLR_NODES ; do 
     48    solr_host=${solr_node%:*} 
     49    solr_port=${solr_node##*:} 
     50     
     51    echo "Starting solr cloud node on: $solr_host [port $solr_port]" 
     52    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s 
     53    done 
    1754fi 
    18  
    19 for solr_node in $SOLR_NODES ; do 
    20   solr_host=${solr_node%:*} 
    21   solr_port=${solr_node##*:} 
    22  
    23   echo "Starting solr cloud node on: $solr_host [port $solr_port]" 
    24   ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s 
    25 done 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

    r31367 r31373  
    1515  export ZOOKEEPER_SERVER=solr1:8181 
    1616  export SOLR_NODES="solr1:8983 solr1:8984 solr1:8985 solr1:8986 solr1:8987" 
    17   export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr:8987" 
    18  
     17  export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr2:8987" 
     18  export SOLR_SHARDS="/disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef" 
     19  export SOLR_SHARDS="$SOLR_SHARDS /disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef" 
    1920else 
    2021  export ZOOKEEPER_SERVER=gchead:8181 
    21   export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983" 
    22   #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984" 
     22#  export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983" 
     23#  #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984" 
     24 
     25  export ZOOKEEPER_SERVER=solr1-s:8181 
     26  export SOLR_NODES="solr1-s:8983 solr1-s:8984 solr1-s:8985 solr1-s:8986 solr1-s:8987" 
     27  export SOLR_NODES="$SOLR_NODES solr2-s:8983 solr2-s:8984 solr2-s:8985 solr2-s:8986 solr2-s:8987" 
     28 
    2329fi 
    2430 
     
    128134  echo "****" 
    129135else 
     136   
    130137  solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets" 
    131138  if [ ! -d "$solr_configsets/htrc_configs" ] ; then 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

    r31301 r31373  
    33_SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr" 
    44 
    5 if [ "$short_hostname" = "nema" ] ; then 
     5if [ "$short_hostname" = "nema" ] || [ "${short_hostname%[1-2]}" = "solr" ] ; then 
    66  _SOLR_HOME="$_SOLR_TOP_LEVEL_HOME/server/solr" 
    77else  
     
    2323  export PATH="$SOLR_TOP_LEVEL_HOME/bin:$PATH" 
    2424 
     25  # on solr1 and solr2 these will be locally overriden when solr-server started/stopped 
    2526  export SOLR_HOME="$_SOLR_HOME" 
    26  
    2727  export SOLR_PID_DIR="$SOLR_HOME" 
    2828