Changeset 31373


Ignore:
Timestamp:
2017-01-31T00:08:35+13:00 (4 years ago)
Author:
davidb
Message:

Changes made to operate on solr1 and solr2 boxes

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-setup-local-disk-all.sh

    r31304 r31373  
    11#!/bin/bash
    22
    3 unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq`
     3if [ "x$SOLR_SHARDS" != "x" ] ; then
     4    SOLR_NODES_ARRAY=($SOLR_NODES)
     5    SOLR_SHARDS_ARRAY=($SOLR_SHARDS)
    46
    5 #local_dir="/tmp/solr-ef"
    6 local_dir1="/hdfsd05/dbbridge/solr-ef"
    7 local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr"
     7    num_shards=${#SOLR_NODES_ARRAY[*]}
     8   
     9    i=0
     10   
     11    while [ $i -lt $num_shards ] ; do
     12    solr_node=${SOLR_NODES_ARRAY[$i]}
     13    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]}
     14   
     15    solr_host=${solr_node%:*}
     16    solr_port=${solr_node##*:}
    817
    9 for solr_host in $unique_hosts ; do
    10   remote_dir1="$solr_host:$local_dir1"
    11   remote_dir2="$solr_host:$local_dir2"
     18    remote_dir="$solr_host:$solr_home_shard_dir"
    1219
    13   echo "Running rsync from networked SOLR_HOME to $remote_dir1"
    14   ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi"
    15   rsync -pav "$SOLR_HOME/." "$remote_dir1/."
     20    echo "Running rsync from networked SOLR_HOME to $remote_dir"
     21    ssh $solr_host "if [ ! -d $solr_home_shard_dir ] ; then mkdir $solr_home_shard_dir ; fi"
     22    rsync -pav "$SOLR_HOME/." "$remote_dir/."
    1623
    17   echo "Running rsync from networked SOLR_HOME to $remote_dir2"
    18   ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi"
    19   rsync -pav "$SOLR_HOME/." "$remote_dir2/."
    20 done
     24    server_dir="/disk0/solr-server-$solr_host-$solr_port"
     25    remote_server_dir="$solr_host:$server_dir"
     26   
     27    echo "Running rsync from networked SOLR_HOME/server to $remote_server_dir"
     28    ssh $solr_host "if [ ! -d $server_dir ] ; then mkdir $server_dir ; fi"
     29    rsync -pav "$SOLR_TOP_LEVEL_HOME/server/." "$remote_server_dir/."
     30   
     31    i=$((i+1))
     32    done
     33else
     34
     35  unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq`
     36
     37  #local_dir="/tmp/solr-ef"
     38  local_dir1="/hdfsd05/dbbridge/solr-ef"
     39  local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr"
     40
     41  for solr_host in $unique_hosts ; do
     42    remote_dir1="$solr_host:$local_dir1"
     43    remote_dir2="$solr_host:$local_dir2"
     44
     45    echo "Running rsync from networked SOLR_HOME to $remote_dir1"
     46    ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi"
     47    rsync -pav "$SOLR_HOME/." "$remote_dir1/."
     48
     49    echo "Running rsync from networked SOLR_HOME to $remote_dir2"
     50    ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi"
     51    rsync -pav "$SOLR_HOME/." "$remote_dir2/."
     52  done
     53fi
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

    r31231 r31373  
    11#!/bin/bash
    22
    3 run_from_local_disk=1
    43
    5 #local_dir="/tmp/solr-ef"
    6 local_dir="/hdfsd05/dbbridge/solr-ef"
     4if [ "x$SOLR_SHARDS" != "x" ] ; then
     5    SOLR_NODES_ARRAY=($SOLR_NODES)
     6    SOLR_SHARDS_ARRAY=($SOLR_SHARDS)
     7
     8    num_shards=${#SOLR_NODES_ARRAY[*]}
     9   
     10    i=0
     11   
     12    while [ $i -lt $num_shards ] ; do
     13    solr_node=${SOLR_NODES_ARRAY[$i]}
     14    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]}
     15   
     16    solr_host=${solr_node%:*}
     17    solr_port=${solr_node##*:}
     18
     19    server_dir="/disk0/solr-server-$solr_host-$solr_port"
     20   
     21    export SOLR_HOME=
     22    export SOLR_PID_DIR=
     23    #export SOLR_PID_DIR="$solr_home_shard_dir"
     24    echo "Starting solr cloud node on: $solr_host:$solr_port solr_home=$solr_home_shard_dir"
     25    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port -d "$server_dir" -s "$solr_home_shard_dir"
     26
     27    i=$((i+1))
     28    done
     29else
     30
     31    run_from_local_disk=1
     32
     33    #local_dir="/tmp/solr-ef"
     34    local_dir="/hdfsd05/dbbridge/solr-ef"
    735
    836
    9 if [ $run_from_local_disk = "1" ] ; then
    10   opt_s="-s $local_dir"
    11   export SOLR_HOME=
    12   echo "****"
    13   echo "* Running solr from local disk: $local_dir"
    14   echo "****"
    15 else
    16   opt_s=
     37    if [ $run_from_local_disk = "1" ] ; then
     38    opt_s="-s $local_dir"
     39    export SOLR_HOME=
     40    echo "****"
     41    echo "* Running solr from local disk: $local_dir"
     42    echo "****"
     43    else
     44    opt_s=
     45    fi
     46   
     47    for solr_node in $SOLR_NODES ; do
     48    solr_host=${solr_node%:*}
     49    solr_port=${solr_node##*:}
     50   
     51    echo "Starting solr cloud node on: $solr_host [port $solr_port]"
     52    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s
     53    done
    1754fi
    18 
    19 for solr_node in $SOLR_NODES ; do
    20   solr_host=${solr_node%:*}
    21   solr_port=${solr_node##*:}
    22 
    23   echo "Starting solr cloud node on: $solr_host [port $solr_port]"
    24   ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s
    25 done
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

    r31367 r31373  
    1515  export ZOOKEEPER_SERVER=solr1:8181
    1616  export SOLR_NODES="solr1:8983 solr1:8984 solr1:8985 solr1:8986 solr1:8987"
    17   export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr:8987"
    18 
     17  export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr2:8987"
     18  export SOLR_SHARDS="/disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
     19  export SOLR_SHARDS="$SOLR_SHARDS /disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
    1920else
    2021  export ZOOKEEPER_SERVER=gchead:8181
    21   export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
    22   #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
     22#  export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
     23#  #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
     24
     25  export ZOOKEEPER_SERVER=solr1-s:8181
     26  export SOLR_NODES="solr1-s:8983 solr1-s:8984 solr1-s:8985 solr1-s:8986 solr1-s:8987"
     27  export SOLR_NODES="$SOLR_NODES solr2-s:8983 solr2-s:8984 solr2-s:8985 solr2-s:8986 solr2-s:8987"
     28
    2329fi
    2430
     
    128134  echo "****"
    129135else
     136 
    130137  solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
    131138  if [ ! -d "$solr_configsets/htrc_configs" ] ; then
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

    r31301 r31373  
    33_SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr"
    44
    5 if [ "$short_hostname" = "nema" ] ; then
     5if [ "$short_hostname" = "nema" ] || [ "${short_hostname%[1-2]}" = "solr" ] ; then
    66  _SOLR_HOME="$_SOLR_TOP_LEVEL_HOME/server/solr"
    77else
     
    2323  export PATH="$SOLR_TOP_LEVEL_HOME/bin:$PATH"
    2424
     25  # on solr1 and solr2 these will be locally overriden when solr-server started/stopped
    2526  export SOLR_HOME="$_SOLR_HOME"
    26 
    2727  export SOLR_PID_DIR="$SOLR_HOME"
    2828
Note: See TracChangeset for help on using the changeset viewer.