Changeset 31373

Timestamp:

2017-01-31T00:08:35+13:00 (7 years ago)

Author:

davidb

Message:

Changes made to operate on solr1 and solr2 boxes

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster

Files:

• SCRIPTS/remote-solr-setup-local-disk-all.sh (modified) (1 diff)
• SCRIPTS/remote-solr-start-all.sh (modified) (1 diff)
• SETUP.bash (modified) (2 diffs)
• SETUP/setup-solr.bash (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-setup-local-disk-all.sh

@@ -1 +1 @@
 #!/bin/bash
 
-unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq`
+if [ "x$SOLR_SHARDS" != "x" ] ; then
+    SOLR_NODES_ARRAY=($SOLR_NODES)
+    SOLR_SHARDS_ARRAY=($SOLR_SHARDS)
 
-#local_dir="/tmp/solr-ef"
-local_dir1="/hdfsd05/dbbridge/solr-ef"
-local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr"
+    num_shards=${#SOLR_NODES_ARRAY[*]}
+    
+    i=0
+    
+    while [ $i -lt $num_shards ] ; do
+    solr_node=${SOLR_NODES_ARRAY[$i]}
+    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]}
+    
+    solr_host=${solr_node%:*}
+    solr_port=${solr_node##*:}
 
-for solr_host in $unique_hosts ; do
-  remote_dir1="$solr_host:$local_dir1"
-  remote_dir2="$solr_host:$local_dir2"
+    remote_dir="$solr_host:$solr_home_shard_dir"
 
-  echo "Running rsync from networked SOLR_HOME to $remote_dir1"
-  ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi"
-  rsync -pav "$SOLR_HOME/." "$remote_dir1/."
+    echo "Running rsync from networked SOLR_HOME to $remote_dir"
+    ssh $solr_host "if [ ! -d $solr_home_shard_dir ] ; then mkdir $solr_home_shard_dir ; fi"
+    rsync -pav "$SOLR_HOME/." "$remote_dir/."
 
-  echo "Running rsync from networked SOLR_HOME to $remote_dir2"
-  ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi"
-  rsync -pav "$SOLR_HOME/." "$remote_dir2/."
-done
+    server_dir="/disk0/solr-server-$solr_host-$solr_port"
+    remote_server_dir="$solr_host:$server_dir"
+    
+    echo "Running rsync from networked SOLR_HOME/server to $remote_server_dir"
+    ssh $solr_host "if [ ! -d $server_dir ] ; then mkdir $server_dir ; fi"
+    rsync -pav "$SOLR_TOP_LEVEL_HOME/server/." "$remote_server_dir/."
+    
+    i=$((i+1))
+    done
+else 
+
+  unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq`
+
+  #local_dir="/tmp/solr-ef"
+  local_dir1="/hdfsd05/dbbridge/solr-ef"
+  local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr"
+
+  for solr_host in $unique_hosts ; do
+    remote_dir1="$solr_host:$local_dir1"
+    remote_dir2="$solr_host:$local_dir2"
+
+    echo "Running rsync from networked SOLR_HOME to $remote_dir1"
+    ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi"
+    rsync -pav "$SOLR_HOME/." "$remote_dir1/."
+
+    echo "Running rsync from networked SOLR_HOME to $remote_dir2"
+    ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi"
+    rsync -pav "$SOLR_HOME/." "$remote_dir2/."
+  done
+fi

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

@@ -1 +1 @@
 #!/bin/bash
 
-run_from_local_disk=1
 
-#local_dir="/tmp/solr-ef"
-local_dir="/hdfsd05/dbbridge/solr-ef"
+if [ "x$SOLR_SHARDS" != "x" ] ; then
+    SOLR_NODES_ARRAY=($SOLR_NODES)
+    SOLR_SHARDS_ARRAY=($SOLR_SHARDS)
+
+    num_shards=${#SOLR_NODES_ARRAY[*]}
+    
+    i=0
+    
+    while [ $i -lt $num_shards ] ; do
+    solr_node=${SOLR_NODES_ARRAY[$i]}
+    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]}
+    
+    solr_host=${solr_node%:*}
+    solr_port=${solr_node##*:}
+
+    server_dir="/disk0/solr-server-$solr_host-$solr_port"
+    
+    export SOLR_HOME=
+    export SOLR_PID_DIR=
+    #export SOLR_PID_DIR="$solr_home_shard_dir"
+    echo "Starting solr cloud node on: $solr_host:$solr_port solr_home=$solr_home_shard_dir"
+    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port -d "$server_dir" -s "$solr_home_shard_dir"
+
+    i=$((i+1))
+    done
+else
+
+    run_from_local_disk=1
+
+    #local_dir="/tmp/solr-ef"
+    local_dir="/hdfsd05/dbbridge/solr-ef"
 
 
-if [ $run_from_local_disk = "1" ] ; then
-  opt_s="-s $local_dir"
-  export SOLR_HOME=
-  echo "****"
-  echo "* Running solr from local disk: $local_dir"
-  echo "****"
-else 
-  opt_s=
+    if [ $run_from_local_disk = "1" ] ; then
+    opt_s="-s $local_dir"
+    export SOLR_HOME=
+    echo "****"
+    echo "* Running solr from local disk: $local_dir"
+    echo "****"
+    else 
+    opt_s=
+    fi
+    
+    for solr_node in $SOLR_NODES ; do
+    solr_host=${solr_node%:*}
+    solr_port=${solr_node##*:}
+    
+    echo "Starting solr cloud node on: $solr_host [port $solr_port]"
+    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s
+    done
 fi
-
-for solr_node in $SOLR_NODES ; do
-  solr_host=${solr_node%:*}
-  solr_port=${solr_node##*:}
-
-  echo "Starting solr cloud node on: $solr_host [port $solr_port]"
-  ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s
-done

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP.bash

@@ -15 +15 @@
   export ZOOKEEPER_SERVER=solr1:8181
   export SOLR_NODES="solr1:8983 solr1:8984 solr1:8985 solr1:8986 solr1:8987"
-  export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr:8987"
-
+  export SOLR_NODES="$SOLR_NODES solr2:8983 solr2:8984 solr2:8985 solr2:8986 solr2:8987"
+  export SOLR_SHARDS="/disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
+  export SOLR_SHARDS="$SOLR_SHARDS /disk0/solr-full-ef /disk1/solr-full-ef /disk2/solr-full-ef /disk3/solr-full-ef /disk4/solr-full-ef"
 else
   export ZOOKEEPER_SERVER=gchead:8181
-  export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
-  #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
+#  export SOLR_NODES="gc0:8983 gc1:8983 gc2:8983 gc3:8983 gc4:8983 gc5:8983 gc6:8983 gc7:8983 gc8:8983 gc9:8983"
+#  #export SOLR_NODES="$SOLR_NODES gc0:8984 gc1:8984 gc2:8984 gc3:8984 gc4:8984 gc5:8984 gc6:8984 gc7:8984 gc8:8984 gc9:8984"
+
+  export ZOOKEEPER_SERVER=solr1-s:8181
+  export SOLR_NODES="solr1-s:8983 solr1-s:8984 solr1-s:8985 solr1-s:8986 solr1-s:8987"
+  export SOLR_NODES="$SOLR_NODES solr2-s:8983 solr2-s:8984 solr2-s:8985 solr2-s:8986 solr2-s:8987"
+
 fi
 
@@ -128 +134 @@
   echo "****"
 else
+  
   solr_configsets="$SOLR_TOP_LEVEL_HOME/server/solr/configsets"
   if [ ! -d "$solr_configsets/htrc_configs" ] ; then

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SETUP/setup-solr.bash

@@ -3 +3 @@
 _SOLR_TOP_LEVEL_HOME="$HTRC_EF_PACKAGE_HOME/solr"
 
-if [ "$short_hostname" = "nema" ] ; then
+if [ "$short_hostname" = "nema" ] || [ "${short_hostname%[1-2]}" = "solr" ] ; then
   _SOLR_HOME="$_SOLR_TOP_LEVEL_HOME/server/solr"
 else 
@@ -23 +23 @@
   export PATH="$SOLR_TOP_LEVEL_HOME/bin:$PATH"
 
+  # on solr1 and solr2 these will be locally overriden when solr-server started/stopped
   export SOLR_HOME="$_SOLR_HOME"
-
   export SOLR_PID_DIR="$SOLR_HOME"