Changeset 31373 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS

Timestamp:

2017-01-31T00:08:35+13:00 (7 years ago)

Author:

davidb

Message:

Changes made to operate on solr1 and solr2 boxes

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS

Files:

• remote-solr-setup-local-disk-all.sh (modified) (1 diff)
• remote-solr-start-all.sh (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-setup-local-disk-all.sh

@@ -1 +1 @@
 #!/bin/bash
 
-unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq`
+if [ "x$SOLR_SHARDS" != "x" ] ; then
+    SOLR_NODES_ARRAY=($SOLR_NODES)
+    SOLR_SHARDS_ARRAY=($SOLR_SHARDS)
 
-#local_dir="/tmp/solr-ef"
-local_dir1="/hdfsd05/dbbridge/solr-ef"
-local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr"
+    num_shards=${#SOLR_NODES_ARRAY[*]}
+    
+    i=0
+    
+    while [ $i -lt $num_shards ] ; do
+    solr_node=${SOLR_NODES_ARRAY[$i]}
+    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]}
+    
+    solr_host=${solr_node%:*}
+    solr_port=${solr_node##*:}
 
-for solr_host in $unique_hosts ; do
-  remote_dir1="$solr_host:$local_dir1"
-  remote_dir2="$solr_host:$local_dir2"
+    remote_dir="$solr_host:$solr_home_shard_dir"
 
-  echo "Running rsync from networked SOLR_HOME to $remote_dir1"
-  ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi"
-  rsync -pav "$SOLR_HOME/." "$remote_dir1/."
+    echo "Running rsync from networked SOLR_HOME to $remote_dir"
+    ssh $solr_host "if [ ! -d $solr_home_shard_dir ] ; then mkdir $solr_home_shard_dir ; fi"
+    rsync -pav "$SOLR_HOME/." "$remote_dir/."
 
-  echo "Running rsync from networked SOLR_HOME to $remote_dir2"
-  ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi"
-  rsync -pav "$SOLR_HOME/." "$remote_dir2/."
-done
+    server_dir="/disk0/solr-server-$solr_host-$solr_port"
+    remote_server_dir="$solr_host:$server_dir"
+    
+    echo "Running rsync from networked SOLR_HOME/server to $remote_server_dir"
+    ssh $solr_host "if [ ! -d $server_dir ] ; then mkdir $server_dir ; fi"
+    rsync -pav "$SOLR_TOP_LEVEL_HOME/server/." "$remote_server_dir/."
+    
+    i=$((i+1))
+    done
+else 
+
+  unique_hosts=`echo $SOLR_NODES | tr ' ' '\n' | sed 's/:.*$//' | sort | uniq`
+
+  #local_dir="/tmp/solr-ef"
+  local_dir1="/hdfsd05/dbbridge/solr-ef"
+  local_dir2="/hdfsd05/dbbridge/gslis-cluster/solr/server/solr"
+
+  for solr_host in $unique_hosts ; do
+    remote_dir1="$solr_host:$local_dir1"
+    remote_dir2="$solr_host:$local_dir2"
+
+    echo "Running rsync from networked SOLR_HOME to $remote_dir1"
+    ssh $solr_host "if [ ! -d $local_dir1 ] ; then mkdir $local_dir1 ; fi"
+    rsync -pav "$SOLR_HOME/." "$remote_dir1/."
+
+    echo "Running rsync from networked SOLR_HOME to $remote_dir2"
+    ssh $solr_host "if [ ! -d $local_dir2 ] ; then mkdir $local_dir2 ; fi"
+    rsync -pav "$SOLR_HOME/." "$remote_dir2/."
+  done
+fi

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/gslis-cluster/SCRIPTS/remote-solr-start-all.sh

@@ -1 +1 @@
 #!/bin/bash
 
-run_from_local_disk=1
 
-#local_dir="/tmp/solr-ef"
-local_dir="/hdfsd05/dbbridge/solr-ef"
+if [ "x$SOLR_SHARDS" != "x" ] ; then
+    SOLR_NODES_ARRAY=($SOLR_NODES)
+    SOLR_SHARDS_ARRAY=($SOLR_SHARDS)
+
+    num_shards=${#SOLR_NODES_ARRAY[*]}
+    
+    i=0
+    
+    while [ $i -lt $num_shards ] ; do
+    solr_node=${SOLR_NODES_ARRAY[$i]}
+    solr_home_shard_dir=${SOLR_SHARDS_ARRAY[$i]}
+    
+    solr_host=${solr_node%:*}
+    solr_port=${solr_node##*:}
+
+    server_dir="/disk0/solr-server-$solr_host-$solr_port"
+    
+    export SOLR_HOME=
+    export SOLR_PID_DIR=
+    #export SOLR_PID_DIR="$solr_home_shard_dir"
+    echo "Starting solr cloud node on: $solr_host:$solr_port solr_home=$solr_home_shard_dir"
+    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port -d "$server_dir" -s "$solr_home_shard_dir"
+
+    i=$((i+1))
+    done
+else
+
+    run_from_local_disk=1
+
+    #local_dir="/tmp/solr-ef"
+    local_dir="/hdfsd05/dbbridge/solr-ef"
 
 
-if [ $run_from_local_disk = "1" ] ; then
-  opt_s="-s $local_dir"
-  export SOLR_HOME=
-  echo "****"
-  echo "* Running solr from local disk: $local_dir"
-  echo "****"
-else 
-  opt_s=
+    if [ $run_from_local_disk = "1" ] ; then
+    opt_s="-s $local_dir"
+    export SOLR_HOME=
+    echo "****"
+    echo "* Running solr from local disk: $local_dir"
+    echo "****"
+    else 
+    opt_s=
+    fi
+    
+    for solr_node in $SOLR_NODES ; do
+    solr_host=${solr_node%:*}
+    solr_port=${solr_node##*:}
+    
+    echo "Starting solr cloud node on: $solr_host [port $solr_port]"
+    ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s
+    done
 fi
-
-for solr_node in $SOLR_NODES ; do
-  solr_host=${solr_node%:*}
-  solr_port=${solr_node##*:}
-
-  echo "Starting solr cloud node on: $solr_host [port $solr_port]"
-  ssh $solr_host solr start -cloud -z $ZOOKEEPER_SERVER -h $solr_host -p $solr_port $opt_s
-done