Improvements and fixes to create cluster with performance tuning (#26)

SolaceProducts · Feb 22, 2019 · 59f39c4 · 59f39c4
1 parent 3c6d32a
commit 59f39c4
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 14 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -32,10 +32,20 @@ script:
   - export EVAL_IMAGE_URL=$SOLACE_IMAGE_URL
   - echo "Testing cluster create script"
   - chmod 755 create_cluster.sh
-  - export TESTCLUSTERNAME="sol-gke-travistest-$(date +%s)"
-  - ./create_cluster.sh -z us-central1-b,us-central1-c,us-central1-f -c $TESTCLUSTERNAME -m g1-small
+  - echo "Testing cluster create script with no perf tuning"
+  - export TESTCLUSTERNAME1="sol-gke-travistest-$(date +%s)"
+  - ./create_cluster.sh -z us-central1-b,us-central1-c,us-central1-f -c $TESTCLUSTERNAME1 -m g1-small
   - kubectl get statefulset,svc,pods,pvc,pv
-
+  - echo "Testing cluster create script with perf tuning"
+  - export TESTCLUSTERNAME2="sol-gke-travistest-$(date +%s)"
+  - ./create_cluster.sh -z us-central1-b,us-central1-c,us-central1-f -c $TESTCLUSTERNAME2 -m g1-small -i cos -p
+  - gcloud compute instances list --filter 'tags.items:$TESTCLUSTERNAME2' --format='table(name,zone,status)' | sed 1d $rpt
+  - command="sudo ls /etc/sysctl.d"
+  - list=`gcloud compute instances list --filter tags.items:$TESTCLUSTERNAME2 --format='table(name,zone,status)' | sed 1d $rpt`
+  - # Check if occurrences of 99-sysctl.conf and 98-solace-sysctl.conf equal
+  - a=`while read -r a b c ; do (gcloud compute ssh --ssh-flag="-T -o StrictHostKeyChecking=no" --zone $b $a -- "$command" &) ; done <<< "$list" ; sleep 2 ; wait` ; echo $a
+  - bash -c "if [[ `echo \"$a\" | grep -o 99 | wc -l` != `echo \"$a\" | grep -o 98 | wc -l` ]] ; then echo \"Occurrences of 99-sysctl.conf and 98-solace-sysctl.conf not equal\" ; exit 1 ; fi"
+
 
 after_success:
   - echo "Test Success - Branch($TRAVIS_BRANCH) Pull Request($TRAVIS_PULL_REQUEST) Tag($TRAVIS_TAG)"
@@ -46,5 +56,6 @@ after_success:
 after_script:
   - gcloud container images delete $STANDARD_IMAGE_URL --quiet
   - gcloud container images delete $EVAL_IMAGE_URL --quiet
-  - gcloud container clusters delete $TESTCLUSTERNAME --quiet --zone us-central1-b
+  - gcloud container clusters delete $TESTCLUSTERNAME1 --quiet --zone us-central1-b
+  - gcloud container clusters delete $TESTCLUSTERNAME2 --quiet --zone us-central1-b
   - gcloud compute disks list | grep travis | sed 1d $rpt | while read -r a b c; do gcloud compute disks delete $a --zone $b --quiet; done
diff --git a/scripts/create_cluster.sh b/scripts/create_cluster.sh
@@ -27,7 +27,7 @@ machine_type="n1-standard-4"
 image_type="UBUNTU"
 number_of_nodes="1"
 zones="us-central1-b"
-bridge_perf_tune=false
+perf_tuning=false
 verbose=0
 
 while getopts "c:i:m:n:z:p" opt; do
@@ -42,7 +42,7 @@ while getopts "c:i:m:n:z:p" opt; do
         ;;
     z)  zones=$OPTARG
         ;;
-    p)  perf_tune=true
+    p)  perf_tuning=true
         ;;
     esac
 done
@@ -51,19 +51,28 @@ shift $((OPTIND-1))
 [ "$1" = "--" ] && shift
 
 verbose=1
-echo "`date` INFO: cluster_name=${cluster_name}, machine_type=${machine_type}, image_type=${image_type}, number_of_nodes=${number_of_nodes}, zones=${zones}, perf_tune=${perf_tune} ,Leftovers: $@"
+echo "`date` INFO: cluster_name=${cluster_name}, machine_type=${machine_type}, image_type=${image_type}, number_of_nodes=${number_of_nodes}, zones=${zones}, perf_tuning=${perf_tuning} ,Leftovers: $@"
 
 # multi-region bridge performance tuning
-node_performance_tune () {
+# arguments: $1=clustername
+node_performance_tuning () {
+  list=`gcloud compute instances list --filter tags.items:$1 --format='table(name,zone,status)' | sed 1d $rpt`
+  echo "List of nodes created:"
+  echo "$list"
+  echo 'Applying multi-region bridge performance tuning to nodes...'
+  # Working around a gcloud command issue of dropping out from loop after first iteration because of key generation
+  command="sudo echo"
+  while read -r a b c ; do
+    gcloud compute ssh --ssh-flag="-T -o StrictHostKeyChecking=no" --zone $b $a -- "$command" &>/dev/null &
+  done <<< "$list"
+  wait
+  # Now the real changes:
   command="echo '
   net.core.rmem_max = 134217728
   net.core.wmem_max = 134217728
   net.ipv4.tcp_rmem = 4096 25165824 67108864
   net.ipv4.tcp_wmem = 4096 25165824 67108864
   net.ipv4.tcp_mtu_probing=1' | sudo tee /etc/sysctl.d/98-solace-sysctl.conf ; sudo sysctl -p /etc/sysctl.d/98-solace-sysctl.conf"
-
-  list=`gcloud compute instances list | grep gke-$1-`
-  echo 'Applying multi-region bridge performance tuning to nodes'
   while read -r a b c ; do
     echo $a
     gcloud compute ssh --ssh-flag="-T -o StrictHostKeyChecking=no" --zone $b $a -- "$command" &>/dev/null &
@@ -79,8 +88,8 @@ gcloud config set compute/zone ${zone_array[0]}
 
 echo "`date` INFO: CREATE CLUSTER"
 echo "#############################################################"
-gcloud container clusters create ${cluster_name} --machine-type=${machine_type} --image-type=${image_type} --node-locations=${zones} --num-nodes=${number_of_nodes}
-if $perf_tune ; then
-  node_performance_tune ${cluster_name}
+gcloud container clusters create ${cluster_name} --machine-type=${machine_type} --image-type=${image_type} --node-locations=${zones} --num-nodes=${number_of_nodes} --tags=${cluster_name}
+if $perf_tuning ; then
+  node_performance_tuning ${cluster_name}
 fi
 gcloud container clusters get-credentials ${cluster_name}