From 18da716be6ef6f9686ec215b495231e501b1e5ad Mon Sep 17 00:00:00 2001
From: KenBarr <ken.barr@solace>
Date: Wed, 23 Jan 2019 15:21:21 -0500
Subject: [PATCH 1/5] Add ability to change scaling tier

---
 README.md                                |  20 ++-
 solace/templates/solaceConfigMap.yaml    | 158 +++++++++++++++++------
 solace/templates/solaceStatefullSet.yaml |   9 +-
 3 files changed, 142 insertions(+), 45 deletions(-)

diff --git a/README.md b/README.md
index eb74e528..cf468068 100644
--- a/README.md
+++ b/README.md
@@ -280,7 +280,7 @@ kubectl delete po/XXX-XXX-solace-<pod-ordinal>
 ```
 > Important: In an HA deployment, delete the pods in this order: 2,1,0 (i.e. Monitoring Node, Backup Messaging Node, Primary Messaging Node). Confirm that the message broker redundancy is up and reconciled before deleting each pod - this can be verified using the CLI `show redundancy` and `show config-sync` commands on the message broker, or by grepping the message broker container logs for `config-sync-check`.
 
-### Modifying the cluster
+### Modifying the ports access
 
 Similarly, to **modify** other deployment parameters, e.g. to change the ports exposed via the loadbalancer, you need to upgrade the release with a new set of ports. In this example we will add the MQTT 1883 tcp port to the loadbalancer.
 
@@ -336,6 +336,24 @@ EOF
 helm upgrade  XXXX-XXXX . --values values.yaml --values port-update.yaml
 ```
 
+### Modifying the scaling tier
+
+Similarly, you can change the scaling tier.  This will change the resource and limits requested to the new values depending on the scaling tier selected.  NOTICE: The change of a scaling teir is done with the HA cluster offline.  Expect the entire HA cluster to be down for 2 minutes or more during this procedure.  In this example we are changing the scaling tier to prod1k.
+
+```
+cd ~/workspace/solace-kubernetes-quickstart/solace
+tee ./scale-tier-update.yaml <<-EOF   # create update file with following contents:
+solace:
+  redundancy: true
+  size: prod1k
+  scalingTierUpgrade: true
+EOF
+helm upgrade  XXXX-XXXX . --values values.yaml --values scale-tier-update.yaml
+kubctrl delete pod XXXX-XXXX-solace-2
+kubctrl delete pod XXXX-XXXX-solace-1
+kubctrl delete pod XXXX-XXXX-solace-0
+```
+
 ## Deleting a deployment
 
 Use Helm to delete a deployment, also called a release:
diff --git a/solace/templates/solaceConfigMap.yaml b/solace/templates/solaceConfigMap.yaml
index 545ce853..6ec9389c 100644
--- a/solace/templates/solaceConfigMap.yaml
+++ b/solace/templates/solaceConfigMap.yaml
@@ -89,7 +89,7 @@ data:
       role=""
       #exclude monitor node from config-sync check
       if [ "${node_ordinal}" != "2" ]; then
-        echo "`date` INFO: ${APP}-Determine if Active or Backup role to know which Virtual router to check"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Determine if Active or Backup role to know which Virtual router to check"
         while [ ${count} -lt ${loop_guard} ]; do 
           role_results=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
                 -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
@@ -105,7 +105,7 @@ data:
             ;;
           esac
           ((count++))
-          echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Redundancy not yet up"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Waited ${run_time} seconds, Redundancy not yet up"
           sleep ${pause}
         done
         if [ ${count} -eq ${loop_guard} ]; then
@@ -114,24 +114,24 @@ data:
         fi
 
         count=0
-        echo "`date` INFO: ${APP}-Active or Backup role is ${role}"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Active or Backup role is ${role}"
         while [ ${count} -lt ${loop_guard} ]; do 
           online_results=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
               -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
               -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${role}/status/activity[text()]"`
           local_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
-          echo "`date` INFO: ${APP}-Local activity state is: ${local_activity}"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Local activity state is ${local_activity}"
           run_time=$((${count} * ${pause}))
           case "${local_activity}" in
             "Local Active")
-              echo "`date` INFO: ${APP}-Redundancy is up locally Active, after ${run_time} seconds"
+              echo "`date -Ins` INFO $(hostname) ${APP}: Redundancy is up locally Active, after ${run_time} seconds"
               echo "`date` INFO: "We should only be here on new cluster create, if not likely a bug"
               echo "`date` INFO: "   will issue a assert master to get back into sync"                              
               resync_step="assert-master"
               break
               ;;
             "Mate Active")
-              echo "`date` INFO: ${APP}-Redundancy is up mate Active, after ${run_time} seconds" 
+              echo "`date -Ins` INFO $(hostname) ${APP}: Redundancy is up mate Active, after ${run_time} seconds" 
               echo "`date` INFO: "This is normal state if we are backup or recreated later on"
               echo "`date` INFO: "   will issue a resync master to get back into sync"            
               resync_step="resync-master"
@@ -139,7 +139,7 @@ data:
               ;;
           esac
           ((count++))
-          echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Redundancy not yet up"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Waited ${run_time} seconds, Redundancy not yet up"
           sleep ${pause}
         done
         if [ ${count} -eq ${loop_guard} ]; then
@@ -151,22 +151,22 @@ data:
         
         if [ "${resync_step}" = "assert-master" ]; then
           count=0
-          echo "`date` INFO: ${APP}-Wait for mate to be 'Standby'"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Wait for mate to be 'Standby'"
           while [ ${count} -lt ${loop_guard} ]; do 
             online_results=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
                 -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
                 -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${role}/status/detail/priority-reported-by-mate/summary[text()]"`
             mate_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
-            echo "`date` INFO: ${APP}-Mate activity state is: ${mate_activity}"
+            echo "`date -Ins` INFO $(hostname) ${APP}: Mate activity state is ${mate_activity}"
             run_time=$((${count} * ${pause}))
             case "${mate_activity}" in
               "Standby")
-                echo "`date` INFO: ${APP}-Redundancy is up end-to-end, Local Active Mate Standby, after ${run_time} seconds"
+                echo "`date -Ins` INFO $(hostname) ${APP}: Redundancy is up end-to-end, Local Active Mate Standby, after ${run_time} seconds"
                 break
                 ;;
             esac
             ((count++))
-            echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Mate not yet in sync"
+            echo "`date -Ins` INFO $(hostname) ${APP}: Waited ${run_time} seconds, Mate not yet in sync"
             sleep ${pause}
           done
           if [ ${count} -eq ${loop_guard} ]; then
@@ -179,7 +179,7 @@ data:
               -q "<rpc><admin><config-sync><${resync_step}><router/></${resync_step}></config-sync></admin></rpc>"
         {{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
               -q "<rpc><admin><config-sync><${resync_step}><vpn-name>default</vpn-name></${resync_step}></config-sync></admin></rpc>"
-        echo "`date` INFO: ${APP}-Solace VMR bringup complete"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Solace VMR bringup complete"
       fi # if not monitor
 {{- end }}
       exit 0
@@ -193,7 +193,7 @@ data:
       #Prevent overdriving Kubernetes infra, don't set activity state to same as previous state
       previous_state=`cat $3`
       if [ "${2}" = "${previous_state}" ]; then
-        echo "`date` INFO: ${APP}-Current and Previous state match, not updating label"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Current and Previous state match, not updating label"
       else
         echo ${2} > ${3}
         echo "[{\"op\": \"add\", \"path\": \"/metadata/labels/${1}\", \"value\": \"${2}\" }]" > /tmp/patch_label.json
@@ -217,7 +217,7 @@ data:
     version=${1}
     IFS='-' read -ra host_array <<< $(hostname)
     node_ordinal=${host_array[-1]}
-    echo "`date` INFO: ${APP}-node ordinal: ${node_ordinal}"
+    echo "`date -Ins` INFO $(hostname) ${APP}: node ordinal ${node_ordinal}"
     password=`cat {{ .Values.filepaths.secrets }}/username_admin_password`
 
     # For upgrade purposes, ensure redundancy is up only when the pod is started
@@ -229,7 +229,7 @@ data:
               -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"`
       nr_node_results=`echo ${role_results} |  xmllint -xpath "string(returnInfo/countSearchResult)" -`
       if [ $nr_node_results -ne 3 ]; then
-        echo "`date` INFO: ${APP}-Not all nodes are online. Query results: ${nr_node_results}"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Not all nodes are online. Query results: ${nr_node_results}"
         exit 1
       fi
       # Then for each node determine the ip address and check redundancy. Note: id starts here from 1 and not 0.
@@ -243,7 +243,7 @@ data:
                 -v "/rpc-reply/rpc/show/redundancy/redundancy-status"`
         redundancystatus_results=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
         if [ "${redundancystatus_results}" != "Up" ]; then
-          echo "`date` INFO: ${APP}-Redundancy state is not yet up. Query results: ${redundancystatus_results}"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Redundancy state is not yet up. Query results ${redundancystatus_results}"
           exit 1
         fi
       done
@@ -254,7 +254,7 @@ data:
                 -v "/rpc-reply/rpc/show/config-sync/status/oper-status"`
         confsyncstatus_results=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
         if [ "${confsyncstatus_results}" != "Up" ]; then
-          echo "`date` INFO: ${APP}-Config-sync state is not yet up. Query results: ${confsyncstatus_results}"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Config-sync state is not yet up. Query results ${confsyncstatus_results}"
           exit 1
         fi
       fi
@@ -266,7 +266,7 @@ data:
         node_ip_address=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
         nr_voter_results=`curl --unix-socket /var/run/consul -s http://$node_ip_address:8500/v1/operator/raft/configuration  | python -m json.tool | grep Voter | grep true | wc -l`
         if [ $nr_voter_results -ne 3 ]; then
-          echo "`date` INFO: ${APP}-Still waiting for all 3 Consul voters to be present for node $node_ip_address. Query results: ${nr_voter_results}"
+          echo "`date -Ins` INFO $(hostname) ${APP}: Still waiting for all 3 Consul voters to be present for node $node_ip_address. Query results ${nr_voter_results}"
           exit 1
         fi
       done
@@ -275,21 +275,20 @@ data:
       # wait at first startup for stability
       #sleep 20
     fi
-    
 
     if [ "${node_ordinal}" = "2" ]; then
-      echo "`date` INFO: ${APP}-Monitor node ready check"
+      echo "`date -Ins` INFO $(hostname) ${APP}: Monitor node ready check"
       # Note that when dealing with Monitor, only need to be concerned and readiness response.
       # active label will always be "false"
-      echo "`date` INFO: ${APP}-For monitor node just check for 3 online nodes in group"
+      echo "`date -Ins` INFO $(hostname) ${APP}: For monitor node just check for 3 online nodes in group"
       role_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
               -q "<rpc><show><redundancy><group/></redundancy></show></rpc>" \
               -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"`
       if [ `echo ${role_results} |  xmllint -xpath "string(returnInfo/countSearchResult)" -` -eq 3 ]; then
-        echo "`date` INFO: ${APP}-Monitor node is redundancy ready"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Monitor node is redundancy ready"
         exit 0
       else
-        echo "`date` INFO: ${APP}-Monitor node is not redundancy ready, result: ${role_results}"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Monitor node is not redundancy ready, result ${role_results}"
         exit 1
       fi
     fi # End Monitor Node
@@ -298,26 +297,26 @@ data:
     
     case "${health_result}" in
       "200")
-         echo "`date` INFO: ${APP}-Message Router is Active and Healthy"
+         echo "`date -Ins` INFO $(hostname) ${APP}: Message Router is Active and Healthy"
         set_label "active" "true" $state_file
         exit 0
         ;;
       "503")
         set_label "active" "false" $state_file
         if (( "$version" < 7 )); then
-         echo "`date` INFO: ${APP}-Message Router is Healthy and not Active, this is K8S 1.6 ready"
+         echo "`date -Ins` INFO $(hostname) ${APP}: Message Router is Healthy and not Active, this is K8S 1.6 ready"
          exit 0
         else
-         echo "`date` INFO: ${APP}-Message Router is Healthy and not Active, further check required"
+         echo "`date -Ins` INFO $(hostname) ${APP}: Message Router is Healthy and not Active, further check required"
         fi
         ;;
       "")
-        echo "`date` WARN: ${APP}-Unable to deturmine config role, failing readiness check"
+        echo "`date -Ins` WARN $(hostname) ${APP}: Unable to deturmine config role, failing readiness check"
         set_label "active" "false" $state_file
         exit 1
     esac
      
-    echo "`date` INFO: ${APP}-Checking if Message Router is Standby"
+    echo "`date -Ins` INFO $(hostname) ${APP}: Checking if Message Router is Standby"
     case "${node_ordinal}" in
       "0")
         config_role="primary"
@@ -330,24 +329,24 @@ data:
             -q "<rpc><show><redundancy><detail/></redundancy></show></rpc>" \
             -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${config_role}/status/activity[text()]"`
     local_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
-    echo "`date` INFO: ${APP}-Local activity state is: ${local_activity}"
+    echo "`date -Ins` INFO $(hostname) ${APP}: Local activity state is ${local_activity}"
     case "${local_activity}" in
       "Local Active")
-        echo "`date` INFO: ${APP}-Redundancy is up locally Active"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Redundancy is up locally Active"
         # Set active label to "true"
         set_label "active" "true" $state_file
         # Pass readiness check
         exit 0
         ;;
       "Mate Active")
-        echo "`date` INFO: ${APP}-Redundancy is up mate Active"
+        echo "`date -Ins` INFO $(hostname) ${APP}: Redundancy is up mate Active"
         # Set active label to "false"
         set_label "active" "false" $state_file
         # Pass readiness check
         exit 0
         ;;
       *)
-        echo "`date` WARN: ${APP}-Redundancy not up or not responding, failing readiness check"
+        echo "`date -Ins` WARN $(hostname) ${APP}: Redundancy not up or not responding, failing readiness check"
         # Set active label to "false"
         set_label "active" "false" $state_file
         # Fail readiness check
@@ -360,22 +359,97 @@ data:
     
     case "${health_result}" in
       "200")
-        echo "`date` INFO: ${APP}-nonHA Message Router is Active and Healthy"
+        echo "`date -Ins` INFO $(hostname) ${APP}: nonHA Message Router is Active and Healthy"
         set_label "active" "true" $state_file
         exit 0
         ;;
       "503")
-        echo "`date` INFO: ${APP}-nonHA Message Router message spool is down"
+        echo "`date -Ins` INFO $(hostname) ${APP}: nonHA Message Router message spool is down"
         set_label "active" "false" $state_file
         exit 1
         ;;
       "")
-        echo "`date` WARN: ${APP}-Unable to determine config role, failing readiness check"
+        echo "`date -Ins` WARN $(hostname) ${APP}: Unable to determine config role, failing readiness check"
         set_label "active" "false" $state_file
         exit 1
     esac
 {{- end }}
 
+
+  vertical_scale_upgrade.sh: |-
+      #!/bin/bash
+      password=`cat {{ .Values.filepaths.secrets }}/username_admin_password`
+      IFS='-' read -ra host_array <<< $(hostname)
+      node_ordinal=${host_array[-1]}
+      APP=`basename "$0"`
+      echo "`date -Ins` INFO $(hostname) ${APP}: Sclaing tier for node ordinal ${node_ordinal}"
+      OPTIND=1         # Reset in case getopts has been used previously in the shell.
+      # Initialize our own variables:
+      namespace="default"
+      release_name=""
+      size=""
+      verbose=0
+      loop_guard=0
+      max_tries=30
+      try_interval=10
+      while getopts "r:s:" opt; do
+          case "$opt" in
+          r)  release_name=$OPTARG
+              ;;
+          s)  size=$OPTARG
+              ;;
+          esac
+      done
+      shift $((OPTIND-1))
+      [ "$1" = "--" ] && shift
+      case ${node_ordinal} in
+        2)
+          echo "`date -Ins` INFO $(hostname) ${APP}: Tier scaling for mnitor node is a no-op exiting script"
+          exit 0
+        ;;
+      esac
+
+      case ${size} in
+      "dev100")
+        connection_count=100
+        ;;
+      "prod100")
+        connection_count=100
+        ;;
+      "prod1k")
+        connection_count=1000
+        ;;
+      "prod10k")
+        connection_count=10000
+        ;;
+      "prod100k")
+        connection_count=100000
+        ;;
+      "prod200k")
+        connection_count=200000
+        ;;
+      esac     
+      echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection count to ${connection_count} would happen here"
+      loop_guard=0
+      while [ $loop_guard -le $max_tries ]; do
+        ((loop_guard++))
+        echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling to ${connection_count} try ${loop_guard}"
+      #  cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+      #    -q "<rpc><system><scaling><max-connections><max-connections>${connection_count}</max-connections></max-connections></scaling></system></rpc>"`
+      cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+          -q "<rpc><reload/></rpc>"`
+        if [[ -z $cc_upgrade_results ]]; then
+          echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling passed: ${cc_upgrade_results} :"
+          break
+        else
+          echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling failed: ${cc_upgrade_results} :"
+        fi
+        sleep ${try_interval}
+      done
+      if [ $loop_guard -eq $max_tries ]; then
+        echo "`date -Ins` INFO $(hostname) ${APP}: Failed to set connection limit to: ${connection_count}"
+      fi
+
   semp_query.sh: |-
       #!/bin/bash
       APP=`basename "$0"`
@@ -408,30 +482,30 @@ data:
       shift $((OPTIND-1))
       [ "$1" = "--" ] && shift
       verbose=1
-      echo "`date` INFO: ${APP}-${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \
+      echo "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \
                   ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2
       if [[ ${url} = "" || ${name} = "" || ${password} = "" || ${query} = "" ]]; then
-          echo "`date` ERROR: ${APP}-${script_name}: url, name, password and query are madatory fields" >&2
+          echo "`date -Ins` ERROR $(hostname) ${APP}:  url, name, password and query are madatory fields" >&2
           echo  '<returnInfo><errorInfo>missing parameter</errorInfo></returnInfo>'
           exit 1
         fi
       query_response=`curl -sS -u ${name}:${password} ${url} -d "${query}"`
       # Validate first char of response is "<", otherwise no hope of being valid xml
       if [[ ${query_response:0:1} != "<" ]] ; then 
-        echo "`date` ERROR: ${APP}-${script_name}: Query failed, non-xml response -${query_response}-" >&2
+        echo "`date -Ins` ERROR $(hostname) ${APP}:  Query failed, non-xml response -${query_response}-" >&2
         exit 1
       fi
       query_response_code=`echo $query_response | xmllint -xpath 'string(/rpc-reply/execute-result/@code)' -`
 
       if [[ -z ${query_response_code} && ${query_response_code} != "ok" ]]; then
-          echo "`date` ERROR: ${APP}-${script_name}: Query failed, bad return code -${query_response}-" >&2
+          echo "`date -Ins` ERROR $(hostname) ${APP}:  Query failed, bad return code -${query_response}-" >&2
           echo  "<returnInfo><errorInfo>query failed -${query_response_code}-</errorInfo></returnInfo>"
           exit 1
       fi
-      echo "`date` INFO: ${APP}-${script_name}: Query passed ${query_response_code}" >&2
+      echo "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: Query passed ${query_response_code}" >&2
       if [[ ! -z $value_search ]]; then
           value_result=`echo $query_response | xmllint -xpath "string($value_search)" -`
-          echo "`date` INFO: ${APP}-${script_name}: Value search $value_search returned ${value_result}" >&2
+          echo "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: Value search $value_search returned ${value_result}" >&2
           echo  "<returnInfo><errorInfo></errorInfo><valueSearchResult>${value_result}</valueSearchResult></returnInfo>"
           exit 0
       fi
@@ -439,7 +513,7 @@ data:
           count_line=`echo $query_response | xmllint -xpath "$count_search" -`
           count_string=`echo $count_search | cut -d '"' -f 2`
           count_result=`echo ${count_line} | tr "><" "\n" | grep -c ${count_string}`
-          echo -e "`date` INFO: ${APP}-${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
+          echo -e "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
           echo  "<returnInfo><errorInfo></errorInfo><countSearchResult>${count_result}</countSearchResult></returnInfo>"
           exit 0
       fi
\ No newline at end of file
diff --git a/solace/templates/solaceStatefullSet.yaml b/solace/templates/solaceStatefullSet.yaml
index 517b81d9..cbc4485c 100644
--- a/solace/templates/solaceStatefullSet.yaml
+++ b/solace/templates/solaceStatefullSet.yaml
@@ -12,9 +12,11 @@ metadata:
 spec:
   serviceName: {{ template "solace.fullname" . }}-discovery
   replicas: {{- if .Values.solace.redundancy }} 3 {{- else }} 1 {{- end }}
-{{- if semverCompare ">=1.7-0" .Capabilities.KubeVersion.GitVersion }}  
   podManagementPolicy: Parallel
-  updateStrategy: 
+  updateStrategy:
+{{- if .Values.solace.scalingTierUpgrade }} 
+    type: OnDelete
+{{- else }} 
     type: RollingUpdate
 {{- end }}
   template:
@@ -114,6 +116,9 @@ spec:
            # not using postinstall hooks because of order dependencies
            # launch config check then Solace so VCMR can provide return code
            nohup {{ .Values.filepaths.configmap }}/config-sync-check.sh &
+{{- if .Values.solace.scalingTierUpgrade }}
+           nohup {{ .Values.filepaths.configmap }}/vertical_scale_upgrade.sh \-s {{ .Values.solace.size }} \-r {{ .Release.Name }} &
+{{- end}}
            /usr/sbin/boot.sh
 
         volumeMounts:

From f2e4ca9e9b076598d33e7402d450dd462f3ae7cc Mon Sep 17 00:00:00 2001
From: KenBarr <ken.barr@solace>
Date: Tue, 5 Mar 2019 14:52:28 -0500
Subject: [PATCH 2/5] Update to support horizontal scale

---
 solace/templates/post-upgrade.yaml | 51 ++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 solace/templates/post-upgrade.yaml

diff --git a/solace/templates/post-upgrade.yaml b/solace/templates/post-upgrade.yaml
new file mode 100644
index 00000000..04f1be4b
--- /dev/null
+++ b/solace/templates/post-upgrade.yaml
@@ -0,0 +1,51 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "solace.fullname" . }}-postupgrade
+  labels:
+    app: {{ template "solace.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": "post-upgrade"
+    "helm.sh/hook-delete-policy": "before-hook-creation"
+spec:
+  template:
+    metadata:
+      labels:
+        app: {{ template "solace.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: post-upgrade-job
+        image: "alpine:latest"
+        env:
+        - name: STATEFULSET_NAME
+          value: {{ template "solace.fullname" . }}
+        - name: STATEFULSET_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        command: ["/bin/sh", "-c"]
+        args: [ "apk add --no-cache bash; apk add --no-cache curl; apk add --no-cache libxml2-utils; {{ .Values.filepaths.configmap }}/post_vertical_scale_upgrade.sh -s {{ .Values.solace.size }} -r {{ .Release.Name }}" ]
+
+        volumeMounts:
+        - name: config-map
+          mountPath: {{ .Values.filepaths.configmap }}
+        - name: secrets
+          mountPath: {{ .Values.filepaths.secrets }}
+          readOnly: true
+
+      volumes:
+      - name: config-map
+        configMap:
+          name: {{ template "solace.fullname" . }}
+          defaultMode: 0755
+      - name: secrets
+        secret:
+          secretName: {{ template "solace.fullname" . }}-secrets
+          defaultMode: 0400
\ No newline at end of file

From eaa0360ec8dae5847c928470ab5c7a86050a0be1 Mon Sep 17 00:00:00 2001
From: KenBarr <ken.barr@solace>
Date: Tue, 5 Mar 2019 14:53:31 -0500
Subject: [PATCH 3/5] Update to support vertical scale

---
 solace/templates/solaceConfigMap.yaml    | 171 ++++++++++++++++++-----
 solace/templates/solaceStatefullSet.yaml |   3 +-
 2 files changed, 141 insertions(+), 33 deletions(-)

diff --git a/solace/templates/solaceConfigMap.yaml b/solace/templates/solaceConfigMap.yaml
index 6ec9389c..c8b33f95 100644
--- a/solace/templates/solaceConfigMap.yaml
+++ b/solace/templates/solaceConfigMap.yaml
@@ -180,8 +180,8 @@ data:
         {{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
               -q "<rpc><admin><config-sync><${resync_step}><vpn-name>default</vpn-name></${resync_step}></config-sync></admin></rpc>"
         echo "`date -Ins` INFO $(hostname) ${APP}: Solace VMR bringup complete"
-      fi # if not monitor
-{{- end }}
+      fi # end if not monitor,(node_ordinal =2)
+{{- end }} # end if redundancy
       exit 0
 
 
@@ -227,7 +227,7 @@ data:
       results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
               -q "<rpc><show><redundancy><group/></redundancy></show></rpc>" \
               -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"`
-      nr_node_results=`echo ${role_results} |  xmllint -xpath "string(returnInfo/countSearchResult)" -`
+      nr_node_results=`echo ${results} |  xmllint -xpath "string(returnInfo/countSearchResult)" -`
       if [ $nr_node_results -ne 3 ]; then
         echo "`date -Ins` INFO $(hostname) ${APP}: Not all nodes are online. Query results: ${nr_node_results}"
         exit 1
@@ -311,7 +311,7 @@ data:
         fi
         ;;
       "")
-        echo "`date -Ins` WARN $(hostname) ${APP}: Unable to deturmine config role, failing readiness check"
+        echo "`date -Ins` WARN $(hostname) ${APP}: Unable to deturmine config role, failing readiness check" >&2
         set_label "active" "false" $state_file
         exit 1
     esac
@@ -346,7 +346,7 @@ data:
         exit 0
         ;;
       *)
-        echo "`date -Ins` WARN $(hostname) ${APP}: Redundancy not up or not responding, failing readiness check"
+        echo "`date -Ins` WARN $(hostname) ${APP}: Redundancy not up or not responding, failing readiness check" >&2
         # Set active label to "false"
         set_label "active" "false" $state_file
         # Fail readiness check
@@ -369,7 +369,7 @@ data:
         exit 1
         ;;
       "")
-        echo "`date -Ins` WARN $(hostname) ${APP}: Unable to determine config role, failing readiness check"
+        echo "`date -Ins` WARN $(hostname) ${APP}: Unable to determine config role, failing readiness check" >&2
         set_label "active" "false" $state_file
         exit 1
     esac
@@ -382,7 +382,7 @@ data:
       IFS='-' read -ra host_array <<< $(hostname)
       node_ordinal=${host_array[-1]}
       APP=`basename "$0"`
-      echo "`date -Ins` INFO $(hostname) ${APP}: Sclaing tier for node ordinal ${node_ordinal}"
+      echo "`date -Ins` INFO $(hostname) ${APP}: Scaling tier for node ordinal ${node_ordinal}"
       OPTIND=1         # Reset in case getopts has been used previously in the shell.
       # Initialize our own variables:
       namespace="default"
@@ -402,13 +402,23 @@ data:
       done
       shift $((OPTIND-1))
       [ "$1" = "--" ] && shift
+
       case ${node_ordinal} in
-        2)
-          echo "`date -Ins` INFO $(hostname) ${APP}: Tier scaling for mnitor node is a no-op exiting script"
-          exit 0
+      2)
+        echo "`date -Ins` INFO $(hostname) ${APP}: Connection scaling on the monitore node is a no-op"
+        exit 0
         ;;
       esac
 
+      # Shut down message spool and message backbone on Primary and backup
+      result=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
+        -q "<rpc><service><msg-backbone><shutdown/></msg-backbone></service></rpc>"`
+      echo "`date -Ins` INFO $(hostname) ${APP}: shutting message-backbone for Hostname: $(host_name) result ${result}"
+      result=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
+        -q "<rpc><hardware><message-spool><shutdown/></message-spool></hardware></rpc>"`
+      echo "`date -Ins` INFO $(hostname) ${APP}: shutting message-spool for Hostname: $(host_name) result ${result}"
+  
+      # Convert kubernetes tee-shirt size to actual solace connection count
       case ${size} in
       "dev100")
         connection_count=100
@@ -428,28 +438,107 @@ data:
       "prod200k")
         connection_count=200000
         ;;
-      esac     
-      echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection count to ${connection_count} would happen here"
+      esac
+
+      echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection count to ${connection_count}"
       loop_guard=0
       while [ $loop_guard -le $max_tries ]; do
         ((loop_guard++))
         echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling to ${connection_count} try ${loop_guard}"
-      #  cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-      #    -q "<rpc><system><scaling><max-connections><max-connections>${connection_count}</max-connections></max-connections></scaling></system></rpc>"`
-      cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-          -q "<rpc><reload/></rpc>"`
+        cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+          -q "<rpc><system><scaling><max-connections><max-connections>${connection_count}</max-connections></max-connections></scaling></system></rpc>"`
+      #cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
+      #    -q "<rpc><reload/></rpc>"`
         if [[ -z $cc_upgrade_results ]]; then
           echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling passed: ${cc_upgrade_results} :"
           break
         else
           echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling failed: ${cc_upgrade_results} :"
+          sleep ${try_interval}
         fi
-        sleep ${try_interval}
       done
       if [ $loop_guard -eq $max_tries ]; then
         echo "`date -Ins` INFO $(hostname) ${APP}: Failed to set connection limit to: ${connection_count}"
       fi
 
+  post_vertical_scale_upgrade.sh: |-
+      #!/bin/bash
+      password=`cat {{ .Values.filepaths.secrets }}/username_admin_password`
+      OPTIND=1         # Reset in case getopts has been used previously in the shell.
+      # Initialize our own variables:
+      release_name=""
+      size=""
+      verbose=0
+      while getopts "r:s:" opt; do
+          case "$opt" in
+          r)  release_name=$OPTARG
+              ;;
+          s)  size=$OPTARG
+              ;;
+          esac
+      done
+      shift $((OPTIND-1))
+      [ "$1" = "--" ] && shift
+      if [[ ! -z `echo $STATEFULSET_NAMESPACE` ]]; then
+        namespace=`echo $STATEFULSET_NAMESPACE`
+      else
+        namespace=default
+      fi
+
+      # Convert kubernetes tee-shirt size to actual solace connection count
+      case ${size} in
+      "dev100")
+        connection_count=100
+        ;;
+      "prod100")
+        connection_count=100
+        ;;
+      "prod1k")
+        connection_count=1000
+        ;;
+      "prod10k")
+        connection_count=10000
+        ;;
+      "prod100k")
+        connection_count=100000
+        ;;
+      "prod200k")
+        connection_count=200000
+        ;;
+      esac
+
+      current_max_connection=0
+      loop_guard=0
+      max_tries=180
+      for i in `seq 0 1`; do
+        host_name="${release_name}-solace-${i}.${release_name}-solace-discovery.${namespace}.svc"
+        while [ $current_max_connection -ne $connection_count ]; do
+          results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://${host_name}:8080/SEMP -r 60 \
+              -q "<rpc><show><system><detail/></system></show></rpc>" \
+              -v "/rpc-reply/rpc/show/system/max-connections[text()]"`
+          current_max_connection=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+          ((loop_guard++))
+          sleep 10
+          if [ $loop_guard -eq $max_tries ]; then
+            echo "`date -Iseconds` ERROR $(hostname) ${APP}: Failed get correct max connection count. Current value: ${current_max-connection} Expected value: ${connection_count}"
+            exit 1
+          fi
+        done
+        echo "`date -Iseconds` INFO $(hostname) ${APP}: Bring up message-backbone for Hostname: ${host_name}"
+        /mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://${host_name}:8080/SEMP -r 60 \
+          -q "<rpc><service><msg-backbone><no><shutdown/></no></msg-backbone></service></rpc>"
+
+        if [ ${i} -eq 0 ]; then
+          role="<primary/>"
+        else
+          role="<backup/>"
+        fi
+        echo "`date -Iseconds` INFO $(hostname) ${APP}: Bring up message-spool for Hostname: ${host_name} role: ${role}"
+        /mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://${host_name}:8080/SEMP -r 60 \
+          -q "<rpc><hardware><message-spool><no><shutdown>${role}</shutdown></no></message-spool></hardware></rpc>"
+      done
+      exit 0
+
   semp_query.sh: |-
       #!/bin/bash
       APP=`basename "$0"`
@@ -461,18 +550,24 @@ data:
       query=""
       url=""
       value_search=""
+      retry_count=0
+      retry_interval=10
       script_name=$0
       verbose=0
-      while getopts "c:n:p:q:u:v:" opt; do
+      while getopts "c:i:n:p:q:r:u:v:" opt; do
           case "$opt" in
           c)  count_search=$OPTARG
               ;;
+          i)  retry_interval=$OPTARG
+              ;;
           n)  name=$OPTARG
               ;;
           p)  password=$OPTARG
               ;;
           q)  query=$OPTARG
               ;;
+          r)  retry_count=$OPTARG
+              ;;
           u)  url=$OPTARG
               ;;
           v)  value_search=$OPTARG
@@ -482,30 +577,42 @@ data:
       shift $((OPTIND-1))
       [ "$1" = "--" ] && shift
       verbose=1
-      echo "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \
+      echo "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: count_search=${count_search} ,name=${name} \
+                  ,retry_count=${retry_count}, retry_interval=${retry_interval}, query=${query} \
                   ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2
       if [[ ${url} = "" || ${name} = "" || ${password} = "" || ${query} = "" ]]; then
-          echo "`date -Ins` ERROR $(hostname) ${APP}:  url, name, password and query are madatory fields" >&2
+          echo "`date -Iseconds` ERROR $(hostname) ${APP}:  url, name, password and query are madatory fields" >&2
           echo  '<returnInfo><errorInfo>missing parameter</errorInfo></returnInfo>'
           exit 1
         fi
-      query_response=`curl -sS -u ${name}:${password} ${url} -d "${query}"`
-      # Validate first char of response is "<", otherwise no hope of being valid xml
-      if [[ ${query_response:0:1} != "<" ]] ; then 
-        echo "`date -Ins` ERROR $(hostname) ${APP}:  Query failed, non-xml response -${query_response}-" >&2
+      retry_guard=0
+      query_success=
+      while [ ${retry_count} -ge ${retry_guard} ]; do
+        query_response=`curl -sS -u ${name}:${password} ${url} -d "${query}"`
+        ((retry_guard++))
+        # Validate first char of response is "<", otherwise no hope of being valid xml
+        if [[ ${query_response:0:1} != "<" ]] ; then 
+          echo "`date -Iseconds` WARN $(hostname) ${APP}:  Query failed count ${retry_guard}, non-xml response -${query_response}-" >&2
+          sleep ${retry_interval}
+        else
+          query_success=1
+          break
+        fi
+      done
+      if [[ -z ${query_success} ]]; then
+        echo "`date --Iseconds` ERROR $(hostname) ${APP}:  Query failed non-xml response -${query_response}-" >&2
         exit 1
       fi
       query_response_code=`echo $query_response | xmllint -xpath 'string(/rpc-reply/execute-result/@code)' -`
-
       if [[ -z ${query_response_code} && ${query_response_code} != "ok" ]]; then
-          echo "`date -Ins` ERROR $(hostname) ${APP}:  Query failed, bad return code -${query_response}-" >&2
-          echo  "<returnInfo><errorInfo>query failed -${query_response_code}-</errorInfo></returnInfo>"
-          exit 1
+        echo "`date -Iseconds` ERROR $(hostname) ${APP}:  Query failed, bad return code -${query_response}-" >&2
+        echo  "<returnInfo><errorInfo>query failed -${query_response_code}-</errorInfo></returnInfo>"
+        exit 1
       fi
-      echo "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: Query passed ${query_response_code}" >&2
+      echo "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: Query passed ${query_response_code}" >&2
       if [[ ! -z $value_search ]]; then
           value_result=`echo $query_response | xmllint -xpath "string($value_search)" -`
-          echo "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: Value search $value_search returned ${value_result}" >&2
+          echo "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: Value search $value_search returned ${value_result}" >&2
           echo  "<returnInfo><errorInfo></errorInfo><valueSearchResult>${value_result}</valueSearchResult></returnInfo>"
           exit 0
       fi
@@ -513,7 +620,7 @@ data:
           count_line=`echo $query_response | xmllint -xpath "$count_search" -`
           count_string=`echo $count_search | cut -d '"' -f 2`
           count_result=`echo ${count_line} | tr "><" "\n" | grep -c ${count_string}`
-          echo -e "`date -Ins` INFO $(hostname) ${APP}: ${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
-          echo  "<returnInfo><errorInfo></errorInfo><countSearchResult>${count_result}</countSearchResult></returnInfo>"
+          echo -e "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
+          echo  "<returnInfo><errorInfo></errorInfo><countSearchResult>${count_result}</countSearchResult></returnInfo>" 
           exit 0
       fi
\ No newline at end of file
diff --git a/solace/templates/solaceStatefullSet.yaml b/solace/templates/solaceStatefullSet.yaml
index cbc4485c..0278b063 100644
--- a/solace/templates/solaceStatefullSet.yaml
+++ b/solace/templates/solaceStatefullSet.yaml
@@ -114,9 +114,10 @@ spec:
          - |
            source {{ .Values.filepaths.configmap }}/init.sh
            # not using postinstall hooks because of order dependencies
-           # launch config check then Solace so VCMR can provide return code
+           # launch config check then Solace so VMR boot can provide return code
            nohup {{ .Values.filepaths.configmap }}/config-sync-check.sh &
 {{- if .Values.solace.scalingTierUpgrade }}
+           # {{ .Values.filepaths.configmap }}/pre_vertical_scale_upgrade.sh \-r {{ .Release.Name }}
            nohup {{ .Values.filepaths.configmap }}/vertical_scale_upgrade.sh \-s {{ .Values.solace.size }} \-r {{ .Release.Name }} &
 {{- end}}
            /usr/sbin/boot.sh

From 903fb104edb1b6b4f05431cb0636509caab7e0d9 Mon Sep 17 00:00:00 2001
From: KenBarr <ken.barr@solace>
Date: Wed, 20 Mar 2019 11:41:06 -0400
Subject: [PATCH 4/5] Add post upgrade script

---
 solace/templates/solaceConfigMap.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/solace/templates/solaceConfigMap.yaml b/solace/templates/solaceConfigMap.yaml
index c8b33f95..9c74adc1 100644
--- a/solace/templates/solaceConfigMap.yaml
+++ b/solace/templates/solaceConfigMap.yaml
@@ -191,10 +191,11 @@ data:
     # Function to set Kubernetes metadata labels
     set_label () {
       #Prevent overdriving Kubernetes infra, don't set activity state to same as previous state
-      previous_state=`cat $3`
+        previous_state=`cat $3`
       if [ "${2}" = "${previous_state}" ]; then
         echo "`date -Ins` INFO $(hostname) ${APP}: Current and Previous state match, not updating label"
       else
+        echo "`date -Ins` INFO $(hostname) ${APP}: Changing local state from ${previous_state} to ${2}"
         echo ${2} > ${3}
         echo "[{\"op\": \"add\", \"path\": \"/metadata/labels/${1}\", \"value\": \"${2}\" }]" > /tmp/patch_label.json
         KUBE_TOKEN=$(</var/run/secrets/kubernetes.io/serviceaccount/token)
@@ -209,7 +210,7 @@ data:
     state_file=/tmp/activity_state
     if [ ! -f ${state_file} ]; then
       echo "State file not found, creating!"
-      echo "false" > ${state_file}
+      echo "unknown" > ${state_file}
     fi
 
 {{- if .Values.solace.redundancy }}

From 8ff462e4bf6451238ce3e85f242298b4ac300f73 Mon Sep 17 00:00:00 2001
From: KenBarr <ken.barr@solace>
Date: Wed, 20 Mar 2019 18:37:06 -0400
Subject: [PATCH 5/5] Fix timing issue

---
 solace/templates/solaceConfigMap.yaml | 92 ++++++++++++++++-----------
 1 file changed, 56 insertions(+), 36 deletions(-)

diff --git a/solace/templates/solaceConfigMap.yaml b/solace/templates/solaceConfigMap.yaml
index c9a75208..ab13d605 100644
--- a/solace/templates/solaceConfigMap.yaml
+++ b/solace/templates/solaceConfigMap.yaml
@@ -411,14 +411,6 @@ data:
         ;;
       esac
 
-      # Shut down message spool and message backbone on Primary and backup
-      result=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
-        -q "<rpc><service><msg-backbone><shutdown/></msg-backbone></service></rpc>"`
-      echo "`date -Ins` INFO $(hostname) ${APP}: shutting message-backbone for Hostname: $(host_name) result ${result}"
-      result=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
-        -q "<rpc><hardware><message-spool><shutdown/></message-spool></hardware></rpc>"`
-      echo "`date -Ins` INFO $(hostname) ${APP}: shutting message-spool for Hostname: $(host_name) result ${result}"
-  
       # Convert kubernetes tee-shirt size to actual solace connection count
       case ${size} in
       "dev100")
@@ -441,26 +433,45 @@ data:
         ;;
       esac
 
-      echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection count to ${connection_count}"
-      loop_guard=0
-      while [ $loop_guard -le $max_tries ]; do
-        ((loop_guard++))
+
+        #Check to see if we are already at the correct scale tier
+        results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
+            -q "<rpc><show><system><detail/></system></show></rpc>" \
+            -v "/rpc-reply/rpc/show/system/max-connections[text()]"`
+        current_max_connection=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+
+        if [[ ${current_max_connection} -eq ${connection_count} ]]; then
+          echo "`date -Ins` INFO $(hostname) ${APP}: Already at correct scalling tier ${size}, nothing to do"
+          exit 0
+        else
+          echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection count from ${current_max_connection} to ${connection_count}"
+        fi
+
+        results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
+            -q "<rpc><show><system><detail/></system></show></rpc>" \
+            -v "/rpc-reply/rpc/show/system/supported-max-connections[text()]"`
+        supported_max_connection=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
+
+        if [[ ${connection_count} -gt ${supported_max_connection} ]]; then
+          echo "`date -Ins` ERROR $(hostname) ${APP}: Requested scale tier of ${connection_count} is larger then supported scale tier of ${supported_max_connection}"
+          exit 1
+        fi
+
+        # Shut down message spool and message backbone on Primary and backup
+        result=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
+          -q "<rpc><service><msg-backbone><shutdown/></msg-backbone></service></rpc>"`
+        echo "`date -Ins` INFO $(hostname) ${APP}: shutting message-backbone for Hostname: $(hostname) result ${result}"
+        result=`{{ .Values.filepaths.configmap }}/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP -r 10 \
+          -q "<rpc><hardware><message-spool><shutdown/></message-spool></hardware></rpc>"`
+        echo "`date -Ins` INFO $(hostname) ${APP}: shutting message-spool for Hostname: $(hostname) result ${result}"
+
+        echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection count to ${connection_count}"
+
         echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling to ${connection_count} try ${loop_guard}"
         cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
           -q "<rpc><system><scaling><max-connections><max-connections>${connection_count}</max-connections></max-connections></scaling></system></rpc>"`
-      #cc_upgrade_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \
-      #    -q "<rpc><reload/></rpc>"`
-        if [[ -z $cc_upgrade_results ]]; then
-          echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling passed: ${cc_upgrade_results} :"
-          break
-        else
-          echo "`date -Ins` INFO $(hostname) ${APP}: Changing connection scalling failed: ${cc_upgrade_results} :"
-          sleep ${try_interval}
-        fi
-      done
-      if [ $loop_guard -eq $max_tries ]; then
-        echo "`date -Ins` INFO $(hostname) ${APP}: Failed to set connection limit to: ${connection_count}"
-      fi
+        exit 0
+
 
   post_vertical_scale_upgrade.sh: |-
       #!/bin/bash
@@ -513,14 +524,14 @@ data:
       max_tries=180
       for i in `seq 0 1`; do
         host_name="${release_name}-solace-${i}.${release_name}-solace-discovery.${namespace}.svc"
-        while [ $current_max_connection -ne $connection_count ]; do
+        while [[ $current_max_connection -ne $connection_count ]]; do
           results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://${host_name}:8080/SEMP -r 60 \
               -q "<rpc><show><system><detail/></system></show></rpc>" \
               -v "/rpc-reply/rpc/show/system/max-connections[text()]"`
           current_max_connection=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`
           ((loop_guard++))
           sleep 10
-          if [ $loop_guard -eq $max_tries ]; then
+          if [[ $loop_guard -eq $max_tries ]]; then
             echo "`date -Iseconds` ERROR $(hostname) ${APP}: Failed get correct max connection count. Current value: ${current_max-connection} Expected value: ${connection_count}"
             exit 1
           fi
@@ -529,7 +540,7 @@ data:
         /mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://${host_name}:8080/SEMP -r 60 \
           -q "<rpc><service><msg-backbone><no><shutdown/></no></msg-backbone></service></rpc>"
 
-        if [ ${i} -eq 0 ]; then
+        if [[ ${i} -eq 0 ]]; then
           role="<primary/>"
         else
           role="<backup/>"
@@ -578,9 +589,18 @@ data:
       shift $((OPTIND-1))
       [ "$1" = "--" ] && shift
       verbose=1
-      echo "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: count_search=${count_search} ,name=${name} \
-                  ,retry_count=${retry_count}, retry_interval=${retry_interval}, query=${query} \
-                  ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2
+      INPUTS=$(cat <<-END
+        count_search=${count_search}
+        name=${name}
+        retry_count=${retry_count}
+        retry_interval=${retry_interval}
+        query=${query}
+        url=${url}
+        value_search=${value_search}
+        Leftovers $@
+      END
+      )
+      echo "`date -Iseconds` INFO $(hostname) ${APP}:  ${INPUTS}" >&2
       if [[ ${url} = "" || ${name} = "" || ${password} = "" || ${query} = "" ]]; then
           echo "`date -Iseconds` ERROR $(hostname) ${APP}:  url, name, password and query are madatory fields" >&2
           echo  '<returnInfo><errorInfo>missing parameter</errorInfo></returnInfo>'
@@ -588,7 +608,7 @@ data:
         fi
       retry_guard=0
       query_success=
-      while [ ${retry_count} -ge ${retry_guard} ]; do
+      while [[ ${retry_count} -ge ${retry_guard} ]]; do
         query_response=`curl -sS -u ${name}:${password} ${url} -d "${query}"`
         ((retry_guard++))
         # Validate first char of response is "<", otherwise no hope of being valid xml
@@ -601,7 +621,7 @@ data:
         fi
       done
       if [[ -z ${query_success} ]]; then
-        echo "`date --Iseconds` ERROR $(hostname) ${APP}:  Query failed non-xml response -${query_response}-" >&2
+        echo "`date -Iseconds` ERROR $(hostname) ${APP}:  Query failed non-xml response -${query_response}-" >&2
         exit 1
       fi
       query_response_code=`echo $query_response | xmllint -xpath 'string(/rpc-reply/execute-result/@code)' -`
@@ -610,10 +630,10 @@ data:
         echo  "<returnInfo><errorInfo>query failed -${query_response_code}-</errorInfo></returnInfo>"
         exit 1
       fi
-      echo "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: Query passed ${query_response_code}" >&2
+      echo "`date -Iseconds` INFO $(hostname) ${APP}: Query passed ${query_response_code}" >&2
       if [[ ! -z $value_search ]]; then
           value_result=`echo $query_response | xmllint -xpath "string($value_search)" -`
-          echo "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: Value search $value_search returned ${value_result}" >&2
+          echo "`date -Iseconds` INFO $(hostname) ${APP}: Value search $value_search returned ${value_result}" >&2
           echo  "<returnInfo><errorInfo></errorInfo><valueSearchResult>${value_result}</valueSearchResult></returnInfo>"
           exit 0
       fi
@@ -621,7 +641,7 @@ data:
           count_line=`echo $query_response | xmllint -xpath "$count_search" -`
           count_string=`echo $count_search | cut -d '"' -f 2`
           count_result=`echo ${count_line} | tr "><" "\n" | grep -c ${count_string}`
-          echo -e "`date -Iseconds` INFO $(hostname) ${APP}: ${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
+          echo -e "`date -Iseconds` INFO $(hostname) ${APP}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2
           echo  "<returnInfo><errorInfo></errorInfo><countSearchResult>${count_result}</countSearchResult></returnInfo>" 
           exit 0
       fi