From 539357732934f0d314b870581b3c435ada8c6b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Czoma?= Date: Fri, 4 Oct 2019 12:49:15 -0400 Subject: [PATCH] Template fix for minishift (#32) * Update Helm version to 2.14, in sync with the solace kubernetes quickstart * Updated templates to match latest solace kubernetes quickstart and to work with MiniShift * Doc enhancements for Minishift --- readme.md | 34 ++- scripts/deployHelm.sh | 2 +- scripts/templates/deployHelmServer.yaml | 2 +- templates/messagebroker_ha_template.yaml | 284 +++++++++++------- .../messagebroker_singlenode_template.yaml | 141 +++++---- 5 files changed, 286 insertions(+), 177 deletions(-) diff --git a/readme.md b/readme.md index 5cd446d..7c32b47 100644 --- a/readme.md +++ b/readme.md @@ -130,11 +130,13 @@ cd ~/workspace/solace-openshift-quickstart/scripts * Use the ‘prepareProject.sh’ script the Solace OpenShift QuickStart to create and configure an OpenShift project that meets requirements of the message broker deployment: ``` +# If using Minishift start with this command: oc login -u system:admin cd ~/workspace/solace-openshift-quickstart/scripts sudo ./prepareProject.sh solace-pubsub # adjust your project name as needed here and in subsequent commands +# In Minishift return to admin user: oc login -u admin ``` -> Note: If using MiniShift on Windows use the command without `sudo`. If necessary, as a workaround, run just this command with logging in as "system:admin" before using `oc login -u system:admin`, then login afterward to the normal "admin" user. Running as the normal "admin" user provides the closest experience of other OpenShift deployments. +> Note: The purpose of using `sudo` is to elevate `admin` user to `system:admin`. This is not available when using MiniShift and apply above workaround for just this step. ### Step 5: Optional: Load the message broker (Docker image) to your Docker Registry @@ -158,17 +160,18 @@ Deployment scripts will pull the Solace message broker image from a [Docker regi Options include: - * You can choose to use [OpenShift's Docker registry.](https://docs.openshift.com/container-platform/3.10/install_config/registry/deploy_registry_existing_clusters.html ) + * You can choose to use [OpenShift's Docker registry.](https://docs.openshift.com/container-platform/3.10/install_config/registry/deploy_registry_existing_clusters.html ). For MiniShift a simple option is to use the [Minishift Docker daemon](//docs.okd.io/latest/minishift/using/docker-daemon.html). * **(Optional / ECR)** You can utilize the AWS Elastic Container Registry (ECR) to host the message broker Docker image. For more information, refer to [Amazon Elastic Container Registry](https://aws.amazon.com/ecr/ ). If you are using ECR as your Docker registry then you must add the ECR login credentials (as an OpenShift secret) to your message broker HA deployment. This project contains a helper script to execute this step: +```shell + # Required if using ECR for Docker registry + cd ~/workspace/solace-openshift-quickstart/scripts + sudo su + aws configure # provide AWS config for root; provide your key ID and key, leave the rest to None. + ./addECRsecret.sh solace-pubsub # adjust your project name as needed ``` -# Required if using ECR for Docker registry -sudo su -aws configure # provide AWS config for root -cd ~/workspace/solace-openshift-quickstart/scripts -./addECRsecret.sh solace-pubsub # adjust your project name as needed -``` + Here is an outline of the additional steps required if loading an image to ECR: * Copy the Solace Docker image location and download the image archive locally using the `wget ` command. @@ -232,10 +235,23 @@ watch oc get pods --show-labels ``` echo -n 'strong@dminPw!' | base64 ``` +3. Switch to the templates directory: +``` +oc project solace-pubsub # adjust your project name as needed +cd ~/workspace/solace-openshift-quickstart/templates +``` **Deploy the message broker:** -You can deploy the message broker in either a single-node or high-availability configuration: +You can deploy the message broker in either a single-node or high-availability configuration. + +Note: DOCKER_REGISTRY_URL and MESSAGEBROKER_IMAGE_TAG default to `solace/solace-pubsub-standard` and `latest`, MESSAGEBROKER_STORAGE_SIZE defaults to 30Gi. + +The template by default provides for a small-footprint Solace message broker deployment deployable in MiniShift. Adjust `export system_scaling_maxconnectioncount` in the template for higher scaling but ensure adequate resources are available to the pod(s). Refer to the [System Requirements in the Solace documentation](//docs.solace.com/Configuring-and-Managing/SW-Broker-Specific-Config/Scaling-Tier-Resources.htm). + +Also note that if a deployment failed and then deleted using `oc delete -f`, ensure to delete any remaining PVCs. Failing to do so and retrying using the same deployment name will result in an already used PV volume mounted and the pod(s) may not come up. + +The template by default provides for a small-footprint Solace message broker deployment deployable in MiniShift. Adjust `export system_scaling_maxconnectioncount` in the template for higher scaling but ensure adequate resources are available to the pod(s). Refer to the [System Requirements in the Solace documentation](//docs.solace.com/Configuring-and-Managing/SW-Broker-Specific-Config/Scaling-Tier-Resources.htm). * For a **Single-Node** configuration: * Process the Solace 'Single Node' OpenShift template to deploy the message broker in a single-node configuration. Specify values for the DOCKER_REGISTRY_URL, MESSAGEBROKER_IMAGE_TAG, MESSAGEBROKER_STORAGE_SIZE, and MESSAGEBROKER_ADMIN_PASSWORD parameters: diff --git a/scripts/deployHelm.sh b/scripts/deployHelm.sh index 80a99c4..c45b818 100755 --- a/scripts/deployHelm.sh +++ b/scripts/deployHelm.sh @@ -13,7 +13,7 @@ # . ./deployHelm.sh server # TILLER_PROJECT=tiller -HELM_VERSION=2.9.1 +HELM_VERSION=2.14.0 function helmVersion() { which helm &> /dev/null diff --git a/scripts/templates/deployHelmServer.yaml b/scripts/templates/deployHelmServer.yaml index 41a0dd4..90ac078 100644 --- a/scripts/templates/deployHelmServer.yaml +++ b/scripts/templates/deployHelmServer.yaml @@ -57,7 +57,7 @@ objects: spec: containers: - name: tiller - image: gcr.io/kubernetes-helm/tiller:v2.9.1 + image: gcr.io/kubernetes-helm/tiller:v2.14.0 env: - name: TILLER_NAMESPACE valueFrom: diff --git a/templates/messagebroker_ha_template.yaml b/templates/messagebroker_ha_template.yaml index 652a460..b8740a1 100644 --- a/templates/messagebroker_ha_template.yaml +++ b/templates/messagebroker_ha_template.yaml @@ -14,7 +14,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace type: Opaque data: @@ -27,7 +27,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace data: init.sh: |- @@ -38,8 +38,8 @@ objects: export service_webtransport_port='60080' export service_webtransport_tlsport='60443' export service_semp_tlsport='60943' - export logging_debug_output=stdout - export system_scaling_maxconnectioncount="1000" + export logging_debug_output=all + export system_scaling_maxconnectioncount="100" # [TODO] KBARR not using correct method of finding ordinal until we bump min Kubernetes release above 1.8.1 # https://github.com/kubernetes/kubernetes/issues/40651 # node_ordinal=$(STATEFULSET_ORDINAL) @@ -88,19 +88,20 @@ objects: # node_ordinal=$(STATEFULSET_ORDINAL) IFS='-' read -ra host_array <<< $(hostname) node_ordinal=${host_array[-1]} + echo "`date` INFO: ${APP}-node ordinal: ${node_ordinal}" password=`cat /mnt/disks/secrets/username_admin_password` - loop_guard=30 + loop_guard=60 pause=10 count=0 resync_step="" role="" #exclude monitor node from config-sync check if [ "${node_ordinal}" != "2" ]; then - echo "`date` INFO: ${APP}-Determine if Active or Backup role to know which Virtual router to check" while [ ${count} -lt ${loop_guard} ]; do role_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "" \ + -q "" \ -v "/rpc-reply/rpc/show/redundancy/active-standby-role[text()]"` + run_time=$((${count} * ${pause})) case "`echo ${role_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -`" in "Primary") role="primary" @@ -112,81 +113,78 @@ objects: ;; esac ((count++)) - echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Redundancy not yet up" + echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Management API not yet accessible" sleep ${pause} done if [ ${count} -eq ${loop_guard} ]; then - echo "`date` ERROR: ${APP}-Solace redundancy group never came up" | tee /dev/stderr + echo "`date` ERROR: ${APP}-Solace Management API never came up" >&2 exit 1 fi count=0 - echo "`date` INFO: ${APP}-Active or Backup role is ${role}" + echo "`date` INFO: ${APP}-Management API is up, determined that this node's active-standby role is: ${role}" while [ ${count} -lt ${loop_guard} ]; do online_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "" \ + -q "" \ -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${role}/status/activity[text()]"` local_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` - echo "`date` INFO: ${APP}-Local activity state is: ${local_activity}" run_time=$((${count} * ${pause})) case "${local_activity}" in "Local Active") - echo "`date` INFO: ${APP}-Redundancy is up locally Active, after ${run_time} seconds" - echo "`date` INFO: "We should only be here on new cluster create, if not likely a bug" - echo "`date` INFO: " will issue a assert master to get back into sync" + echo "`date` INFO: ${APP}-Node activity status is Local Active, after ${run_time} seconds" + # We should only be here on new cluster create, if not likely a bug + # Need to issue assert master to get back into sync" resync_step="assert-master" break ;; "Mate Active") - echo "`date` INFO: ${APP}-Redundancy is up mate Active, after ${run_time} seconds" - echo "`date` INFO: "This is normal state if we are backup or recreated later on" - echo "`date` INFO: " will issue a resync master to get back into sync" + echo "`date` INFO: ${APP}-Node activity status is Mate Active, after ${run_time} seconds" + # This is normal state if we are backup or recreated later on + # will issue a resync master to get back into sync resync_step="resync-master" break ;; esac ((count++)) - echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Redundancy not yet up" + echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Local activity state is: ${local_activity}" sleep ${pause} done if [ ${count} -eq ${loop_guard} ]; then - echo "`date` ERROR: ${APP}-Solace redundancy group never came up" | tee /dev/stderr + echo "`date` ERROR: ${APP}-Local activity state never become Local Active or Mate Active" >&2 exit 1 fi - # If we need to assert master, then we need to wait for mate to reconsile - + # If we need to assert master, then we need to wait for mate to reconcile if [ "${resync_step}" = "assert-master" ]; then count=0 - echo "`date` INFO: ${APP}-Wait for mate to be 'Standby'" + echo "`date` INFO: ${APP}-Waiting for mate activity state to be 'Standby'" while [ ${count} -lt ${loop_guard} ]; do online_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "" \ + -q "" \ -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${role}/status/detail/priority-reported-by-mate/summary[text()]"` mate_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` - echo "`date` INFO: ${APP}-Mate activity state is: ${mate_activity}" run_time=$((${count} * ${pause})) case "${mate_activity}" in "Standby") - echo "`date` INFO: ${APP}-Redundancy is up end-to-end, Local Active Mate Standby, after ${run_time} seconds" + echo "`date` INFO: ${APP}-Activity state reported by mate is Standby, after ${run_time} seconds" break ;; esac ((count++)) - echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Mate not yet in sync" + echo "`date` INFO: ${APP}-Waited ${run_time} seconds, Mate activity state is: ${mate_activity}, not yet in sync" sleep ${pause} done if [ ${count} -eq ${loop_guard} ]; then - echo "`date` ERROR: ${APP}-Solace redundancy group never came up" | tee /dev/stderr + echo "`date` ERROR: ${APP}-Mate not in sync, never reached Standby" >&2 exit 1 fi fi # if assert-master - # If only needs to do is resync master, can issue cammand and exit. + # Now can issue {resync_step} command and exit. /mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "<${resync_step}>" + -q "<${resync_step}>" /mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "<${resync_step}>default" - echo "`date` INFO: ${APP}-Solace Message Broker bringup complete" + -q "<${resync_step}>default" + echo "`date` INFO: ${APP}-Solace message broker bringup complete for this node." fi # if not monitor exit 0 @@ -199,42 +197,116 @@ objects: #Prevent overdriving Kubernetes infra, don't set activity state to same as previous state previous_state=`cat $3` if [ "${2}" = "${previous_state}" ]; then - echo "`date` INFO: ${APP}-Current and Previous state match, not updating label" + #echo "`date` INFO: ${APP}-Current and Previous state match, not updating label" + : else echo ${2} > ${3} echo "[{\"op\": \"add\", \"path\": \"/metadata/labels/${1}\", \"value\": \"${2}\" }]" > /tmp/patch_label.json + K8S=https://kubernetes.default.svc.cluster.local:$KUBERNETES_SERVICE_PORT KUBE_TOKEN=$(&2 + exit 1 + fi fi } # note that there are no re-tries here, if check fails the return not ready. APP=`basename "$0"` - version=${1} - password=`cat /mnt/disks/secrets/username_admin_password` state_file=/tmp/activity_state - if [ ! -f ${state_file} ]; then - echo "State file not found, creating!" - echo "false" > ${state_file} + if [ ! -f ${state_file} ]; then # State file not found, creating + echo "unknown" > ${state_file} fi - echo "`date` INFO: ${APP}-node ordinal: ${node_ordinal}" + # HA config + version=${1} IFS='-' read -ra host_array <<< $(hostname) node_ordinal=${host_array[-1]} + password=`cat /mnt/disks/secrets/username_admin_password` + + # For upgrade purposes, ensure redundancy is up only when the pod is started + redundacycheck_file=/tmp/redundacycheck + if [ ! -f ${redundacycheck_file} ]; then + # First check all nodes are online + results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ + -q "" \ + -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"` + if [[ ${results} != *""* ]]; then + errorinfo=`echo ${results} | xmllint -xpath "string(returnInfo/errorInfo)" -` || errorinfo= + echo "`date` INFO: ${APP}-Waiting for valid server status response, got ${errorinfo}" + exit 1 + fi + nr_node_results=`echo ${results} | xmllint -xpath "string(returnInfo/countSearchResult)" -` + if [ "$nr_node_results" -ne "3" ]; then + echo "`date` INFO: ${APP}-Waiting for all 3 nodes online, got ${nr_node_results}" + exit 1 + fi + # Then for each node determine the ip address and check redundancy. Note: id starts here from 1 and not 0. + for id in 1 2 3; do + results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ + -q "" \ + -v "//ip-address[$id]"` + node_ip_address=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` + results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://$node_ip_address:8080/SEMP \ + -q "" \ + -v "/rpc-reply/rpc/show/redundancy/redundancy-status"` + redundancystatus_results=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` + if [ "${redundancystatus_results}" != "Up" ]; then + echo "`date` INFO: ${APP}-Redundancy state is not yet up." + exit 1 + fi + done + # Additionally check config-sync status for non-monitoring nodes + if [ "${node_ordinal}" != "2" ]; then + results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ + -q "" \ + -v "/rpc-reply/rpc/show/config-sync/status/oper-status"` + confsyncstatus_results=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` + if [ "${confsyncstatus_results}" != "Up" ]; then + echo "`date` INFO: ${APP}-Config-sync state is not yet up." + exit 1 + fi + fi + # Then for each node check that they report 3 Consul voters. + for id in 1 2 3; do + results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ + -q "" \ + -v "//ip-address[$id]"` + node_ip_address=`echo ${results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` + nr_voter_results=`curl --unix-socket /var/run/solace/consul -s http://$node_ip_address:8500/v1/operator/raft/configuration || echo {} | python -m json.tool | grep Voter | grep true | wc -l` + if [ $nr_voter_results -ne 3 ]; then + # For backwards compatibility - will revise. + nr_voter_results=`curl --unix-socket /var/run/consul -s http://$node_ip_address:8500/v1/operator/raft/configuration | python -m json.tool | grep Voter | grep true | wc -l` + if [ $nr_voter_results -ne 3 ]; then + echo "`date` INFO: ${APP}-Waiting for all 3 Consul voters to be present for node $node_ip_address, got ${nr_voter_results}" + exit 1 + fi + fi + done + # Creating marker - important that after initial startup pod keeps being ready to serve traffic during failover while redundancy is down + echo "true" > ${redundacycheck_file} + fi + if [ "${node_ordinal}" = "2" ]; then - echo "`date` INFO: ${APP}-Monitor node ready check" - # Note that when dealing with Monitor, only need to be concerned and readiness response. - # active label will always be "false" - echo "`date` INFO: ${APP}-For monitor node just check for 3 online nodes in group" + # For monitor node just check for 3 online nodes in group; active label will always be "false" role_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "" \ + -q "" \ -c "/rpc-reply/rpc/show/redundancy/group-node/status[text() = \"Online\"]"` - if [ `echo ${role_results} | xmllint -xpath "string(returnInfo/countSearchResult)" -` -eq 3 ]; then + if [[ ${role_results} != *""* ]]; then + errorinfo=`echo ${results} | xmllint -xpath "string(returnInfo/errorInfo)" -` || errorinfo= + echo "`date` INFO: ${APP}-Waiting for valid server status response, got ${errorinfo}" + exit 1 + fi + nodes_online=`echo ${role_results} | xmllint -xpath "string(returnInfo/countSearchResult)" -` + if [ "$nodes_online" -eq "3" ]; then echo "`date` INFO: ${APP}-Monitor node is redundancy ready" exit 0 else - echo "`date` INFO: ${APP}-Monitor node is not redundancy ready, result: ${role_results}" + echo "`date` INFO: ${APP}-Monitor node is not redundancy ready, ${nodes_online} of 3 nodes online" exit 1 fi fi # End Monitor Node @@ -250,10 +322,10 @@ objects: "503") set_label "active" "false" $state_file if (( "$version" < 7 )); then - echo "`date` INFO: ${APP}-Message Router is Healthy and not Active, this is K8S 1.6 ready" + echo "`date` INFO: ${APP}-Message Router is Healthy but not Active, this is K8S 1.6 ready" exit 0 else - echo "`date` INFO: ${APP}-Message Router is Healthy and not Active, further check required" + echo "`date` INFO: ${APP}-Message Router is Healthy but not Active, further check required" fi ;; "") @@ -262,7 +334,7 @@ objects: exit 1 esac - echo "`date` INFO: ${APP}-Checking if Message Router is Standby" + # Checking if Message Router is Standby case "${node_ordinal}" in "0") config_role="primary" @@ -272,27 +344,26 @@ objects: ;; esac online_results=`/mnt/disks/solace/semp_query.sh -n admin -p ${password} -u http://localhost:8080/SEMP \ - -q "" \ + -q "" \ -v "/rpc-reply/rpc/show/redundancy/virtual-routers/${config_role}/status/activity[text()]"` local_activity=`echo ${online_results} | xmllint -xpath "string(returnInfo/valueSearchResult)" -` - echo "`date` INFO: ${APP}-Local activity state is: ${local_activity}" case "${local_activity}" in "Local Active") - echo "`date` INFO: ${APP}-Redundancy is up locally Active" + # Redundancy is up and node is locally Active" # Set active label to "true" set_label "active" "true" $state_file # Pass readiness check exit 0 ;; "Mate Active") - echo "`date` INFO: ${APP}-Redundancy is up mate Active" + # Redundancy is up and node is mate Active" # Set active label to "false" set_label "active" "false" $state_file # Pass readiness check exit 0 ;; *) - echo "`date` WARN: ${APP}-Redundancy not up or not responding, failing readiness check" + echo "`date` WARN: ${APP}-Redundancy not up or not responding, failing readiness check. Local activity state is: ${local_activity}" # Set active label to "false" set_label "active" "false" $state_file # Fail readiness check @@ -332,30 +403,32 @@ objects: shift $((OPTIND-1)) [ "$1" = "--" ] && shift verbose=1 - echo "`date` INFO: ${APP}-${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \ - ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2 + #echo "`date` INFO: ${APP}-${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \ + # ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2 if [[ ${url} = "" || ${name} = "" || ${password} = "" || ${query} = "" ]]; then - echo "`date` ERROR: ${APP}-${script_name}: url, name, password and query are madatory fields" >&2 - echo 'missing parameter' - exit 1 - fi + echo "`date` ERROR: ${APP}-${script_name}: url, name, password and query are madatory fields" >&2 + echo 'missing parameter' + exit 1 + fi + if [ `curl --write-out '%{http_code}' --silent --output /dev/null -u ${name}:${password} ${url}` != "200" ] ; then + echo "management host is not responding" + exit 1 + fi query_response=`curl -sS -u ${name}:${password} ${url} -d "${query}"` # Validate first char of response is "<", otherwise no hope of being valid xml if [[ ${query_response:0:1} != "<" ]] ; then - echo "`date` ERROR: ${APP}-${script_name}: Query failed, non-xml response -${query_response}-" >&2 + echo "no valid xml returned" exit 1 fi query_response_code=`echo $query_response | xmllint -xpath 'string(/rpc-reply/execute-result/@code)' -` if [[ -z ${query_response_code} && ${query_response_code} != "ok" ]]; then - echo "`date` ERROR: ${APP}-${script_name}: Query failed, bad return code -${query_response}-" >&2 echo "query failed -${query_response_code}-" exit 1 fi - echo "`date` INFO: ${APP}-${script_name}: Query passed ${query_response_code}" >&2 + #echo "`date` INFO: ${APP}-${script_name}: query passed ${query_response_code}" >&2 if [[ ! -z $value_search ]]; then value_result=`echo $query_response | xmllint -xpath "string($value_search)" -` - echo "`date` INFO: ${APP}-${script_name}: Value search $value_search returned ${value_result}" >&2 echo "${value_result}" exit 0 fi @@ -363,19 +436,20 @@ objects: count_line=`echo $query_response | xmllint -xpath "$count_search" -` count_string=`echo $count_search | cut -d '"' -f 2` count_result=`echo ${count_line} | tr "><" "\n" | grep -c ${count_string}` - echo -e "`date` INFO: ${APP}-${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2 echo "${count_result}" exit 0 fi -- kind: StorageClass - apiVersion: storage.k8s.io/v1 - metadata: - name: "${DEPLOYMENT_NAME}-standard" - provisioner: kubernetes.io/aws-ebs - parameters: - #Default don't set zone or zones, allow k8s to strip across active zones - type: gp2 +# Uncomment if using provider-specific StorageClass +# Also uncomment further down at volumeClaimTemplates +# This is an example for aws +#- kind: StorageClass +# apiVersion: storage.k8s.io/v1 +# metadata: +# name: "${DEPLOYMENT_NAME}-standard" +# provisioner: kubernetes.io/aws-ebs +# parameters: +# type: gp2 - kind: Service apiVersion: v1 @@ -384,7 +458,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace annotations: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" @@ -404,7 +478,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace #end gcp spec: type: LoadBalancer @@ -414,13 +488,21 @@ objects: protocol: TCP name: ssh - port: 8080 - targetPort: 8080 + targetPort: protocol: TCP name: semp - port: 55555 - targetPort: 55555 + targetPort: protocol: TCP name: smf + - port: 55003 + targetPort: + protocol: TCP + name: smfcompr + - port: 55443 + targetPort: + protocol: TCP + name: smftls - port: 943 targetPort: 60943 protocol: TCP @@ -444,7 +526,7 @@ objects: name: "${DEPLOYMENT_NAME}-solace" labels: app: solace - chart: solace-0.3.0 + chart: solace-1.0.1 release: "${DEPLOYMENT_NAME}" heritage: Tiller spec: @@ -465,11 +547,11 @@ objects: imagePullPolicy: IfNotPresent resources: requests: - cpu: "1.2" - memory: 6.0Gi + cpu: "0.2" + memory: 0.8Gi limits: - cpu: "2" - memory: 7.5Gi + cpu: "1" + memory: 2Gi livenessProbe: tcpSocket: port: 8080 @@ -488,20 +570,6 @@ objects: add: - IPC_LOCK - SYS_NICE - - SETPCAP - - MKNOD - - AUDIT_WRITE - - CHOWN - - NET_RAW - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - SETGID - - SETUID - - NET_BIND_SERVICE - - SYS_CHROOT - - SETFCAP env: - name: STATEFULSET_NAME value: "${DEPLOYMENT_NAME}-solace" @@ -545,9 +613,13 @@ objects: - name: data mountPath: /usr/sw/adb subPath: adb + - name: data + mountPath: /var/lib/solace/diags + subPath: diags + # only mount when not using nfs - name: data mountPath: /usr/sw/internalSpool/softAdb - subPath: softAdb + subPath: softAdb #end !nfs ports: - containerPort: 2222 protocol: TCP @@ -555,6 +627,10 @@ objects: protocol: TCP - containerPort: 55555 protocol: TCP + - containerPort: 55003 + protocol: TCP + - containerPort: 55443 + protocol: TCP - containerPort: 60943 protocol: TCP - containerPort: 60080 @@ -577,7 +653,9 @@ objects: - metadata: name: data spec: - storageClassName: "${DEPLOYMENT_NAME}-standard" + # Uncomment if using provider-specific StorageClass + # Also uncomment StorageClass definition above + # storageClassName: "${DEPLOYMENT_NAME}-standard" accessModes: [ "ReadWriteOnce" ] resources: requests: @@ -594,12 +672,12 @@ parameters: - name: DOCKER_REGISTRY_URL displayName: Docker Registry URL description: The Docker registry URL for the registry containing the Solace Message Broker docker image - value: docker_registry_url + value: solace/solace-pubsub-standard required: true - name: MESSAGEBROKER_IMAGE_TAG displayName: Solace Message Broker Docker Image Tag description: The Docker image tag for the Solace Message Broker docker image from your Docker registry - value: solace_messagebroker_image_tag + value: latest required: true - name: MESSAGEBROKER_ADMIN_PASSWORD displayName: Base64 encoded password for Solace username 'admin' diff --git a/templates/messagebroker_singlenode_template.yaml b/templates/messagebroker_singlenode_template.yaml index ea6a758..e70378c 100644 --- a/templates/messagebroker_singlenode_template.yaml +++ b/templates/messagebroker_singlenode_template.yaml @@ -14,7 +14,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace type: Opaque data: @@ -27,7 +27,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace data: init.sh: |- @@ -38,8 +38,8 @@ objects: export service_webtransport_port='60080' export service_webtransport_tlsport='60443' export service_semp_tlsport='60943' - export logging_debug_output=stdout - export system_scaling_maxconnectioncount="1000" + export logging_debug_output=all + export system_scaling_maxconnectioncount="100" config-sync-check.sh: |- #!/bin/bash @@ -54,27 +54,32 @@ objects: #Prevent overdriving Kubernetes infra, don't set activity state to same as previous state previous_state=`cat $3` if [ "${2}" = "${previous_state}" ]; then - echo "`date` INFO: ${APP}-Current and Previous state match, not updating label" + #echo "`date` INFO: ${APP}-Current and Previous state match, not updating label" + : else echo ${2} > ${3} echo "[{\"op\": \"add\", \"path\": \"/metadata/labels/${1}\", \"value\": \"${2}\" }]" > /tmp/patch_label.json + K8S=https://kubernetes.default.svc.cluster.local:$KUBERNETES_SERVICE_PORT KUBE_TOKEN=$(&2 + exit 1 + fi fi } # note that there are no re-tries here, if check fails the return not ready. APP=`basename "$0"` - - password=`cat /mnt/disks/secrets/username_admin_password` state_file=/tmp/activity_state - if [ ! -f ${state_file} ]; then - echo "State file not found, creating!" - echo "false" > ${state_file} + if [ ! -f ${state_file} ]; then # State file not found, creating + echo "unknown" > ${state_file} fi - + # nonHA config health_result=`curl -s -o /dev/null -w "%{http_code}" http://localhost:5550/health-check/guaranteed-active` case "${health_result}" in @@ -126,50 +131,53 @@ objects: shift $((OPTIND-1)) [ "$1" = "--" ] && shift verbose=1 - echo "`date` INFO: ${APP}-${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \ - ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2 + #echo "`date` INFO: ${APP}-${script_name}: count_search=${count_search} ,name=${name} ,password=xxx query=${query} \ + # ,url=${url} ,value_search=${value_search} ,Leftovers: $@" >&2 if [[ ${url} = "" || ${name} = "" || ${password} = "" || ${query} = "" ]]; then - echo "`date` ERROR: ${APP}-${script_name}: url, name, password and query are madatory fields" >&2 - echo '{"errorInfo":"missing parameter"}' - exit 1 - fi + echo "`date` ERROR: ${APP}-${script_name}: url, name, password and query are madatory fields" >&2 + echo 'missing parameter' + exit 1 + fi + if [ `curl --write-out '%{http_code}' --silent --output /dev/null -u ${name}:${password} ${url}` != "200" ] ; then + echo "management host is not responding" + exit 1 + fi query_response=`curl -sS -u ${name}:${password} ${url} -d "${query}"` # Validate first char of response is "<", otherwise no hope of being valid xml if [[ ${query_response:0:1} != "<" ]] ; then - echo "`date` ERROR: ${APP}-${script_name}: Query failed, non-xml response -${query_response}-" >&2 + echo "no valid xml returned" exit 1 fi query_response_code=`echo $query_response | xmllint -xpath 'string(/rpc-reply/execute-result/@code)' -` if [[ -z ${query_response_code} && ${query_response_code} != "ok" ]]; then - echo "`date` ERROR: ${APP}-${script_name}: Query failed, bad return code -${query_response}-" >&2 - echo "{\"errorInfo\":\"query failed -${query_response_code}-\"}" + echo "query failed -${query_response_code}-" exit 1 fi - echo "`date` INFO: ${APP}-${script_name}: Query passed ${query_response_code}" >&2 + #echo "`date` INFO: ${APP}-${script_name}: query passed ${query_response_code}" >&2 if [[ ! -z $value_search ]]; then value_result=`echo $query_response | xmllint -xpath "string($value_search)" -` - echo "`date` INFO: ${APP}-${script_name}: Value search $value_search returned ${value_result}" >&2 - echo "{\"errorInfo\":\"\",\"valueSearchResult\":\"${value_result}\"}" + echo "${value_result}" exit 0 fi if [[ ! -z $count_search ]]; then count_line=`echo $query_response | xmllint -xpath "$count_search" -` count_string=`echo $count_search | cut -d '"' -f 2` count_result=`echo ${count_line} | tr "><" "\n" | grep -c ${count_string}` - echo -e "`date` INFO: ${APP}-${script_name}: \n\t count search: $count_search \n\t count_line: ${count_line} \n\t count_string: ${count_string} \n\t count_result: ${count_result}" >&2 - echo "{\"errorInfo\":\"\",\"countSearchResult\":${count_result}}" + echo "${count_result}" exit 0 fi -- kind: StorageClass - apiVersion: storage.k8s.io/v1 - metadata: - name: "${DEPLOYMENT_NAME}-standard" - provisioner: kubernetes.io/aws-ebs - parameters: - #Default don't set zone or zones, allow k8s to strip across active zones - type: gp2 +# Uncomment if using provider-specific StorageClass +# Also uncomment further down at volumeClaimTemplates +# This is an example for aws +#- kind: StorageClass +# apiVersion: storage.k8s.io/v1 +# metadata: +# name: "${DEPLOYMENT_NAME}-standard" +# provisioner: kubernetes.io/aws-ebs +# parameters: +# type: gp2 - kind: Service apiVersion: v1 @@ -178,7 +186,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace annotations: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" @@ -198,7 +206,7 @@ objects: labels: heritage: Tiller release: "${DEPLOYMENT_NAME}" - chart: solace-0.3.0 + chart: solace-1.0.1 app: solace #end gcp spec: type: LoadBalancer @@ -208,13 +216,21 @@ objects: protocol: TCP name: ssh - port: 8080 - targetPort: 8080 + targetPort: protocol: TCP name: semp - port: 55555 - targetPort: 55555 + targetPort: protocol: TCP name: smf + - port: 55003 + targetPort: + protocol: TCP + name: smfcompr + - port: 55443 + targetPort: + protocol: TCP + name: smftls - port: 943 targetPort: 60943 protocol: TCP @@ -238,12 +254,15 @@ objects: name: "${DEPLOYMENT_NAME}-solace" labels: app: solace - chart: solace-0.3.0 + chart: solace-1.0.1 release: "${DEPLOYMENT_NAME}" heritage: Tiller spec: serviceName: "${DEPLOYMENT_NAME}-solace-discovery" replicas: 1 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate template: metadata: labels: @@ -256,11 +275,11 @@ objects: imagePullPolicy: IfNotPresent resources: requests: - cpu: "1.2" - memory: 6.0Gi + cpu: "0.2" + memory: 0.8Gi limits: - cpu: "2" - memory: 7.5Gi + cpu: "1" + memory: 2Gi livenessProbe: tcpSocket: port: 8080 @@ -279,20 +298,6 @@ objects: add: - IPC_LOCK - SYS_NICE - - SETPCAP - - MKNOD - - AUDIT_WRITE - - CHOWN - - NET_RAW - - DAC_OVERRIDE - - FOWNER - - FSETID - - KILL - - SETGID - - SETUID - - NET_BIND_SERVICE - - SYS_CHROOT - - SETFCAP env: - name: STATEFULSET_NAME value: "${DEPLOYMENT_NAME}-solace" @@ -336,9 +341,13 @@ objects: - name: data mountPath: /usr/sw/adb subPath: adb + - name: data + mountPath: /var/lib/solace/diags + subPath: diags + # only mount when not using nfs - name: data mountPath: /usr/sw/internalSpool/softAdb - subPath: softAdb + subPath: softAdb #end !nfs ports: - containerPort: 2222 protocol: TCP @@ -346,6 +355,10 @@ objects: protocol: TCP - containerPort: 55555 protocol: TCP + - containerPort: 55003 + protocol: TCP + - containerPort: 55443 + protocol: TCP - containerPort: 60943 protocol: TCP - containerPort: 60080 @@ -368,7 +381,9 @@ objects: - metadata: name: data spec: - storageClassName: "${DEPLOYMENT_NAME}-standard" + # Uncomment if using provider-specific StorageClass + # Also uncomment StorageClass definition above + # storageClassName: "${DEPLOYMENT_NAME}-standard" accessModes: [ "ReadWriteOnce" ] resources: requests: @@ -385,12 +400,12 @@ parameters: - name: DOCKER_REGISTRY_URL displayName: Docker Registry URL description: The Docker registry URL for the registry containing the Solace Message Broker docker image - value: docker_registry_url + value: solace/solace-pubsub-standard required: true - name: MESSAGEBROKER_IMAGE_TAG displayName: Solace Message Broker Docker Image Tag description: The Docker image tag for the Solace Message Broker docker image from your Docker registry - value: solace_messagebroker_image_tag + value: latest required: true - name: MESSAGEBROKER_ADMIN_PASSWORD displayName: Base64 encoded password for Solace username 'admin'