diff --git a/config/Makefile b/config/Makefile index 595050382..deb33c1fb 100644 --- a/config/Makefile +++ b/config/Makefile @@ -15,7 +15,7 @@ detect-change: materialize echo "Please review the diffs below:\n\n"; \ echo "$$diff_output"; \ echo "\n\n===================================================="; \ - echo "\n\nOnce you reviewed the changes and consider them meaningful, commit them."; \ + echo "\n\nOnce you reviewed the changes and consider them meaningful, update them by running "make -C config/ materialize" and commit your changes."; \ echo "\n\n===================================================="; \ exit 1; \ else \ diff --git a/config/config.msft.yaml b/config/config.msft.yaml index 202cdb473..6124c724b 100644 --- a/config/config.msft.yaml +++ b/config/config.msft.yaml @@ -14,7 +14,6 @@ defaults: # General AKS config kubernetesVersion: 1.30.6 - istioVersion: "asm-1-22" vnetAddressPrefix: "10.128.0.0/14" subnetPrefix: "10.128.8.0/21" podSubnetPrefix: "10.128.64.0/18" @@ -31,6 +30,11 @@ defaults: etcd: kvName: arohcp-etcd-{{ .ctx.regionShort }} kvSoftDelete: true + istio: + istioctlVersion: "1.24.1" + tag: "prod-stable" + targetVersion: "asm-1-23" + versions: "asm-1-22,asm-1-23" # MGMT cluster specifics mgmt: diff --git a/config/config.schema.json b/config/config.schema.json index 071ec9dab..fddcc29f0 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -323,9 +323,6 @@ "rg" ] }, - "istioVersion": { - "type": "string" - }, "kubernetesVersion": { "type": "string" }, @@ -672,7 +669,30 @@ }, "subscription": { "type": "string" - } + }, + "istio": { + "type": "object", + "properties": { + "istioctlVersion": { + "description": "Latest version of istioctl. This is used to download the istioctl", + "type": "string" + }, + "tag": { + "description": "The revision tag used in istioctl", + "type": "string" + }, + "versions": { + "description": "The current istio version in the AKS cluster as CSV, will need to change to array once tooling is ready", + "type": "string", + "pattern": "^[a-zA-Z0-9-]+(,[a-zA-Z0-9-]+)*$" + }, + "targetVersion": { + "description": "The target istio version that will be updated to", + "type": "string", + "pattern": "^[a-zA-Z0-9-]+$" + } + } + } }, "additionalProperties": false, "required": [ @@ -703,7 +723,6 @@ "hypershift", "hypershiftOperator", "imageSync", - "istioVersion", "kubernetesVersion", "maestro", "mgmt", diff --git a/config/config.yaml b/config/config.yaml index 52fd074f3..162547c7e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -12,7 +12,6 @@ defaults: # General AKS config kubernetesVersion: 1.30.6 - istioVersion: "asm-1-22" vnetAddressPrefix: "10.128.0.0/14" subnetPrefix: "10.128.8.0/21" podSubnetPrefix: "10.128.64.0/18" @@ -29,6 +28,11 @@ defaults: etcd: kvName: arohcp-etcd-{{ .ctx.regionShort }} kvSoftDelete: true + istio: + istioctlVersion: "1.23.1" + tag: "prod-stable" + targetVersion: "asm-1-22" + versions: "asm-1-22" # MGMT cluster specifics mgmt: diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index 2b70860ec..b7cc8711f 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -77,7 +77,6 @@ }, "rg": "hcp-underlay-westus3-imagesync-dev" }, - "istioVersion": "asm-1-22", "kubernetesVersion": "1.30.6", "maestro": { "certDomain": "selfsigned.maestro.keyvault.azure.com", @@ -159,6 +158,12 @@ "kvName": "arohcp-etcd-cspr", "kvSoftDelete": false }, + "istio": { + "istioctlVersion": "1.23.1", + "tag": "prod-stable", + "targetVersion": "asm-1-22", + "versions": "asm-1-22" + }, "rg": "hcp-underlay-cspr-svc", "subscription": "ARO Hosted Control Planes (EA Subscription 1)", "userAgentPool": { diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index 0220889b0..9322d9317 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -77,7 +77,6 @@ }, "rg": "hcp-underlay-westus3-imagesync-dev" }, - "istioVersion": "asm-1-22", "kubernetesVersion": "1.30.6", "maestro": { "certDomain": "selfsigned.maestro.keyvault.azure.com", @@ -159,6 +158,12 @@ "kvName": "arohcp-etcd-dev", "kvSoftDelete": false }, + "istio": { + "istioctlVersion": "1.23.1", + "tag": "prod-stable", + "targetVersion": "asm-1-22", + "versions": "asm-1-22" + }, "rg": "hcp-underlay-dev-svc", "subscription": "ARO Hosted Control Planes (EA Subscription 1)", "userAgentPool": { diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json index 90668adf3..0aa3e7bef 100644 --- a/config/public-cloud-msft-int.json +++ b/config/public-cloud-msft-int.json @@ -76,7 +76,6 @@ }, "rg": "hcp-underlay-imagesync" }, - "istioVersion": "asm-1-22", "kubernetesVersion": "1.30.6", "maestro": { "certDomain": "selfsigned.maestro.keyvault.azure.com", @@ -158,6 +157,12 @@ "kvName": "arohcp-etcd-int", "kvSoftDelete": true }, + "istio": { + "istioctlVersion": "1.24.1", + "tag": "prod-stable", + "targetVersion": "asm-1-23", + "versions": "asm-1-22,asm-1-23" + }, "rg": "hcp-underlay-westus3-svc", "subscription": "hcp-westus3", "userAgentPool": { diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index 894ec7ad8..8ba1673e4 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -77,7 +77,6 @@ }, "rg": "hcp-underlay-westus3-imagesync-dev" }, - "istioVersion": "asm-1-22", "kubernetesVersion": "1.30.6", "maestro": { "certDomain": "selfsigned.maestro.keyvault.azure.com", @@ -159,6 +158,12 @@ "kvName": "arohcp-etcd-usw3tst", "kvSoftDelete": false }, + "istio": { + "istioctlVersion": "1.23.1", + "tag": "prod-stable", + "targetVersion": "asm-1-22", + "versions": "asm-1-22" + }, "rg": "hcp-underlay-usw3tst-svc", "subscription": "ARO Hosted Control Planes (EA Subscription 1)", "userAgentPool": { diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam index 6ff960752..882f6777c 100644 --- a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -1,7 +1,7 @@ using '../templates/svc-cluster.bicep' param kubernetesVersion = '{{ .kubernetesVersion }}' -param istioVersion = ['{{ .istioVersion }}'] +param istioVersions = '{{ .svc.istio.versions }}' param vnetAddressPrefix = '{{ .vnetAddressPrefix }}' param subnetPrefix = '{{ .subnetPrefix }}' param podSubnetPrefix = '{{ .podSubnetPrefix }}' diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep index ee09f41f7..fcf0b644d 100644 --- a/dev-infrastructure/modules/aks-cluster-base.bicep +++ b/dev-infrastructure/modules/aks-cluster-base.bicep @@ -25,7 +25,7 @@ param persist bool = false param kubernetesVersion string param deployIstio bool -param istioVersion array = ['asm-1-22'] +param istioVersions array = [] param vnetAddressPrefix string param subnetPrefix string param podSubnetPrefix string @@ -358,7 +358,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ } ] } - revisions: istioVersion + revisions: istioVersions } } : null diff --git a/dev-infrastructure/scripts/istio.sh b/dev-infrastructure/scripts/istio.sh new file mode 100755 index 000000000..c9568d5dc --- /dev/null +++ b/dev-infrastructure/scripts/istio.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +set -euo pipefail + +echo "********** Download istioctl **************" +# Determines the operating system. +OS="${TARGET_OS:-$(uname)}" +if [ "${OS}" = "Darwin" ] ; then + OSEXT="osx" +else + OSEXT="linux" +fi +# Determine arch +LOCAL_ARCH=$(uname -m) +case "${LOCAL_ARCH}" in + x86_64|amd64) + ISTIO_ARCH=amd64 + ;; + armv8*|aarch64*|arm64) + ISTIO_ARCH=arm64 + ;; + armv*) + ISTIO_ARCH=armv7 + ;; + *) + echo "This system's architecture, ${LOCAL_ARCH}, isn't supported" + exit 1 + ;; +esac + + +ISTIO_URL="https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istio-${ISTIOCTL_VERSION}-${OSEXT}-${ISTIO_ARCH}.tar.gz" +SHA256_URL="https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istio-${ISTIOCTL_VERSION}-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256" +# Download the Istioctl binary +wget -q "$ISTIO_URL" -O istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz + +# Download the SHA-256 checksum file +wget -q "$SHA256_URL" -O istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256 + +# Verify the downloaded file +sha256sum -c istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256 + +# Check the result of the verification +if sha256sum -c istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256; then + echo "Verification successful: The file is intact." +else + echo "Verification failed: The file is corrupted." + exit 1 +fi + +tar -xzf istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz +cd istio-"${ISTIOCTL_VERSION}" +export PATH=$PWD/bin:$PATH +echo "==========================================================================" + +ISTIO_NAMESPACE="aks-istio-system" + +echo "********** ISTIO Upgrade **************" +# Followed this guide for istio upgrade https://learn.microsoft.com/en-us/azure/aks/istio-upgrade +# To upgrade or rollback, change the targetVersion to the desire version, and version to the current version. +if [[ -z "$TARGET_VERSION" ]]; then + echo "Target version is not set, Please set the target version" + exit 1 +fi + +NEWVERSION="$TARGET_VERSION" +echo "********** Istio Upgrade Started with version ${NEWVERSION} **************" + +istioctl tag set "$TAG" --revision "${NEWVERSION}" --istioNamespace ${ISTIO_NAMESPACE} --overwrite +for namespace in $(kubectl get namespaces --selector=istio.io/rev="$TAG" -o jsonpath='{.items[*].metadata.name}'); do + echo "in namespace $namespace" + # bare pods + for pod in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | select(.metadata.ownerReferences | length == 0) | .metadata.name'); do + echo "recycle pod $pod" + kubectl delete pod "$pod" -n "$namespace" + done + # pods with owners + currentDeloyment="" + for owner in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | select(.metadata.ownerReferences) | "\(.metadata.ownerReferences[0].kind)/\(.metadata.ownerReferences[0].name)"' | sort | uniq); do + echo "process pod owner ${owner}" + case "$owner" in + "ReplicaSet"*) + deployment=$(kubectl get "${owner}" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].name}') + if [[ -n "$deployment" ]] && [[ "$currentDeloyment" != "$deployment" ]]; then + currentDeloyment="$deployment" + echo "in ReplicaSet restart deployment $deployment" + kubectl rollout restart deployment "$deployment" -n "$namespace" + kubectl rollout status deployment "${deployment}" -n "$namespace" + else + echo "in ReplicaSet delete pod $owner" + kubectl delete pod "$owner" -n "$namespace" + fi + ;; + "StatefulSet"*) + echo "restart statefulset $owner" + kubectl rollout restart "${owner}" -n "$namespace" + kubectl rollout status "${owner}" -n "$namespace" + ;; + *) + # Don't do anything for (Cron)Job, or no owner pod for now. + ;; + esac + # etc + done +done + +echo "********** ISTIO Upgrade Finished**************" diff --git a/dev-infrastructure/svc-pipeline.yaml b/dev-infrastructure/svc-pipeline.yaml index 195368086..0b5755ab5 100644 --- a/dev-infrastructure/svc-pipeline.yaml +++ b/dev-infrastructure/svc-pipeline.yaml @@ -32,3 +32,15 @@ resourceGroups: configRef: monitoring.grafanaName dependsOn: - svc + - name: istio + action: Shell + command: scripts/istio.sh + variables: + - name: TARGET_VERSION + configRef: svc.istio.targetVersion + - name: ISTIOCTL_VERSION + configRef: svc.istio.istioctlVersion + - name: TAG + configRef: svc.istio.tag + dependsOn: + - enable-metrics diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index 2587539c0..302709a3c 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -43,8 +43,8 @@ param podSubnetPrefix string @description('Kuberentes version to use with AKS') param kubernetesVersion string -@description('Istio control plane version to use with AKS') -param istioVersion array +@description('Istio control plane versions to use with AKS. CSV format') +param istioVersions string @description('The name of the keyvault for AKS.') @maxLength(24) @@ -158,7 +158,7 @@ module svcCluster '../modules/aks-cluster-base.bicep' = { aksEtcdKVEnableSoftDelete: aksEtcdKVEnableSoftDelete kubernetesVersion: kubernetesVersion deployIstio: true - istioVersion: istioVersion + istioVersions: split(istioVersions, ',') vnetAddressPrefix: vnetAddressPrefix subnetPrefix: subnetPrefix podSubnetPrefix: podSubnetPrefix