Skip to content

Commit

Permalink
Move istio to svc cluster rollout (#979)
Browse files Browse the repository at this point in the history
Move istio from hcp to ARO-HCP. Will run istio sh in every svc deployment. But will only upgrade if there's new version

Signed-off-by: Gerd Oberlechner <[email protected]>
Co-authored-by: Jonathan Chang <[email protected]>
Co-authored-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
3 people authored Dec 20, 2024
1 parent 0bd4db8 commit b6c5808
Show file tree
Hide file tree
Showing 13 changed files with 184 additions and 18 deletions.
2 changes: 1 addition & 1 deletion config/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ detect-change: materialize
echo "Please review the diffs below:\n\n"; \
echo "$$diff_output"; \
echo "\n\n===================================================="; \
echo "\n\nOnce you reviewed the changes and consider them meaningful, commit them."; \
echo "\n\nOnce you reviewed the changes and consider them meaningful, update them by running "make -C config/ materialize" and commit your changes."; \
echo "\n\n===================================================="; \
exit 1; \
else \
Expand Down
6 changes: 5 additions & 1 deletion config/config.msft.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ defaults:

# General AKS config
kubernetesVersion: 1.30.6
istioVersion: "asm-1-22"
vnetAddressPrefix: "10.128.0.0/14"
subnetPrefix: "10.128.8.0/21"
podSubnetPrefix: "10.128.64.0/18"
Expand All @@ -31,6 +30,11 @@ defaults:
etcd:
kvName: arohcp-etcd-{{ .ctx.regionShort }}
kvSoftDelete: true
istio:
istioctlVersion: "1.24.1"
tag: "prod-stable"
targetVersion: "asm-1-23"
versions: "asm-1-22,asm-1-23"

# MGMT cluster specifics
mgmt:
Expand Down
29 changes: 24 additions & 5 deletions config/config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -323,9 +323,6 @@
"rg"
]
},
"istioVersion": {
"type": "string"
},
"kubernetesVersion": {
"type": "string"
},
Expand Down Expand Up @@ -672,7 +669,30 @@
},
"subscription": {
"type": "string"
}
},
"istio": {
"type": "object",
"properties": {
"istioctlVersion": {
"description": "Latest version of istioctl. This is used to download the istioctl",
"type": "string"
},
"tag": {
"description": "The revision tag used in istioctl",
"type": "string"
},
"versions": {
"description": "The current istio version in the AKS cluster as CSV, will need to change to array once tooling is ready",
"type": "string",
"pattern": "^[a-zA-Z0-9-]+(,[a-zA-Z0-9-]+)*$"
},
"targetVersion": {
"description": "The target istio version that will be updated to",
"type": "string",
"pattern": "^[a-zA-Z0-9-]+$"
}
}
}
},
"additionalProperties": false,
"required": [
Expand Down Expand Up @@ -703,7 +723,6 @@
"hypershift",
"hypershiftOperator",
"imageSync",
"istioVersion",
"kubernetesVersion",
"maestro",
"mgmt",
Expand Down
6 changes: 5 additions & 1 deletion config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ defaults:

# General AKS config
kubernetesVersion: 1.30.6
istioVersion: "asm-1-22"
vnetAddressPrefix: "10.128.0.0/14"
subnetPrefix: "10.128.8.0/21"
podSubnetPrefix: "10.128.64.0/18"
Expand All @@ -29,6 +28,11 @@ defaults:
etcd:
kvName: arohcp-etcd-{{ .ctx.regionShort }}
kvSoftDelete: true
istio:
istioctlVersion: "1.23.1"
tag: "prod-stable"
targetVersion: "asm-1-22"
versions: "asm-1-22"

# MGMT cluster specifics
mgmt:
Expand Down
7 changes: 6 additions & 1 deletion config/public-cloud-cs-pr.json
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@
},
"rg": "hcp-underlay-westus3-imagesync-dev"
},
"istioVersion": "asm-1-22",
"kubernetesVersion": "1.30.6",
"maestro": {
"certDomain": "selfsigned.maestro.keyvault.azure.com",
Expand Down Expand Up @@ -159,6 +158,12 @@
"kvName": "arohcp-etcd-cspr",
"kvSoftDelete": false
},
"istio": {
"istioctlVersion": "1.23.1",
"tag": "prod-stable",
"targetVersion": "asm-1-22",
"versions": "asm-1-22"
},
"rg": "hcp-underlay-cspr-svc",
"subscription": "ARO Hosted Control Planes (EA Subscription 1)",
"userAgentPool": {
Expand Down
7 changes: 6 additions & 1 deletion config/public-cloud-dev.json
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@
},
"rg": "hcp-underlay-westus3-imagesync-dev"
},
"istioVersion": "asm-1-22",
"kubernetesVersion": "1.30.6",
"maestro": {
"certDomain": "selfsigned.maestro.keyvault.azure.com",
Expand Down Expand Up @@ -159,6 +158,12 @@
"kvName": "arohcp-etcd-dev",
"kvSoftDelete": false
},
"istio": {
"istioctlVersion": "1.23.1",
"tag": "prod-stable",
"targetVersion": "asm-1-22",
"versions": "asm-1-22"
},
"rg": "hcp-underlay-dev-svc",
"subscription": "ARO Hosted Control Planes (EA Subscription 1)",
"userAgentPool": {
Expand Down
7 changes: 6 additions & 1 deletion config/public-cloud-msft-int.json
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@
},
"rg": "hcp-underlay-imagesync"
},
"istioVersion": "asm-1-22",
"kubernetesVersion": "1.30.6",
"maestro": {
"certDomain": "selfsigned.maestro.keyvault.azure.com",
Expand Down Expand Up @@ -158,6 +157,12 @@
"kvName": "arohcp-etcd-int",
"kvSoftDelete": true
},
"istio": {
"istioctlVersion": "1.24.1",
"tag": "prod-stable",
"targetVersion": "asm-1-23",
"versions": "asm-1-22,asm-1-23"
},
"rg": "hcp-underlay-westus3-svc",
"subscription": "hcp-westus3",
"userAgentPool": {
Expand Down
7 changes: 6 additions & 1 deletion config/public-cloud-personal-dev.json
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@
},
"rg": "hcp-underlay-westus3-imagesync-dev"
},
"istioVersion": "asm-1-22",
"kubernetesVersion": "1.30.6",
"maestro": {
"certDomain": "selfsigned.maestro.keyvault.azure.com",
Expand Down Expand Up @@ -159,6 +158,12 @@
"kvName": "arohcp-etcd-usw3tst",
"kvSoftDelete": false
},
"istio": {
"istioctlVersion": "1.23.1",
"tag": "prod-stable",
"targetVersion": "asm-1-22",
"versions": "asm-1-22"
},
"rg": "hcp-underlay-usw3tst-svc",
"subscription": "ARO Hosted Control Planes (EA Subscription 1)",
"userAgentPool": {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using '../templates/svc-cluster.bicep'

param kubernetesVersion = '{{ .kubernetesVersion }}'
param istioVersion = ['{{ .istioVersion }}']
param istioVersions = '{{ .svc.istio.versions }}'
param vnetAddressPrefix = '{{ .vnetAddressPrefix }}'
param subnetPrefix = '{{ .subnetPrefix }}'
param podSubnetPrefix = '{{ .podSubnetPrefix }}'
Expand Down
4 changes: 2 additions & 2 deletions dev-infrastructure/modules/aks-cluster-base.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ param persist bool = false

param kubernetesVersion string
param deployIstio bool
param istioVersion array = ['asm-1-22']
param istioVersions array = []
param vnetAddressPrefix string
param subnetPrefix string
param podSubnetPrefix string
Expand Down Expand Up @@ -358,7 +358,7 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-previ
}
]
}
revisions: istioVersion
revisions: istioVersions
}
}
: null
Expand Down
107 changes: 107 additions & 0 deletions dev-infrastructure/scripts/istio.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#!/bin/bash

set -euo pipefail

echo "********** Download istioctl **************"
# Determines the operating system.
OS="${TARGET_OS:-$(uname)}"
if [ "${OS}" = "Darwin" ] ; then
OSEXT="osx"
else
OSEXT="linux"
fi
# Determine arch
LOCAL_ARCH=$(uname -m)
case "${LOCAL_ARCH}" in
x86_64|amd64)
ISTIO_ARCH=amd64
;;
armv8*|aarch64*|arm64)
ISTIO_ARCH=arm64
;;
armv*)
ISTIO_ARCH=armv7
;;
*)
echo "This system's architecture, ${LOCAL_ARCH}, isn't supported"
exit 1
;;
esac


ISTIO_URL="https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istio-${ISTIOCTL_VERSION}-${OSEXT}-${ISTIO_ARCH}.tar.gz"
SHA256_URL="https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istio-${ISTIOCTL_VERSION}-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256"
# Download the Istioctl binary
wget -q "$ISTIO_URL" -O istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz

# Download the SHA-256 checksum file
wget -q "$SHA256_URL" -O istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256

# Verify the downloaded file
sha256sum -c istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256

# Check the result of the verification
if sha256sum -c istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz.sha256; then
echo "Verification successful: The file is intact."
else
echo "Verification failed: The file is corrupted."
exit 1
fi

tar -xzf istio-"${ISTIOCTL_VERSION}"-${OSEXT}-${ISTIO_ARCH}.tar.gz
cd istio-"${ISTIOCTL_VERSION}"
export PATH=$PWD/bin:$PATH
echo "=========================================================================="

ISTIO_NAMESPACE="aks-istio-system"

echo "********** ISTIO Upgrade **************"
# Followed this guide for istio upgrade https://learn.microsoft.com/en-us/azure/aks/istio-upgrade
# To upgrade or rollback, change the targetVersion to the desire version, and version to the current version.
if [[ -z "$TARGET_VERSION" ]]; then
echo "Target version is not set, Please set the target version"
exit 1
fi

NEWVERSION="$TARGET_VERSION"
echo "********** Istio Upgrade Started with version ${NEWVERSION} **************"

istioctl tag set "$TAG" --revision "${NEWVERSION}" --istioNamespace ${ISTIO_NAMESPACE} --overwrite
for namespace in $(kubectl get namespaces --selector=istio.io/rev="$TAG" -o jsonpath='{.items[*].metadata.name}'); do
echo "in namespace $namespace"
# bare pods
for pod in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | select(.metadata.ownerReferences | length == 0) | .metadata.name'); do
echo "recycle pod $pod"
kubectl delete pod "$pod" -n "$namespace"
done
# pods with owners
currentDeloyment=""
for owner in $(kubectl get pods --namespace "${namespace}" -o json | jq -r --arg NEWVERSION "${NEWVERSION}" '.items[] | select(.metadata.annotations["sidecar.istio.io/status"] | fromjson.revision != $NEWVERSION) | select(.metadata.ownerReferences) | "\(.metadata.ownerReferences[0].kind)/\(.metadata.ownerReferences[0].name)"' | sort | uniq); do
echo "process pod owner ${owner}"
case "$owner" in
"ReplicaSet"*)
deployment=$(kubectl get "${owner}" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[0].name}')
if [[ -n "$deployment" ]] && [[ "$currentDeloyment" != "$deployment" ]]; then
currentDeloyment="$deployment"
echo "in ReplicaSet restart deployment $deployment"
kubectl rollout restart deployment "$deployment" -n "$namespace"
kubectl rollout status deployment "${deployment}" -n "$namespace"
else
echo "in ReplicaSet delete pod $owner"
kubectl delete pod "$owner" -n "$namespace"
fi
;;
"StatefulSet"*)
echo "restart statefulset $owner"
kubectl rollout restart "${owner}" -n "$namespace"
kubectl rollout status "${owner}" -n "$namespace"
;;
*)
# Don't do anything for (Cron)Job, or no owner pod for now.
;;
esac
# etc
done
done

echo "********** ISTIO Upgrade Finished**************"
12 changes: 12 additions & 0 deletions dev-infrastructure/svc-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,15 @@ resourceGroups:
configRef: monitoring.grafanaName
dependsOn:
- svc
- name: istio
action: Shell
command: scripts/istio.sh
variables:
- name: TARGET_VERSION
configRef: svc.istio.targetVersion
- name: ISTIOCTL_VERSION
configRef: svc.istio.istioctlVersion
- name: TAG
configRef: svc.istio.tag
dependsOn:
- enable-metrics
6 changes: 3 additions & 3 deletions dev-infrastructure/templates/svc-cluster.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ param podSubnetPrefix string
@description('Kuberentes version to use with AKS')
param kubernetesVersion string

@description('Istio control plane version to use with AKS')
param istioVersion array
@description('Istio control plane versions to use with AKS. CSV format')
param istioVersions string

@description('The name of the keyvault for AKS.')
@maxLength(24)
Expand Down Expand Up @@ -158,7 +158,7 @@ module svcCluster '../modules/aks-cluster-base.bicep' = {
aksEtcdKVEnableSoftDelete: aksEtcdKVEnableSoftDelete
kubernetesVersion: kubernetesVersion
deployIstio: true
istioVersion: istioVersion
istioVersions: split(istioVersions, ',')
vnetAddressPrefix: vnetAddressPrefix
subnetPrefix: subnetPrefix
podSubnetPrefix: podSubnetPrefix
Expand Down

0 comments on commit b6c5808

Please sign in to comment.