leverage helm chart for hypershift install/upgrade

https://issues.redhat.com/browse/ARO-11084

Signed-off-by: Gerd Oberlechner <[email protected]>

config/config.yaml

@@ -44,6 +44,11 @@ defaults:
   maestroRestrictIstioIngress: true
   maestroConsumerName: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-mgmt-{{ .ctx.cxStamp }}
 
+  # Hypershift
+  hypershiftNamespace: hypershift
+  externalDNSManagedIdentityName: external-dns
+  externalDNSServiceAccountName: external-dns
+
   # Cluster Service
   clusterServicePostgresName: {{ azurePostgresName "cs" 5 .ctx.region .ctx.regionStamp }}
   clusterServicePostgresDeploy: true
@@ -106,7 +111,7 @@ clouds:
       clusterServiceImageTag: a23276d
       clusterServiceImageRepo: app-sre/uhc-clusters-service
       # Hypershift Operator
-      hypershiftOperatorImageTag: 99a256f
+      hypershiftOperatorImageTag: a95fc46
       externalDNSImageTag: v0.14.2
       # Shared SVC KV
       serviceKeyVaultName: 'aro-hcp-dev-svc-kv'

config/public-cloud-cs-pr.json

@@ -14,14 +14,17 @@
   "cxKeyVaultPrivate": false,
   "cxKeyVaultSoftDelete": false,
   "externalDNSImageTag": "v0.14.2",
+  "externalDNSManagedIdentityName": "external-dns",
+  "externalDNSServiceAccountName": "external-dns",
   "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
   "frontendCosmosDBDeploy": true,
   "frontendCosmosDBDisableLocalAuth": true,
   "frontendCosmosDBName": "aro-hcp-rp-9c782",
   "globalRG": "global",
   "grafanaAdminGroupPrincipalId": "6b6d3adf-8476-4727-9812-20ffdef2b85c",
   "grafanaName": "aro-hcp-grafana-9c782",
-  "hypershiftOperatorImageTag": "99a256f",
+  "hypershiftNamespace": "hypershift",
+  "hypershiftOperatorImageTag": "a95fc46",
   "imageSyncAcrRG": "global",
   "imageSyncEnvironmentName": "aro-hcp-image-sync",
   "imageSyncImageRepo": "image-sync/component-sync",

config/public-cloud-dev.json

@@ -14,14 +14,17 @@
   "cxKeyVaultPrivate": false,
   "cxKeyVaultSoftDelete": false,
   "externalDNSImageTag": "v0.14.2",
+  "externalDNSManagedIdentityName": "external-dns",
+  "externalDNSServiceAccountName": "external-dns",
   "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
   "frontendCosmosDBDeploy": true,
   "frontendCosmosDBDisableLocalAuth": true,
   "frontendCosmosDBName": "aro-hcp-rp-157ff",
   "globalRG": "global",
   "grafanaAdminGroupPrincipalId": "6b6d3adf-8476-4727-9812-20ffdef2b85c",
   "grafanaName": "aro-hcp-grafana-157ff",
-  "hypershiftOperatorImageTag": "99a256f",
+  "hypershiftNamespace": "hypershift",
+  "hypershiftOperatorImageTag": "a95fc46",
   "imageSyncAcrRG": "global",
   "imageSyncEnvironmentName": "aro-hcp-image-sync",
   "imageSyncImageRepo": "image-sync/component-sync",

config/public-cloud-personal-dev.json

@@ -14,14 +14,17 @@
   "cxKeyVaultPrivate": false,
   "cxKeyVaultSoftDelete": false,
   "externalDNSImageTag": "v0.14.2",
+  "externalDNSManagedIdentityName": "external-dns",
+  "externalDNSServiceAccountName": "external-dns",
   "firstPartyAppClientId": "57e54810-3138-4f38-bd3b-29cb33f4c358",
   "frontendCosmosDBDeploy": true,
   "frontendCosmosDBDisableLocalAuth": true,
   "frontendCosmosDBName": "aro-hcp-rp-76fc6",
   "globalRG": "global",
   "grafanaAdminGroupPrincipalId": "6b6d3adf-8476-4727-9812-20ffdef2b85c",
   "grafanaName": "aro-hcp-grafana-76fc6",
-  "hypershiftOperatorImageTag": "99a256f",
+  "hypershiftNamespace": "hypershift",
+  "hypershiftOperatorImageTag": "a95fc46",
   "imageSyncAcrRG": "global",
   "imageSyncEnvironmentName": "aro-hcp-image-sync",
   "imageSyncImageRepo": "image-sync/component-sync",

dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam

@@ -24,6 +24,11 @@ param maestroKeyVaultName = '{{ .maestroKeyVaultName }}'
 param maestroEventGridNamespacesName = '{{ .maestroEventgridName }}'
 param maestroCertDomain = '{{ .maestroCertDomain }}'
 
+// Hypershift
+param hypershiftNamespace = '{{ .hypershiftNamespace }}'
+param externalDNSManagedIdentityName = '{{ .externalDNSManagedIdentityName }}'
+param externalDNSServiceAccountName = '{{ .externalDNSServiceAccountName }}'
+
 // DNS
 param regionalDNSZoneName = '{{ .regionalDNSSubdomain}}.{{ .baseDnsZoneName }}'
 

dev-infrastructure/templates/mgmt-cluster.bicep

@@ -59,6 +59,15 @@ param aksKeyVaultName string
 @description('Manage soft delete setting for AKS etcd key-value store')
 param aksEtcdKVEnableSoftDelete bool = true
 
+@description('The name of the hypershift namespace.')
+param hypershiftNamespace string
+
+@description('The name of the external DNS managed identity.')
+param externalDNSManagedIdentityName string
+
+@description('The name of the external DNS service account.')
+param externalDNSServiceAccountName string
+
 @description('The name of the maestro consumer.')
 param maestroConsumerName string
 
@@ -140,9 +149,9 @@ module mgmtCluster '../modules/aks-cluster-base.bicep' = {
         serviceAccountName: 'maestro'
       }
       external_dns_wi: {
-        uamiName: 'external-dns'
-        namespace: 'hypershift'
-        serviceAccountName: 'external-dns'
+        uamiName: externalDNSManagedIdentityName
+        namespace: hypershiftNamespace
+        serviceAccountName: externalDNSServiceAccountName
       }
     })
     aksKeyVaultName: aksKeyVaultName
@@ -188,11 +197,11 @@ module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = {
 
 var externalDnsManagedIdentityPrincipalId = filter(
   mgmtCluster.outputs.userAssignedIdentities,
-  id => id.uamiName == 'external-dns'
+  id => id.uamiName == externalDNSManagedIdentityName
 )[0].uamiPrincipalID
 
 module dnsZoneContributor '../modules/dns/zone-contributor.bicep' = {
-  name: guid(regionalDNSZoneName, mgmtCluster.name, 'external-dns')
+  name: guid(regionalDNSZoneName, mgmtCluster.name, externalDNSManagedIdentityName)
   scope: resourceGroup(regionalResourceGroup)
   params: {
     zoneName: regionalDNSZoneName

hypershiftoperator/.gitignore

@@ -1,7 +1 @@
-kubectl-slice
-edo-azure-credentials.json
-domain.txt
-patch-serviceaccount-external-dns.json
-test
-deploy/overlays/dev/txt_owner_id.txt
 config.mk

hypershiftoperator/Makefile

@@ -3,69 +3,24 @@ DEPLOY_ENV ?= personal-dev
 $(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
 include config.mk
 
-HO_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/acm-d/rhtap-hypershift-operator:${HO_IMAGE_TAG}
-EDO_IMAGE ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io/external-dns/external-dns:${ED_IMAGE_TAG}
-
-create-edo-azure-creds:
-	@echo '{' > deploy/overlays/dev/edo-azure-credentials.json
-	@AZURE_TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
-	echo "  \"tenantId\": \"$$AZURE_TENANT_ID\"," >> deploy/overlays/dev/edo-azure-credentials.json
-	@AZURE_SUBSCRIPTION_ID=$(shell az account show --query id --output tsv) && \
-	echo "  \"subscriptionId\": \"$$AZURE_SUBSCRIPTION_ID\"," >> deploy/overlays/dev/edo-azure-credentials.json
-	@echo '  "resourceGroup": "${REGIONAL_RESOURCEGROUP}",' >> deploy/overlays/dev/edo-azure-credentials.json
-	@echo '  "useWorkloadIdentityExtension": true,' >> deploy/overlays/dev/edo-azure-credentials.json
-	@EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n external-dns --query clientId -o tsv) && \
-	echo "  \"userAssignedIdentityID\": \"$$EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID\"" >> deploy/overlays/dev/edo-azure-credentials.json
-	@echo '}' >> deploy/overlays/dev/edo-azure-credentials.json
-
-create-edo-sa-patch:
-	@EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n external-dns --query clientId -o tsv) && \
-	echo "[{\"op\": \"add\", \"path\": \"/metadata/annotations/azure.workload.identity~1client-id\", \"value\": \"$$EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID\"}]" > deploy/overlays/dev/patch-serviceaccount-external-dns.json
-
-create-domain-file:
-	@echo "${ZONE_NAME}" > deploy/overlays/dev/domain.txt
-
-create-txt-owner-id-file:
-	@echo "${RESOURCEGROUP}" > deploy/overlays/dev/txt_owner_id.txt
-
-deploy: create-edo-azure-creds create-edo-sa-patch create-domain-file create-txt-owner-id-file
-	kubectl apply --server-side --force-conflicts -k deploy/crds
-	kubectl apply --server-side --force-conflicts -k deploy/overlays/dev
-
-# run this task whenever HO_IMAGE or EDO_IMAGE is updated
-# commit the changes to deploy/base to the repo since we don't have podman or docker in CI
-prepare-ho-manifests:
-	@curl -sfLo - "https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.3.1/kubectl-slice_${shell uname -s | tr '[:upper:]' '[:lower:]'}_${shell uname -m}.tar.gz" | tar xzf - kubectl-slice
-	@rm deploy/base/*
-	curl https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml -o deploy/base/customresourcedefinition-monitoring.coreos.com_servicemonitors.yaml
-	curl https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml -o deploy/base/customresourcedefinition-monitoring.coreos.com_prometheusrules.yaml
-	curl https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml -o deploy/base/customresourcedefinition-monitoring.coreos.com_podmonitors.yaml
-	curl https://raw.githubusercontent.com/openshift/api/master/route/v1/zz_generated.crd-manifests/routes-Default.crd.yaml -o deploy/base/customresourcedefinition-routes-default.crd.yaml
-	# Currently the hypershift install render command prints an error out to stdout if there is no available kubeconfig
-	# TODO: Get a fix into the hypershift install render command so we don't have to do this
-	@podman run -it --rm ${HO_IMAGE} install render \
-		--hypershift-image ${HO_IMAGE} \
-		--enable-conversion-webhook=false \
-		--external-dns-provider azure \
-		--external-dns-secret external-dns-azure \
-		--external-dns-domain-filter \$$\(DOMAIN\) \
-		--external-dns-image ${EDO_IMAGE} \
-		--external-dns-txt-owner-id \$$\(TXT_OWNER_ID\) \
-		--managed-service ARO-HCP \
-		| tail -n +2 \
-		| ./kubectl-slice -f - -o deploy/base
-	@rm deploy/crds/*
-	@mkdir -p deploy/crds
-	@mv deploy/base/customresourcedefinition*.yaml deploy/crds
-
-	@echo "apiVersion: kustomize.config.k8s.io/v1beta1" > deploy/crds/kustomization.yml
-	@echo "kind: Kustomization" >> deploy/crds/kustomization.yml
-	@echo "resources:" >> deploy/crds/kustomization.yml
-	@find deploy/crds -type f -name "customresourcedefinition*.yaml" ! -name "kustomization.yml" | sed 's/^deploy\/crds\//  - /' >> deploy/crds/kustomization.yml
-
-	@echo "apiVersion: kustomize.config.k8s.io/v1beta1" > deploy/base/kustomization.yml
-	@echo "kind: Kustomization" >> deploy/base/kustomization.yml
-	@echo "resources:" >> deploy/base/kustomization.yml
-	@find deploy/base -type f -name "*.yaml" ! -name "kustomization.yml" | sed 's/^deploy\/base\//  - /' >> deploy/base/kustomization.yml
-
-.PHONY: create-edo-azure-creds create-edo-sa-patch create-domain-file create-txt-owner-id-file deploy unpack-ho-manifests
+deploy:
+	@EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n ${EXTERNAL_DNS_MI_NAME} --query clientId -o tsv) && \
+	AZURE_TENANT_ID=$(shell az account show --query tenantId --output tsv) && \
+	AZURE_SUBSCRIPTION_ID=$(shell az account show --query id --output tsv) && \
+	CSI_SECRET_STORE_CLIENT_ID=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query 'addonProfiles.azureKeyvaultSecretsProvider.identity.clientId' -o tsv) && \
+	helm upgrade --install hypershift deploy/helm \
+		--create-namespace --namespace ${HYPERSHIFT_NAMESPACE} \
+		--set image=${HO_IMAGE_BASE} \
+		--set imageTag=${HO_IMAGE_TAG} \
+		--set registryOverrides="quay.io/openshift-release-dev/ocp-v4.0-art-dev=${ARO_HCP_OCP_ACR}.azurecr.io/openshift/release\,quay.io/openshift-release-dev/ocp-release=${ARO_HCP_OCP_ACR}.azurecr.io/openshift/release-images\,registry.redhat.io/redhat=${ARO_HCP_OCP_ACR}.azurecr.io/redhat" \
+		--set azureKeyVaultClientId=$${CSI_SECRET_STORE_CLIENT_ID} \
+		--set external-dns.image=${ED_IMAGE_BASE} \
+		--set external-dns.imageTag=${ED_IMAGE_TAG} \
+		--set external-dns.txtOwnerId=${RESOURCEGROUP} \
+		--set external-dns.domain=${ZONE_NAME} \
+		--set external-dns.credentials.tenantId=$${AZURE_TENANT_ID} \
+		--set external-dns.credentials.subscriptionId=$${AZURE_SUBSCRIPTION_ID} \
+		--set external-dns.credentials.resourceGroup=${REGIONAL_RESOURCEGROUP} \
+		--set external-dns.credentials.userAssignedIdentityID=$${EXTERNAL_DNS_OPERATOR_MI_CLIENT_ID}
+
+.PHONY: helm-chart deploy

hypershiftoperator/README.md

@@ -1,3 +1,3 @@
 # HyperShift Operator
 
-To promote HyperShift Operator, increment the `HO_IMAGE` variable in the Makefile, then run `make prepare-ho-manifests` to generate the corresponding manifests for that image.
+To promote HyperShift Operator, increment the `hypershiftOperatorImageTag` parameter in the `config/config.yaml` file.

hypershiftoperator/config.tmpl.mk

@@ -1,6 +1,18 @@
-ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }}
+ARO_HCP_SVC_ACR ?= {{ .svcAcrName }}
+ARO_HCP_OCP_ACR ?= {{ .ocpAcrName }}
 HO_IMAGE_TAG ?= {{ .hypershiftOperatorImageTag }}
+HO_IMAGE_BASE ?= ${ARO_HCP_SVC_ACR}.azurecr.io/acm-d/rhtap-hypershift-operator
+HO_IMAGE ?= ${HO_IMAGE_BASE}:${HO_IMAGE_TAG}
+ED_IMAGE ?= ${ARO_HCP_SVC_ACR}.azurecr.io/external-dns/external-dns:${ED_IMAGE_TAG}
 ED_IMAGE_TAG ?= {{ .externalDNSImageTag }}
+ED_IMAGE_BASE ?= ${ARO_HCP_SVC_ACR}.azurecr.io/external-dns/external-dns
+ED_IMAGE ?= ${ED_IMAGE_BASE}:${ED_IMAGE_TAG}
+
 RESOURCEGROUP ?= {{ .managementClusterRG }}
 REGIONAL_RESOURCEGROUP ?= {{ .regionRG }}
 ZONE_NAME ?= {{ .regionalDNSSubdomain }}.{{ .baseDnsZoneName }}
+AKS_NAME ?= {{ .aksName }}
+HYPERSHIFT_NAMESPACE ?= {{ .hypershiftNamespace}}
+EXTERNAL_DNS_MI_NAME ?= {{ .externalDNSManagedIdentityName }}
+
+HO_CHART_DIR ?= deploy/helm/charts/hypershift-operator