Skip to content

Commit

Permalink
CS pipeline.yaml
Browse files Browse the repository at this point in the history
Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
geoberle committed Dec 20, 2024
1 parent 0bd4db8 commit 3246099
Show file tree
Hide file tree
Showing 16 changed files with 389 additions and 142 deletions.
42 changes: 28 additions & 14 deletions cluster-service/Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
SHELL = /bin/bash
DEPLOY_ENV ?= personal-dev
$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
include config.mk
-include ../setup-env.mk

ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${BASE_DNS_ZONE_NAME}"


deploy: provision-shard
@ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
Expand All @@ -14,8 +14,17 @@ deploy: provision-shard
OIDC_ISSUER_BASE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.web -o tsv) && \
OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \
OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \
helm upgrade --install cluster-service --namespace cluster-service \
deploy/helm/ \
DB_HOST=$$(if [ "${USE_AZURE_DB}" = "true" ]; then az postgres flexible-server show -g ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv; else echo "ocm-cs-db"; fi) && \
OVERRIDES=$$(if [ "${USE_AZURE_DB}" = "true" ]; then echo "azuredb.values.yaml"; else echo "containerdb.values.yaml"; fi) && \
OP_CLOUD_CONTROLLER_MANAGER_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_INGRESS_ROLE_ID=$(shell az role definition list --name "${OP_INGRESS_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_DISK_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_DISK_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_FILE_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_FILE_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_IMAGE_REGISTRY_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_CLOUD_NETWORK_CONFIG_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" --query "[].name" -o tsv) && \
helm upgrade --install --wait ${HELM_DRY_RUN} cluster-service deploy/helm \
--namespace cluster-service \
-f deploy/helm/$${OVERRIDES} \
--set azureCsMiClientId=$${AZURE_CS_MI_CLIENT_ID} \
--set oidcIssuerBlobServiceUrl=$${OIDC_BLOB_SERVICE_ENDPOINT} \
--set oidcIssuerBaseUrl=$${OIDC_ISSUER_BASE_ENDPOINT} \
Expand All @@ -30,21 +39,26 @@ deploy: provision-shard
--set fpaCertName=${FPA_CERT_NAME} \
--set ocpAcrResourceId=$${OCP_ACR_RESOURCE_ID} \
--set ocpAcrUrl=$${OCP_ACR_URL} \
--set databaseDisableTls=${DATABASE_DISABLE_TLS} \
--set databaseAuthMethod=${DATABASE_AUTH_METHOD} \
--set provisionShardsConfig="$(shell base64 -i deploy/provisioning-shards.yml | tr -d '\n')" \
--set deployLocalDatabase=${DEPLOY_LOCAL_DB} \
--set databaseHost=${DB_HOST} \
--set databaseName=${DB_NAME} \
--set databaseUser=${DB_USERNAME} \
--set databasePassword=${DB_PASSWORD} \
--set databaseHost=$${DB_HOST} \
--set azureMiMockServicePrincipalPrincipalId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID} \
--set azureMiMockServicePrincipalClientId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID} \
--set azureMiMockServicePrincipalCertName=${MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME} \
--set azureArmHelperIdentityCertName=${ARM_HELPER_CERT_NAME} \
--set azureArmHelperIdentityClientId=${AZURE_ARM_HELPER_IDENTITY_CLIENT_ID} \
--set azureArmHelperMockFpaPrincipalId=${AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID} \
--set azureOperatorsManagedIdentitiesConfig=${AZURE_OPERATORS_MANAGED_IDENTITIES_CONFIG}
--set azureOperatorsMI.cloudControllerManager.roleName="${OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME}" \
--set azureOperatorsMI.cloudControllerManager.roleId="$${OP_CLOUD_CONTROLLER_MANAGER_ROLE_ID}" \
--set azureOperatorsMI.ingress.roleName="${OP_INGRESS_ROLE_NAME}" \
--set azureOperatorsMI.ingress.roleId="$${OP_INGRESS_ROLE_ID}" \
--set azureOperatorsMI.diskCsiDriver.roleName="${OP_DISK_CSI_DRIVER_ROLE_NAME}" \
--set azureOperatorsMI.diskCsiDriver.roleId="$${OP_DISK_CSI_DRIVER_ROLE_ID}" \
--set azureOperatorsMI.fileCsiDriver.roleName="${OP_FILE_CSI_DRIVER_ROLE_NAME}" \
--set azureOperatorsMI.fileCsiDriver.roleId="$${OP_FILE_CSI_DRIVER_ROLE_ID}" \
--set azureOperatorsMI.imageRegistry.roleName="${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" \
--set azureOperatorsMI.imageRegistry.roleId="$${OP_IMAGE_REGISTRY_DRIVER_ROLE_ID}" \
--set azureOperatorsMI.cloudNetworkConfig.roleName="${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" \
--set azureOperatorsMI.cloudNetworkConfig.roleId="$${OP_CLOUD_NETWORK_CONFIG_ROLE_ID}"

deploy-pr-env-deps:
AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \
Expand Down
39 changes: 0 additions & 39 deletions cluster-service/config.tmpl.mk

This file was deleted.

6 changes: 6 additions & 0 deletions cluster-service/deploy/helm/azuredb.values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
databaseDisableTls: false
databaseAuthMethod: az-entra
deployLocalDatabase: false
databaseName: clusters-service
databaseUser: clusters-service
databasePassword: ''
6 changes: 6 additions & 0 deletions cluster-service/deploy/helm/containerdb.values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
databaseDisableTls: true
databaseAuthMethod: postgres
deployLocalDatabase: true
databaseName: ocm-cs-db
databaseUser: ocm
databasePassword: TheBlurstOfTimes
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,83 @@ metadata:
namespace: {{ .Release.Namespace }}
data:
azure-operators-managed-identities-config.yaml: |
{{ .Values.azureOperatorsManagedIdentitiesConfig | b64dec | indent 4 }}
controlPlaneOperatorsIdentities:
cloud-controller-manager:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudControllerManager.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudControllerManager.roleName }}'
optional: false
ingress:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.ingress.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.ingress.roleName }}'
optional: false
disk-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.diskCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.diskCsiDriver.roleName }}'
optional: false
file-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.fileCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.fileCsiDriver.roleName }}'
optional: false
image-registry:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.imageRegistry.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.imageRegistry.roleName }}'
optional: false
cloud-network-config:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleName }}'
optional: false
dataPlaneOperatorsIdentities:
disk-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.diskCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.diskCsiDriver.roleName }}'
k8sServiceAccounts:
- name: 'azure-disk-csi-driver-operator'
namespace: 'openshift-cluster-csi-drivers'
- name: 'azure-disk-csi-driver-controller-sa'
namespace: 'openshift-cluster-csi-drivers'
optional: false
image-registry:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.imageRegistry.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.imageRegistry.roleName }}'
k8sServiceAccounts:
- name: 'cluster-image-registry-operator'
namespace: 'openshift-image-registry'
- name: 'registry'
namespace: 'openshift-image-registry'
optional: false
file-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.fileCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.fileCsiDriver.roleName }}'
k8sServiceAccounts:
- name: 'azure-file-csi-driver-operator'
namespace: 'openshift-cluster-csi-drivers'
- name: 'azure-file-csi-driver-controller-sa'
namespace: 'openshift-cluster-csi-drivers'
- name: 'azure-file-csi-driver-node-sa'
namespace: 'openshift-cluster-csi-drivers'
optional: false
ingress:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.ingress.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.ingress.roleName }}'
k8sServiceAccounts:
- name: 'ingress-operator'
namespace: 'openshift-ingress-operator'
optional: false
cloud-network-config:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleName }}'
k8sServiceAccounts:
- name: 'cloud-network-config-controller'
namespace: 'openshift-cloud-network-config-controller'
optional: false
8 changes: 8 additions & 0 deletions cluster-service/deploy/helm/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ spec:
labels:
app: clusters-service
azure.workload.identity/use: "true"
annotations:
checksum/db: {{ include (print $.Template.BasePath "/database.secret.yaml") . | sha256sum }}
checksum/azurecreds: {{ include (print $.Template.BasePath "/azure-credentials.secret.yaml") . | sha256sum }}
checksum/operatorcfg: {{ include (print $.Template.BasePath "/azure-operators-managed-identities-config.configmap.yaml") . | sha256sum }}
checksum/cskv: {{ include (print $.Template.BasePath "/cs-keyvault.secret.yaml") . | sha256sum }}
checksum/provisionshard: {{ include (print $.Template.BasePath "/provisioning-shards.secret.yaml") . | sha256sum }}
checksum/cs: {{ include (print $.Template.BasePath "/clusters-service.secret.yaml") . | sha256sum }}
checksum/runtime: {{ include (print $.Template.BasePath "/azure-runtime-config.configmap.yaml") . | sha256sum }}
spec:
serviceAccount: clusters-service
serviceAccountName: clusters-service
Expand Down
20 changes: 19 additions & 1 deletion cluster-service/deploy/helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -262,4 +262,22 @@ databasePort: "5432"
managedIdentitiesDataPlaneAudienceResource: "https://dummy.org"

# The Azure Operator Managed Identities.
azureOperatorsManagedIdentitiesConfig: ""
azureOperatorsMI:
cloudControllerManager:
roleName: ''
roleId: ''
ingress:
roleName: ''
roleId: ''
diskCsiDriver:
roleName: ''
roleId: ''
fileCsiDriver:
roleName: ''
roleId: ''
imageRegistry:
roleName: ''
roleId: ''
cloudNetworkConfig:
roleName: ''
roleId: ''
87 changes: 87 additions & 0 deletions cluster-service/pipeline.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.ClusterService
rolloutName: Cluster Service Rollout
resourceGroups:
- name: {{ .svc.rg }}
subscription: {{ .svc.subscription }}
aksCluster: {{ .aksName }}
steps:
- name: deploy
action: Shell
command: make deploy
dryRun:
variables:
- name: HELM_DRY_RUN
value: "--dry-run=server --debug"
- name: KUBECTL_DRY_RUN
value: "--dry-run=server"
variables:
- name: REGION
configRef: region
- name: RESOURCEGROUP
configRef: svc.rg
- name: AKS_NAME
configRef: aksName
- name: SERVICE_KV
configRef: serviceKeyVault.name
- name: OIDC_STORAGE_ACCOUNT
configRef: oidcStorageAccountName
- name: IMAGE_REPO
configRef: clusterService.imageRepo
- name: IMAGE_TAG
configRef: clusterService.imageTag
- name: ACR_NAME
configRef: svcAcrName
- name: OCP_ACR_NAME
configRef: ocpAcrName
- name: AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID
configRef: firstPartyAppClientId
- name: FPA_CERT_NAME
value: firstPartyCert
- name: AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID
configRef: miMockPrincipalId
- name: AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID
configRef: miMockClientId
- name: AZURE_ARM_HELPER_IDENTITY_CLIENT_ID
configRef: armHelperClientId
- name: AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID
configRef: armHelperFPAPrincipalId
- name: MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME
value: msiMockCert
- name: ARM_HELPER_CERT_NAME
value: armHelperCert
- name: BASE_DNS_ZONE_NAME
configRef: baseDnsZoneName
- name: REGIONAL_DNS_SUBDOMAIN
configRef: regionalDNSSubdomain
- name: USE_AZURE_DB
configRef: clusterService.postgres.deploy
- name: DATABASE_SERVER_NAME
configRef: clusterService.postgres.name
- name: DEVOPS_MSI_ID
configRef: aroDevopsMsiId
- name: OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.cloudControllerManager.roleName
- name: OP_INGRESS_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.ingress.roleName
- name: OP_DISK_CSI_DRIVER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.diskCsiDriver.roleName
- name: OP_FILE_CSI_DRIVER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.fileCsiDriver.roleName
- name: OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.imageRegistry.roleName
- name: OP_CLOUD_NETWORK_CONFIG_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.cloudNetworkConfig.roleName

# this is maestro consumer registration stuff
# this goes away when we have a real registration process
- name: CONSUMER_NAME
configRef: maestro.consumerName
- name: REGIONAL_RESOURCEGROUP
configRef: regionRG
- name: MGMT_RESOURCEGROUP
configRef: mgmt.rg
- name: CX_SECRETS_KV_NAME
configRef: cxKeyVault.name
- name: CX_MI_KV_NAME
configRef: msiKeyVault.name
13 changes: 13 additions & 0 deletions config/config.msft.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,19 @@ clouds:
clusterService:
imageTag: ecd15ad
imageRepo: app-sre/uhc-clusters-service
azureOperatorsManagedIdentities:
cloudControllerManager:
roleName: Azure Red Hat OpenShift Cloud Controller Manager Role
ingress:
roleName: Azure Red Hat OpenShift Cluster Ingress Operator Role
diskCsiDriver:
roleName: Azure Red Hat OpenShift Disk Storage Operator Role
fileCsiDriver:
roleName: Azure Red Hat OpenShift File Storage Operator Role
imageRegistry:
roleName: Azure Red Hat OpenShift Image Registry Operator Role
cloudNetworkConfig:
roleName: Azure Red Hat OpenShift Network Operator Role
hypershiftOperator:
imageTag: 9aca808
imageSync:
Expand Down
Loading

0 comments on commit 3246099

Please sign in to comment.