From 4f3c39b04ee1a1c5bbb40a7008327fe835867029 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Fri, 22 Nov 2024 11:12:26 +0100 Subject: [PATCH] maestro KV refactor Signed-off-by: Gerd Oberlechner --- config/config.msft.yaml | 3 +- config/config.schema.json | 14 +-- config/config.yaml | 10 ++- config/public-cloud-cs-pr.json | 7 +- config/public-cloud-dev.json | 7 +- config/public-cloud-msft-int.json | 7 +- config/public-cloud-personal-dev.json | 7 +- .../mgmt-cluster.tmpl.bicepparam | 4 +- .../configurations/region.tmpl.bicepparam | 2 +- .../svc-cluster.tmpl.bicepparam | 2 +- dev-infrastructure/mgmt-pipeline.yaml | 27 ++++++ .../modules/key-vault-cert.bicep | 4 +- .../modules/maestro/maestro-access-cert.bicep | 85 +++++++++++++++++++ .../modules/maestro/maestro-consumer.bicep | 40 ++++++--- .../maestro/maestro-eventgrid-access.bicep | 85 ++----------------- .../modules/maestro/maestro-infra.bicep | 84 +----------------- .../modules/maestro/maestro-server.bicep | 43 ++++++---- dev-infrastructure/svc-pipeline.yaml | 2 +- .../templates/mgmt-cluster.bicep | 21 +++-- dev-infrastructure/templates/region.bicep | 13 +-- .../templates/svc-cluster.bicep | 21 +++-- maestro/agent/config.tmpl.mk | 2 +- maestro/server/Makefile | 1 + maestro/server/config.tmpl.mk | 3 +- maestro/server/pipeline.yaml | 16 ++-- 25 files changed, 259 insertions(+), 251 deletions(-) create mode 100644 dev-infrastructure/mgmt-pipeline.yaml create mode 100644 dev-infrastructure/modules/maestro/maestro-access-cert.bicep diff --git a/config/config.msft.yaml b/config/config.msft.yaml index 1480b1f4e..1d46567e6 100644 --- a/config/config.msft.yaml +++ b/config/config.msft.yaml @@ -49,10 +49,10 @@ defaults: # Maestro maestro: - keyVaultName: arohcp-maestro-{{ .ctx.regionShort }} eventGrid: name: arohcp-maestro-{{ .ctx.regionShort }} maxClientSessionsPerAuthName: '4' + private: false certDomain: 'selfsigned.maestro.keyvault.azure.com' postgres: name: arohcp-maestro-{{ .ctx.regionShort }} @@ -63,6 +63,7 @@ defaults: minTLSVersion: 'TLSV1.2' restrictIstioIngress: true consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }} + serverMqttClientName: maestro-server imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro # Cluster Service diff --git a/config/config.schema.json b/config/config.schema.json index 4ef2e75cb..d734856b9 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -222,6 +222,9 @@ "consumerName": { "type": "string" }, + "serverMqttClientName": { + "type": "string" + }, "eventGrid": { "type": "object", "properties": { @@ -230,12 +233,16 @@ }, "name": { "type": "string" + }, + "private": { + "type": "boolean" } }, "additionalProperties": false, "required": [ "maxClientSessionsPerAuthName", - "name" + "name", + "private" ] }, "imageBase": { @@ -244,9 +251,6 @@ "imageTag": { "type": "string" }, - "keyVaultName": { - "type": "string" - }, "postgres": { "type": "object", "properties": { @@ -288,10 +292,10 @@ "required": [ "certDomain", "consumerName", + "serverMqttClientName", "eventGrid", "imageBase", "imageTag", - "keyVaultName", "postgres", "restrictIstioIngress" ] diff --git a/config/config.yaml b/config/config.yaml index 1bcf72f8a..ccc41b05e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -49,10 +49,10 @@ defaults: # Maestro maestro: - keyVaultName: arohcp-maestro-{{ .ctx.regionShort }} eventGrid: name: arohcp-maestro-{{ .ctx.regionShort }} maxClientSessionsPerAuthName: '4' + private: false certDomain: 'selfsigned.maestro.keyvault.azure.com' postgres: name: arohcp-maestro-{{ .ctx.regionShort }} @@ -62,6 +62,7 @@ defaults: private: false minTLSVersion: 'TLSV1.2' restrictIstioIngress: true + serverMqttClientName: maestro-server consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }} imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro @@ -203,6 +204,9 @@ clouds: maxCount: 12 # DNS regionalDNSSubdomain: '{{ .ctx.region }}' + # Maestro + maestro: + serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}-dev' # Frontend frontend: cosmosDB: @@ -229,6 +233,7 @@ clouds: # Maestro maestro: restrictIstioIngress: false + serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}-cs' # Frontend frontend: cosmosDB: @@ -242,6 +247,9 @@ clouds: deploy: false # DNS regionalDNSSubdomain: '{{ .ctx.regionShort }}' + # Maestro + maestro: + serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}' # Frontend frontend: cosmosDB: diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index 08cb94bc3..4a35dff7c 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -58,11 +58,11 @@ "consumerName": "hcp-underlay-cspr-mgmt-1", "eventGrid": { "maxClientSessionsPerAuthName": "4", - "name": "arohcp-maestro-cspr" + "name": "arohcp-maestro-cspr", + "private": false }, "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", "imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f", - "keyVaultName": "arohcp-maestro-cspr", "postgres": { "deploy": false, "minTLSVersion": "TLSV1.2", @@ -71,7 +71,8 @@ "serverStorageSizeGB": "32", "serverVersion": "15" }, - "restrictIstioIngress": false + "restrictIstioIngress": false, + "serverMqttClientName": "maestro-server-cspr-cs" }, "mgmt": { "clusterServicePrincipalId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-cspr-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service", diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index 43639f1e4..9c71d444c 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -58,11 +58,11 @@ "consumerName": "hcp-underlay-dev-mgmt-1", "eventGrid": { "maxClientSessionsPerAuthName": "4", - "name": "arohcp-maestro-dev" + "name": "arohcp-maestro-dev", + "private": false }, "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", "imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f", - "keyVaultName": "arohcp-maestro-dev", "postgres": { "deploy": false, "minTLSVersion": "TLSV1.2", @@ -71,7 +71,8 @@ "serverStorageSizeGB": "32", "serverVersion": "15" }, - "restrictIstioIngress": true + "restrictIstioIngress": true, + "serverMqttClientName": "maestro-server-dev-dev" }, "mgmt": { "clusterServicePrincipalId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-dev-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service", diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json index 7ee97b12c..64e1fe688 100644 --- a/config/public-cloud-msft-int.json +++ b/config/public-cloud-msft-int.json @@ -58,11 +58,11 @@ "consumerName": "hcp-underlay-int-mgmt-1", "eventGrid": { "maxClientSessionsPerAuthName": "4", - "name": "arohcp-maestro-int" + "name": "arohcp-maestro-int", + "private": false }, "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", "imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f", - "keyVaultName": "arohcp-maestro-int", "postgres": { "deploy": false, "minTLSVersion": "TLSV1.2", @@ -71,7 +71,8 @@ "serverStorageSizeGB": "32", "serverVersion": "15" }, - "restrictIstioIngress": true + "restrictIstioIngress": true, + "serverMqttClientName": "maestro-server" }, "mgmt": { "clusterServicePrincipalId": "todo", diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index ffee39632..a715c0d8d 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -58,11 +58,11 @@ "consumerName": "hcp-underlay-usw3tst-mgmt-1", "eventGrid": { "maxClientSessionsPerAuthName": "4", - "name": "arohcp-maestro-usw3tst" + "name": "arohcp-maestro-usw3tst", + "private": false }, "imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro", "imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f", - "keyVaultName": "arohcp-maestro-usw3tst", "postgres": { "deploy": false, "minTLSVersion": "TLSV1.2", @@ -71,7 +71,8 @@ "serverStorageSizeGB": "32", "serverVersion": "15" }, - "restrictIstioIngress": true + "restrictIstioIngress": true, + "serverMqttClientName": "maestro-server-usw3tst" }, "mgmt": { "clusterServicePrincipalId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-usw3tst-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service", diff --git a/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam index 47f2df1f0..316d93108 100644 --- a/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam +++ b/dev-infrastructure/configurations/mgmt-cluster.tmpl.bicepparam @@ -20,7 +20,6 @@ param userAgentPoolAZCount = {{ .mgmt.userAgentPool.azCount }} // Maestro param maestroConsumerName = '{{ .maestro.consumerName }}' -param maestroKeyVaultName = '{{ .maestro.keyVaultName }}' param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}' param maestroCertDomain = '{{ .maestro.certDomain }}' @@ -56,3 +55,6 @@ param mgmtKeyVaultSoftDelete = {{ .mgmtKeyVault.softDelete }} // Cluster Service identity // used for Key Vault access param clusterServicePrincipalId = '{{ .mgmt.clusterServicePrincipalId }}' + +// MI for deployment scripts +param aroDevopsMsiId = '{{ .aroDevopsMsiId }}' diff --git a/dev-infrastructure/configurations/region.tmpl.bicepparam b/dev-infrastructure/configurations/region.tmpl.bicepparam index e0cb3940b..65ed549df 100644 --- a/dev-infrastructure/configurations/region.tmpl.bicepparam +++ b/dev-infrastructure/configurations/region.tmpl.bicepparam @@ -6,6 +6,6 @@ param baseDNSZoneResourceGroup = '{{ .baseDnsZoneRG }}' param regionalDNSSubdomain = '{{ .regionalDNSSubdomain }}' // maestro -param maestroKeyVaultName = '{{ .maestro.keyVaultName }}' param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}' param maestroEventGridMaxClientSessionsPerAuthName = {{ .maestro.eventGrid.maxClientSessionsPerAuthName }} +param maestroEventGridPrivate = {{ .maestro.eventGrid.private }} diff --git a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam index 480056259..077fca8d9 100644 --- a/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam +++ b/dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam @@ -20,8 +20,8 @@ param deployFrontendCosmos = {{ .frontend.cosmosDB.deploy }} param rpCosmosDbName = '{{ .frontend.cosmosDB.name }}' param rpCosmosDbPrivate = {{ .frontend.cosmosDB.private }} -param maestroKeyVaultName = '{{ .maestro.keyVaultName }}' param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}' +param maestroServerMqttClientName = '{{ .maestro.serverMqttClientName }}' param maestroCertDomain = '{{ .maestro.certDomain}}' param maestroPostgresServerName = '{{ .maestro.postgres.name }}' param maestroPostgresServerMinTLSVersion = '{{ .maestro.postgres.minTLSVersion }}' diff --git a/dev-infrastructure/mgmt-pipeline.yaml b/dev-infrastructure/mgmt-pipeline.yaml new file mode 100644 index 000000000..07d998bb7 --- /dev/null +++ b/dev-infrastructure/mgmt-pipeline.yaml @@ -0,0 +1,27 @@ +serviceGroup: Microsoft.Azure.ARO.Test +rolloutName: Management Cluster Rollout +resourceGroups: +- name: {{ .mgmt.rg }} + subscription: {{ .mgmt.subscription }} + aksCluster: {{ .aksName }} + steps: + - name: mgmt + action: ARM + template: templates/mgmt-cluster.bicep + parameters: configurations/mgmt-cluster.tmpl.bicepparam + - name: enable-metrics + action: Shell + command: ["scripts/enable-aks-metrics.sh"] + env: + - name: RESOURCEGROUP + configRef: mgmt.rg + - name: AKS_NAME + configRef: aksName + - name: GRAFANA_RESOURCEGROUP + configRef: regionRG + - name: MONITORING_WORKSPACE_NAME + configRef: monitoring.workspaceName + - name: GRAFANA_NAME + configRef: monitoring.grafanaName + dependsOn: + - mgmt diff --git a/dev-infrastructure/modules/key-vault-cert.bicep b/dev-infrastructure/modules/key-vault-cert.bicep index 7225b0fd3..b8dfa91eb 100644 --- a/dev-infrastructure/modules/key-vault-cert.bicep +++ b/dev-infrastructure/modules/key-vault-cert.bicep @@ -14,7 +14,7 @@ param issuerName string param dnsNames array param now string = utcNow('F') param keyVaultManagedIdentityId string -param location string +param location string = resourceGroup().location param force bool = false var boolstring = force == false ? '$false' : '$true' param validityInMonths int = 12 @@ -30,7 +30,7 @@ resource newCertwithRotationKV 'Microsoft.Resources/deploymentScripts@2023-08-01 location: location kind: 'AzurePowerShell' properties: { - azPowerShellVersion: '7.5.0' + azPowerShellVersion: '12.0.0' arguments: ' -VaultName ${keyVaultName} -ValidityInMonths ${validityInMonths} -IssuerName ${issuerName} -CertName ${certName} -SubjectName ${subjectName} -DnsNames ${join(dnsNames,'_')} -Force ${boolstring}' scriptContent: loadTextContent('../scripts/key-vault-cert.ps1') forceUpdateTag: now diff --git a/dev-infrastructure/modules/maestro/maestro-access-cert.bicep b/dev-infrastructure/modules/maestro/maestro-access-cert.bicep new file mode 100644 index 000000000..8df4b260b --- /dev/null +++ b/dev-infrastructure/modules/maestro/maestro-access-cert.bicep @@ -0,0 +1,85 @@ +/* +Creates a certificate in Key Vault signed by the specified issuer. +For dev environments `Self` is used as issuer, for higher environments +OneCertV2 Private will be used. + +The specified managed identity `certificateAccessManagedIdentityPrincipalId` +is granted access to the certificate in Key Vault. This will be leveraged +with CSI secret store to access the certificate from the maestro pods. + +Execution scope: the resourcegroup of the Key Vault where the certificate will be stored +*/ + +@description('The Key Vault where the certificate for Event Grid access will be stored') +param keyVaultName string + +@description('The managed identity that will be used to manage the certificate in Key Vault') +param kvCertOfficerManagedIdentityResourceId string + +@description('The base domain name to be used for the certificates DNS name.') +param certDomain string + +@description('The name of the client that will be created in the EventGrid Namespace') +param clientName string + +@description('The name of the certificate in Key Vault.') +param keyVaultCertificateName string + +@description('The issuer of the certificate.') +param certificateIssuer string = 'Self' + +@description('Grant this managed identity access to the certificate in Key Vault.') +param certificateAccessManagedIdentityPrincipalId string + +// +// C E R T I F I C A T E C R E A T I O N +// + +var clientAuthenticationName = '${clientName}.${certDomain}' + +module clientCertificate '../key-vault-cert.bicep' = { + name: '${clientName}-client-cert' + params: { + keyVaultName: keyVaultName + subjectName: 'CN=${clientName}' + certName: keyVaultCertificateName + keyVaultManagedIdentityId: kvCertOfficerManagedIdentityResourceId + dnsNames: [ + clientAuthenticationName + ] + issuerName: certificateIssuer + } +} + +// +// C E R T I F I C A T E A C C E S S P E R M I S S I O N +// + +var keyVaultSecretUserRoleId = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions/', + '4633458b-17de-408a-b874-0445c86b69e6' +) + +resource kv 'Microsoft.KeyVault/vaults@2023-07-01' existing = { + name: keyVaultName +} + +// grant permissions on the secret that contains the certificate + +resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' existing = { + parent: kv + name: keyVaultCertificateName +} + +resource secretAccessPermission 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: secret + name: guid(certificateAccessManagedIdentityPrincipalId, keyVaultSecretUserRoleId, kv.id, keyVaultCertificateName) + properties: { + roleDefinitionId: keyVaultSecretUserRoleId + principalId: certificateAccessManagedIdentityPrincipalId + principalType: 'ServicePrincipal' + } +} + +output certificateThumbprint string = clientCertificate.outputs.Thumbprint +output certificateSAN string = clientAuthenticationName diff --git a/dev-infrastructure/modules/maestro/maestro-consumer.bicep b/dev-infrastructure/modules/maestro/maestro-consumer.bicep index b7b1beeff..fb585eb9e 100644 --- a/dev-infrastructure/modules/maestro/maestro-consumer.bicep +++ b/dev-infrastructure/modules/maestro/maestro-consumer.bicep @@ -1,25 +1,41 @@ -param maestroServerManagedIdentityPrincipalId string +/* +This module is responsible for setting up EventGrid access for the maestro consumer by +- create a client certificate +- register the client in the EventGrid namespace + +Execution scope: the resourcegroup of the MC where the agent is deployed. +*/ + +param maestroAgentManagedIdentityPrincipalId string @minLength(1) param maestroConsumerName string param maestroInfraResourceGroup string param maestroEventGridNamespaceName string -param maestroKeyVaultName string -param maestroKeyVaultOfficerManagedIdentityName string -param maestroKeyVaultCertificateDomain string -param location string +param certKeyVaultName string +param keyVaultOfficerManagedIdentityName string +param maestroCertificateDomain string + +module eventGridClientCert 'maestro-access-cert.bicep' = { + name: '${deployment().name}-eg-crt-${uniqueString(maestroConsumerName)}' + params: { + keyVaultName: certKeyVaultName + kvCertOfficerManagedIdentityResourceId: keyVaultOfficerManagedIdentityName + certDomain: maestroCertificateDomain + clientName: maestroConsumerName + keyVaultCertificateName: maestroConsumerName + certificateAccessManagedIdentityPrincipalId: maestroAgentManagedIdentityPrincipalId + } +} -module evengGridAccess './maestro-eventgrid-access.bicep' = { - name: 'event-grid-access-${uniqueString(maestroConsumerName)}' +module evengGridAccess 'maestro-eventgrid-access.bicep' = { + name: '${deployment().name}-eg-access' scope: resourceGroup(maestroInfraResourceGroup) params: { eventGridNamespaceName: maestroEventGridNamespaceName - keyVaultName: maestroKeyVaultName - kvCertOfficerManagedIdentityName: maestroKeyVaultOfficerManagedIdentityName - certDomain: maestroKeyVaultCertificateDomain clientName: maestroConsumerName clientRole: 'consumer' - certificateAccessManagedIdentityPrincipalId: maestroServerManagedIdentityPrincipalId - location: location + certificateThumbprint: eventGridClientCert.outputs.certificateThumbprint + certificateSAN: eventGridClientCert.outputs.certificateSAN } } diff --git a/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep b/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep index c3b982f4c..3a8868011 100644 --- a/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep +++ b/dev-infrastructure/modules/maestro/maestro-eventgrid-access.bicep @@ -2,33 +2,18 @@ This module manages access to EventGrid for a maestro client, which can be the server or a consumer. -- Creates a certificate in Key Vault signed by the specified issuer. - For dev environments `Self` is used as issuer, for higher environments - OneCertV2 Private will be used. - An MQTT client is registered within eventgrid. Depending on the certificate issuer, the certificate validation schema will be thumbprint based for self-signed certificates and DNS based for OneCertV2 Private certificates. - The MQTT client is placed into the right MQTT client group based on the client role. This defines the topic access permissions for the client. -- The specified managed identity `certificateAccessManagedIdentityPrincipalId` - is granted access to the certificate in Key Vault. This will be leveraged - with CSI secret store to access the certificate from the maestro pods. -Execution scope: the resourcegroup of the maestro infrastructure +Execution scope: the resourcegroup of the eventgrid namespace instance */ @description('The EventGrid Namespace name where access will be managed') param eventGridNamespaceName string -@description('The Key Vault name where the certificate for Event Grid access will be stored') -param keyVaultName string - -@description('The name of the managed identity that will be used to manage the certificate in Key Vault') -param kvCertOfficerManagedIdentityName string - -@description('The base domain name to be used for the certificates DNS name.') -param certDomain string - @description('The name of the client that will be created in the EventGrid Namespace') param clientName string @@ -39,42 +24,20 @@ param clientName string ]) param clientRole string -@description('Grant this managed identity access to the certificate in Key Vault.') -param certificateAccessManagedIdentityPrincipalId string - @description('The issuer of the certificate.') param certificateIssuer string = 'Self' -param location string +@description('The thumbprint of the certificate that should get access. Dont use in production') +param certificateThumbprint string -var clientAuthenticationName = '${clientName}.${certDomain}' +@description('The subject alternative name of the certificate') +param certificateSAN string resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2023-12-15-preview' existing = { name: eventGridNamespaceName } -resource kvCertOfficerManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: kvCertOfficerManagedIdentityName -} - -// certificate for MQTT authentication -module clientCertificate '../key-vault-cert.bicep' = { - name: '${deployment().name}-client-cert' - params: { - location: location - keyVaultName: keyVaultName - subjectName: 'CN=${clientName}' - certName: clientName - keyVaultManagedIdentityId: kvCertOfficerManagedIdentity.id - dnsNames: [ - clientAuthenticationName - ] - // todo - use Private OnceCertV2 in higher environments - issuerName: certificateIssuer - } -} - -// D O N ' T U S E T H I S I N P R O D U C T I O N +// D O N ' T U S E T H U M B P R I N T I N P R O D U C T I O N // eventgrid MQTT client trusting the certificate by thumbprint if // Key Vault self-signed certificates are used. trusting self-signed certificates // as CAs is not supported in EventGrid @@ -82,14 +45,14 @@ resource mqttClient 'Microsoft.EventGrid/namespaces/clients@2023-12-15-preview' name: clientName parent: eventGridNamespace properties: { - authenticationName: clientAuthenticationName + authenticationName: certificateSAN attributes: { role: clientRole consumer_name: clientName } clientCertificateAuthentication: { allowedThumbprints: [ - clientCertificate.outputs.Thumbprint + certificateThumbprint ] validationScheme: 'ThumbprintMatch' } @@ -99,35 +62,3 @@ resource mqttClient 'Microsoft.EventGrid/namespaces/clients@2023-12-15-preview' // TODO - implement issuer CA registration with EventGrid + register the mqtt client with // the DnsMatchesAuthenticationName authentication validation scheme - -var keyVaultSecretUserRoleId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions/', - '4633458b-17de-408a-b874-0445c86b69e6' -) - -resource kv 'Microsoft.KeyVault/vaults@2023-07-01' existing = { - name: keyVaultName -} - -// grant permissions on the secret that contains the certificate - -resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' existing = { - parent: kv - name: clientName -} - -resource secretAccessPermission 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: secret - name: guid(certificateAccessManagedIdentityPrincipalId, keyVaultSecretUserRoleId, kv.id) - properties: { - roleDefinitionId: keyVaultSecretUserRoleId - principalId: certificateAccessManagedIdentityPrincipalId - principalType: 'ServicePrincipal' - } -} - -// output - -output KeyVaultCertId string = clientCertificate.outputs.KeyVaultCertId -output KeyVaultCertName string = clientName -output EventGridHostname string = eventGridNamespace.properties.topicSpacesConfiguration.hostname diff --git a/dev-infrastructure/modules/maestro/maestro-infra.bicep b/dev-infrastructure/modules/maestro/maestro-infra.bicep index 24ae8cec2..780760b6b 100644 --- a/dev-infrastructure/modules/maestro/maestro-infra.bicep +++ b/dev-infrastructure/modules/maestro/maestro-infra.bicep @@ -1,23 +1,11 @@ /* This module creates the infrastructure required by maestro to run. This includes: -- A KeyVault where the client certificates for EventGrid MQTT broker access - are generated and stored -- A managed identity to create and manage certificates in Key Vault. This is - used by the maestro-eventgrid-access bicep module deploymentscripts. - - Why is this needed? There are no bicep modules for KeyVault certificate management, - so we need deploymentscripts + a managed identity with Key Vault access to run them. - - Create an EventGrid namespaces instance with MQTT enabled. - Create EventGrid client groups for the server and consumers and define topic access permissions. Execution scope: the resourcegroup of the maestro infrastructure - -TODO: -- Key Vault network access restrictions (e.g. privatelink) -- EventGrid network access restrictions (e.g. privatelink) */ @description('The Maestro Event Grid Namespaces name') @@ -29,79 +17,12 @@ param location string @description('The maximum client sessions per authentication name for the EventGrid MQTT broker') param maxClientSessionsPerAuthName int -@description('The name for the Key Vault for Maestro certificates') -param maestroKeyVaultName string - -@description('The name for the Managed Identity that will be created for Key Vault Certificate management.') -param kvCertOfficerManagedIdentityName string - @description('Allow public network access to the EventGrid Namespace') @allowed([ 'Enabled' 'Disabled' ]) -param publicNetworkAccess string = 'Enabled' - -// -// K E Y V A U L T -// - -resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = { - name: maestroKeyVaultName - location: location - tags: { - resourceGroup: resourceGroup().name - } - properties: { - accessPolicies: [] - enableRbacAuthorization: true - enabledForDeployment: false - enabledForDiskEncryption: false - enabledForTemplateDeployment: false - enableSoftDelete: false - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Allow' - ipRules: [ - { - // TODO: restrict in higher environments - value: '0.0.0.0/0' - } - ] - } - // TODO: disabled in higher environments - publicNetworkAccess: 'Enabled' - sku: { - family: 'A' - name: 'standard' - } - tenantId: subscription().tenantId - } -} - -// -// C E R T I F I C A T E O F F I C E R M S I -// - -resource kvCertOfficerManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: kvCertOfficerManagedIdentityName - location: location -} - -var keyVaultCertificateOfficerRoleId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions/', - 'a4417e6f-fecd-4de8-b567-7b0420556985' -) - -resource kvManagedIdentityRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: kv - name: guid(kvCertOfficerManagedIdentity.id, keyVaultCertificateOfficerRoleId, kv.id) - properties: { - roleDefinitionId: keyVaultCertificateOfficerRoleId - principalId: kvCertOfficerManagedIdentity.properties.principalId - principalType: 'ServicePrincipal' - } -} +param publicNetworkAccess string // // E V E N T G R I D @@ -253,6 +174,3 @@ resource maestroConsumersPublishTopicspacePermissionBinding 'Microsoft.EventGrid topicSpaceName: maestroConsumersPublishTopicspace.name } } - -output keyVaultName string = kv.name -output eventGridNamespaceName string = eventGridNamespace.name diff --git a/dev-infrastructure/modules/maestro/maestro-server.bicep b/dev-infrastructure/modules/maestro/maestro-server.bicep index 097124d0e..ebfda3ab3 100644 --- a/dev-infrastructure/modules/maestro/maestro-server.bicep +++ b/dev-infrastructure/modules/maestro/maestro-server.bicep @@ -1,20 +1,21 @@ /* This module is responsible for: + - setting up Postgres access for the maestro server - setting up EventGrid access for the maestro server Execution scope: the resourcegroup of the AKS cluster where the maestro server will be deployed. - -TODO: -- once Key Vault and EventGrid have network access restrictions enabled, - this module needs to be enhanced to manage access to both (e.g. privatelink) */ param maestroInfraResourceGroup string param maestroEventGridNamespaceName string -param maestroKeyVaultName string -param maestroKeyVaultOfficerManagedIdentityName string -param maestroKeyVaultCertificateDomain string +param certKeyVaultName string +param certKeyVaultResourceGroup string +param keyVaultOfficerManagedIdentityName string +param maestroCertificateDomain string + +@description('The name of the MQTT client that will be created in the EventGrid Namespace') +param mqttClientName string @description('Whether to deploy the Postgres server for Maestro') param deployPostgres bool @@ -131,20 +132,30 @@ module csManagedIdentityDatabaseAccess '../postgres/postgres-access.bicep' = if } // -// E V E N T G R I D +// E V E N T G R I D A C C E S S // -module evengGridAccess './maestro-eventgrid-access.bicep' = { - name: '${deployment().name}-event-grid-access' +module eventGridClientCert 'maestro-access-cert.bicep' = { + name: '${deployment().name}-eg-crt-${uniqueString(mqttClientName)}' + scope: resourceGroup(certKeyVaultResourceGroup) + params: { + keyVaultName: certKeyVaultName + kvCertOfficerManagedIdentityResourceId: keyVaultOfficerManagedIdentityName + certDomain: maestroCertificateDomain + clientName: mqttClientName + keyVaultCertificateName: mqttClientName + certificateAccessManagedIdentityPrincipalId: maestroServerManagedIdentityPrincipalId + } +} + +module evengGridAccess 'maestro-eventgrid-access.bicep' = { + name: '${deployment().name}-eg-access' scope: resourceGroup(maestroInfraResourceGroup) params: { eventGridNamespaceName: maestroEventGridNamespaceName - keyVaultName: maestroKeyVaultName - kvCertOfficerManagedIdentityName: maestroKeyVaultOfficerManagedIdentityName - certDomain: maestroKeyVaultCertificateDomain - clientName: 'maestro-server' + clientName: mqttClientName clientRole: 'server' - certificateAccessManagedIdentityPrincipalId: maestroServerManagedIdentityPrincipalId - location: location + certificateThumbprint: eventGridClientCert.outputs.certificateThumbprint + certificateSAN: eventGridClientCert.outputs.certificateSAN } } diff --git a/dev-infrastructure/svc-pipeline.yaml b/dev-infrastructure/svc-pipeline.yaml index bb1091804..743b0415a 100644 --- a/dev-infrastructure/svc-pipeline.yaml +++ b/dev-infrastructure/svc-pipeline.yaml @@ -11,7 +11,7 @@ resourceGroups: parameters: configurations/svc-cluster.tmpl.bicepparam - name: enable-metrics action: Shell - command: ["/bin/bash", "-c", "scripts/enable-aks-metrics.sh"] + command: ["scripts/enable-aks-metrics.sh"] env: - name: RESOURCEGROUP configRef: svc.rg diff --git a/dev-infrastructure/templates/mgmt-cluster.bicep b/dev-infrastructure/templates/mgmt-cluster.bicep index 75839fd70..2a31f4f9f 100644 --- a/dev-infrastructure/templates/mgmt-cluster.bicep +++ b/dev-infrastructure/templates/mgmt-cluster.bicep @@ -74,12 +74,6 @@ param maestroConsumerName string @description('The domain to use to use for the maestro certificate. Relevant only for environments where OneCert can be used.') param maestroCertDomain string -@description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -param maestroKeyVaultName string - -@description('The name of the managed identity that will manage certificates in maestros keyvault.') -param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-officer-msi' - @description('The name of the eventgrid namespace for Maestro.') param maestroEventGridNamespacesName string @@ -119,6 +113,9 @@ param mgmtKeyVaultSoftDelete bool @description('Cluster user assigned identity principal id, used to grant KeyVault access') param clusterServicePrincipalId string +@description('MSI that will be used to run deploymentScripts') +param aroDevopsMsiId string + // Tags the resource group resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { name: 'default' @@ -180,18 +177,20 @@ output aksClusterName string = mgmtCluster.outputs.aksClusterName module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = { name: 'maestro-consumer' params: { - maestroServerManagedIdentityPrincipalId: filter( + maestroAgentManagedIdentityPrincipalId: filter( mgmtCluster.outputs.userAssignedIdentities, id => id.uamiName == 'maestro-consumer' )[0].uamiPrincipalID maestroInfraResourceGroup: regionalResourceGroup maestroConsumerName: maestroConsumerName maestroEventGridNamespaceName: maestroEventGridNamespacesName - maestroKeyVaultName: maestroKeyVaultName - maestroKeyVaultOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName - maestroKeyVaultCertificateDomain: maestroCertDomain - location: location + certKeyVaultName: mgmtKeyVaultName + keyVaultOfficerManagedIdentityName: aroDevopsMsiId + maestroCertificateDomain: maestroCertDomain } + dependsOn: [ + mgmtKeyVault + ] } // diff --git a/dev-infrastructure/templates/region.bicep b/dev-infrastructure/templates/region.bicep index ae4b8d8ef..7c88b7072 100644 --- a/dev-infrastructure/templates/region.bicep +++ b/dev-infrastructure/templates/region.bicep @@ -1,18 +1,15 @@ @description('Azure Region Location') param location string = resourceGroup().location -@description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -param maestroKeyVaultName string - -@description('The name of the managed identity that will manage certificates in maestros keyvault.') -param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-officer-msi' - @description('The name of the eventgrid namespace for Maestro.') param maestroEventGridNamespacesName string @description('The maximum client sessions per authentication name for the EventGrid MQTT broker') param maestroEventGridMaxClientSessionsPerAuthName int +@description('Allow/deny public network access to the Maestro EventGrid Namespace') +param maestroEventGridPrivate bool + @description('Set to true to prevent resources from being pruned after 48 hours') param persist bool = false @@ -64,8 +61,6 @@ module maestroInfra '../modules/maestro/maestro-infra.bicep' = { eventGridNamespaceName: maestroEventGridNamespacesName location: location maxClientSessionsPerAuthName: maestroEventGridMaxClientSessionsPerAuthName - maestroKeyVaultName: maestroKeyVaultName - kvCertOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName - publicNetworkAccess: 'Enabled' + publicNetworkAccess: maestroEventGridPrivate ? 'Disabled' : 'Enabled' } } diff --git a/dev-infrastructure/templates/svc-cluster.bicep b/dev-infrastructure/templates/svc-cluster.bicep index 3aa979827..69edf4d99 100644 --- a/dev-infrastructure/templates/svc-cluster.bicep +++ b/dev-infrastructure/templates/svc-cluster.bicep @@ -75,12 +75,6 @@ param maestroCertDomain string @description('The name of the eventgrid namespace for Maestro.') param maestroEventGridNamespacesName string -@description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -param maestroKeyVaultName string - -@description('The name of the managed identity that will manage certificates in maestros keyvault.') -param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-officer-msi' - @description('Deploy ARO HCP CS Infrastructure if true') param deployCsInfra bool @@ -113,6 +107,9 @@ param maestroPostgresServerMinTLSVersion string @description('The size of the Postgres server for Maestro') param maestroPostgresServerStorageSizeGB int +@description('The name of Maestro Server MQTT client') +param maestroServerMqttClientName string + @description('The name of the service keyvault') param serviceKeyVaultName string @@ -243,9 +240,11 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = { params: { maestroInfraResourceGroup: regionalResourceGroup maestroEventGridNamespaceName: maestroEventGridNamespacesName - maestroKeyVaultName: maestroKeyVaultName - maestroKeyVaultOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName - maestroKeyVaultCertificateDomain: maestroCertDomain + mqttClientName: maestroServerMqttClientName + certKeyVaultName: serviceKeyVaultName + certKeyVaultResourceGroup: serviceKeyVaultResourceGroup + keyVaultOfficerManagedIdentityName: aroDevopsMsiId + maestroCertificateDomain: maestroCertDomain deployPostgres: deployMaestroPostgres postgresServerName: maestroPostgresServerName postgresServerVersion: maestroPostgresServerVersion @@ -264,6 +263,9 @@ module maestroServer '../modules/maestro/maestro-server.bicep' = { )[0].uamiName location: location } + dependsOn: [ + serviceKeyVault + ] } // @@ -433,6 +435,7 @@ resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2024-06-01-preview' scope: resourceGroup(regionalResourceGroup) } +// todo manage only if maestro.eventgrid is not set to private module eventGrindPrivateEndpoint '../modules/private-endpoint.bicep' = { name: 'eventGridPrivateEndpoint' params: { diff --git a/maestro/agent/config.tmpl.mk b/maestro/agent/config.tmpl.mk index afae37b87..cec7dad35 100644 --- a/maestro/agent/config.tmpl.mk +++ b/maestro/agent/config.tmpl.mk @@ -2,6 +2,6 @@ EVENTGRID_NAME ?= {{ .maestro.eventGrid.name }} REGION_RG ?= {{ .regionRG }} MGMT_RG ?= {{ .mgmt.rg }} CONSUMER_NAME ?= {{ .maestro.consumerName }} -KEYVAULT_NAME ?= {{ .maestro.keyVaultName }} +KEYVAULT_NAME ?= {{ .mgmtKeyVault.name }} IMAGE_BASE ?= {{ .maestro.imageBase }} IMAGE_TAG ?= {{ .maestro.imageTag }} diff --git a/maestro/server/Makefile b/maestro/server/Makefile index 2f97063fb..13ea98c2f 100644 --- a/maestro/server/Makefile +++ b/maestro/server/Makefile @@ -14,6 +14,7 @@ deploy: --namespace maestro \ --set broker.host=$${EVENTGRID_HOSTNAME} \ --set credsKeyVault.name=${KEYVAULT_NAME} \ + --set credsKeyVault.secret=${MQTT_CLIENT_NAME} \ --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ --set azure.tenantId=$${TENANT_ID} \ --set istio.restrictIngress=${ISTIO_RESTRICT_INGRESS} \ diff --git a/maestro/server/config.tmpl.mk b/maestro/server/config.tmpl.mk index 316c3b7fc..fb67afeac 100644 --- a/maestro/server/config.tmpl.mk +++ b/maestro/server/config.tmpl.mk @@ -7,4 +7,5 @@ IMAGE_TAG ?= {{ .maestro.imageTag }} USE_CONTAINERIZED_DB ?= {{ not .maestro.postgres.deploy }} USE_DATABASE_SSL ?= {{ ternary "enable" "disable" .maestro.postgres.deploy }} ISTIO_RESTRICT_INGRESS ?= {{ .maestro.restrictIstioIngress }} -KEYVAULT_NAME ?= {{ .maestro.keyVaultName }} +KEYVAULT_NAME ?= {{ .serviceKeyVault.name }} +MQTT_CLIENT_NAME ?= {{ .maestro.serverMqttClientName }} diff --git a/maestro/server/pipeline.yaml b/maestro/server/pipeline.yaml index 500802b76..eefadb3ce 100644 --- a/maestro/server/pipeline.yaml +++ b/maestro/server/pipeline.yaml @@ -11,25 +11,27 @@ resourceGroups: # parameters: test.bicepparam - name: deploy action: Shell - command: ["/bin/bash", "-c", "make deploy"] + command: ["make", "deploy"] # pwd is inferred from the location of this file # so all path references in the command are relativ to the files location env: - name: EVENTGRID_NAME - configRef: maestroEventgridName + configRef: maestro.eventGrid.name - name: REGION_RG configRef: regionRG - name: SVC_RG - configRef: serviceClusterRG + configRef: svc.rg - name: AKS_NAME configRef: aksName - name: IMAGE_BASE - configRef: maestroImageBase + configRef: maestro.imageBase - name: IMAGE_TAG - configRef: maestroImageTag + configRef: maestro.imageTag - name: USE_AZURE_DB configRef: maestroPostgresDeploy - name: ISTIO_RESTRICT_INGRESS - configRef: maestroRestrictIstioIngress + configRef: maestro.restrictIstioIngress - name: KEYVAULT_NAME - configRef: maestroKeyVaultName + configRef: serviceKeyVault.name + - name: MQTT_CLIENT_NAME + configRef: maestro.serverMqttClientName