Skip to content

Commit

Permalink
maestro KV refactor
Browse files Browse the repository at this point in the history
Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
geoberle committed Nov 22, 2024
1 parent 6f408ea commit 4f3c39b
Show file tree
Hide file tree
Showing 25 changed files with 259 additions and 251 deletions.
3 changes: 2 additions & 1 deletion config/config.msft.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ defaults:

# Maestro
maestro:
keyVaultName: arohcp-maestro-{{ .ctx.regionShort }}
eventGrid:
name: arohcp-maestro-{{ .ctx.regionShort }}
maxClientSessionsPerAuthName: '4'
private: false
certDomain: 'selfsigned.maestro.keyvault.azure.com'
postgres:
name: arohcp-maestro-{{ .ctx.regionShort }}
Expand All @@ -63,6 +63,7 @@ defaults:
minTLSVersion: 'TLSV1.2'
restrictIstioIngress: true
consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
serverMqttClientName: maestro-server
imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro

# Cluster Service
Expand Down
14 changes: 9 additions & 5 deletions config/config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,9 @@
"consumerName": {
"type": "string"
},
"serverMqttClientName": {
"type": "string"
},
"eventGrid": {
"type": "object",
"properties": {
Expand All @@ -230,12 +233,16 @@
},
"name": {
"type": "string"
},
"private": {
"type": "boolean"
}
},
"additionalProperties": false,
"required": [
"maxClientSessionsPerAuthName",
"name"
"name",
"private"
]
},
"imageBase": {
Expand All @@ -244,9 +251,6 @@
"imageTag": {
"type": "string"
},
"keyVaultName": {
"type": "string"
},
"postgres": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -288,10 +292,10 @@
"required": [
"certDomain",
"consumerName",
"serverMqttClientName",
"eventGrid",
"imageBase",
"imageTag",
"keyVaultName",
"postgres",
"restrictIstioIngress"
]
Expand Down
10 changes: 9 additions & 1 deletion config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ defaults:

# Maestro
maestro:
keyVaultName: arohcp-maestro-{{ .ctx.regionShort }}
eventGrid:
name: arohcp-maestro-{{ .ctx.regionShort }}
maxClientSessionsPerAuthName: '4'
private: false
certDomain: 'selfsigned.maestro.keyvault.azure.com'
postgres:
name: arohcp-maestro-{{ .ctx.regionShort }}
Expand All @@ -62,6 +62,7 @@ defaults:
private: false
minTLSVersion: 'TLSV1.2'
restrictIstioIngress: true
serverMqttClientName: maestro-server
consumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }}
imageBase: quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro

Expand Down Expand Up @@ -203,6 +204,9 @@ clouds:
maxCount: 12
# DNS
regionalDNSSubdomain: '{{ .ctx.region }}'
# Maestro
maestro:
serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}-dev'
# Frontend
frontend:
cosmosDB:
Expand All @@ -229,6 +233,7 @@ clouds:
# Maestro
maestro:
restrictIstioIngress: false
serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}-cs'
# Frontend
frontend:
cosmosDB:
Expand All @@ -242,6 +247,9 @@ clouds:
deploy: false
# DNS
regionalDNSSubdomain: '{{ .ctx.regionShort }}'
# Maestro
maestro:
serverMqttClientName: 'maestro-server-{{ .ctx.regionShort }}'
# Frontend
frontend:
cosmosDB:
Expand Down
7 changes: 4 additions & 3 deletions config/public-cloud-cs-pr.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@
"consumerName": "hcp-underlay-cspr-mgmt-1",
"eventGrid": {
"maxClientSessionsPerAuthName": "4",
"name": "arohcp-maestro-cspr"
"name": "arohcp-maestro-cspr",
"private": false
},
"imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
"imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f",
"keyVaultName": "arohcp-maestro-cspr",
"postgres": {
"deploy": false,
"minTLSVersion": "TLSV1.2",
Expand All @@ -71,7 +71,8 @@
"serverStorageSizeGB": "32",
"serverVersion": "15"
},
"restrictIstioIngress": false
"restrictIstioIngress": false,
"serverMqttClientName": "maestro-server-cspr-cs"
},
"mgmt": {
"clusterServicePrincipalId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-cspr-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service",
Expand Down
7 changes: 4 additions & 3 deletions config/public-cloud-dev.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@
"consumerName": "hcp-underlay-dev-mgmt-1",
"eventGrid": {
"maxClientSessionsPerAuthName": "4",
"name": "arohcp-maestro-dev"
"name": "arohcp-maestro-dev",
"private": false
},
"imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
"imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f",
"keyVaultName": "arohcp-maestro-dev",
"postgres": {
"deploy": false,
"minTLSVersion": "TLSV1.2",
Expand All @@ -71,7 +71,8 @@
"serverStorageSizeGB": "32",
"serverVersion": "15"
},
"restrictIstioIngress": true
"restrictIstioIngress": true,
"serverMqttClientName": "maestro-server-dev-dev"
},
"mgmt": {
"clusterServicePrincipalId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-dev-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service",
Expand Down
7 changes: 4 additions & 3 deletions config/public-cloud-msft-int.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@
"consumerName": "hcp-underlay-int-mgmt-1",
"eventGrid": {
"maxClientSessionsPerAuthName": "4",
"name": "arohcp-maestro-int"
"name": "arohcp-maestro-int",
"private": false
},
"imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
"imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f",
"keyVaultName": "arohcp-maestro-int",
"postgres": {
"deploy": false,
"minTLSVersion": "TLSV1.2",
Expand All @@ -71,7 +71,8 @@
"serverStorageSizeGB": "32",
"serverVersion": "15"
},
"restrictIstioIngress": true
"restrictIstioIngress": true,
"serverMqttClientName": "maestro-server"
},
"mgmt": {
"clusterServicePrincipalId": "todo",
Expand Down
7 changes: 4 additions & 3 deletions config/public-cloud-personal-dev.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@
"consumerName": "hcp-underlay-usw3tst-mgmt-1",
"eventGrid": {
"maxClientSessionsPerAuthName": "4",
"name": "arohcp-maestro-usw3tst"
"name": "arohcp-maestro-usw3tst",
"private": false
},
"imageBase": "quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro",
"imageTag": "ea066c250a002f0cc458711945165591bc9f6d3f",
"keyVaultName": "arohcp-maestro-usw3tst",
"postgres": {
"deploy": false,
"minTLSVersion": "TLSV1.2",
Expand All @@ -71,7 +71,8 @@
"serverStorageSizeGB": "32",
"serverVersion": "15"
},
"restrictIstioIngress": true
"restrictIstioIngress": true,
"serverMqttClientName": "maestro-server-usw3tst"
},
"mgmt": {
"clusterServicePrincipalId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-usw3tst-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ param userAgentPoolAZCount = {{ .mgmt.userAgentPool.azCount }}

// Maestro
param maestroConsumerName = '{{ .maestro.consumerName }}'
param maestroKeyVaultName = '{{ .maestro.keyVaultName }}'
param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}'
param maestroCertDomain = '{{ .maestro.certDomain }}'

Expand Down Expand Up @@ -56,3 +55,6 @@ param mgmtKeyVaultSoftDelete = {{ .mgmtKeyVault.softDelete }}
// Cluster Service identity
// used for Key Vault access
param clusterServicePrincipalId = '{{ .mgmt.clusterServicePrincipalId }}'

// MI for deployment scripts
param aroDevopsMsiId = '{{ .aroDevopsMsiId }}'
2 changes: 1 addition & 1 deletion dev-infrastructure/configurations/region.tmpl.bicepparam
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ param baseDNSZoneResourceGroup = '{{ .baseDnsZoneRG }}'
param regionalDNSSubdomain = '{{ .regionalDNSSubdomain }}'

// maestro
param maestroKeyVaultName = '{{ .maestro.keyVaultName }}'
param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}'
param maestroEventGridMaxClientSessionsPerAuthName = {{ .maestro.eventGrid.maxClientSessionsPerAuthName }}
param maestroEventGridPrivate = {{ .maestro.eventGrid.private }}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ param deployFrontendCosmos = {{ .frontend.cosmosDB.deploy }}
param rpCosmosDbName = '{{ .frontend.cosmosDB.name }}'
param rpCosmosDbPrivate = {{ .frontend.cosmosDB.private }}

param maestroKeyVaultName = '{{ .maestro.keyVaultName }}'
param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}'
param maestroServerMqttClientName = '{{ .maestro.serverMqttClientName }}'
param maestroCertDomain = '{{ .maestro.certDomain}}'
param maestroPostgresServerName = '{{ .maestro.postgres.name }}'
param maestroPostgresServerMinTLSVersion = '{{ .maestro.postgres.minTLSVersion }}'
Expand Down
27 changes: 27 additions & 0 deletions dev-infrastructure/mgmt-pipeline.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
serviceGroup: Microsoft.Azure.ARO.Test
rolloutName: Management Cluster Rollout
resourceGroups:
- name: {{ .mgmt.rg }}
subscription: {{ .mgmt.subscription }}
aksCluster: {{ .aksName }}
steps:
- name: mgmt
action: ARM
template: templates/mgmt-cluster.bicep
parameters: configurations/mgmt-cluster.tmpl.bicepparam
- name: enable-metrics
action: Shell
command: ["scripts/enable-aks-metrics.sh"]
env:
- name: RESOURCEGROUP
configRef: mgmt.rg
- name: AKS_NAME
configRef: aksName
- name: GRAFANA_RESOURCEGROUP
configRef: regionRG
- name: MONITORING_WORKSPACE_NAME
configRef: monitoring.workspaceName
- name: GRAFANA_NAME
configRef: monitoring.grafanaName
dependsOn:
- mgmt
4 changes: 2 additions & 2 deletions dev-infrastructure/modules/key-vault-cert.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ param issuerName string
param dnsNames array
param now string = utcNow('F')
param keyVaultManagedIdentityId string
param location string
param location string = resourceGroup().location
param force bool = false
var boolstring = force == false ? '$false' : '$true'
param validityInMonths int = 12
Expand All @@ -30,7 +30,7 @@ resource newCertwithRotationKV 'Microsoft.Resources/deploymentScripts@2023-08-01
location: location
kind: 'AzurePowerShell'
properties: {
azPowerShellVersion: '7.5.0'
azPowerShellVersion: '12.0.0'
arguments: ' -VaultName ${keyVaultName} -ValidityInMonths ${validityInMonths} -IssuerName ${issuerName} -CertName ${certName} -SubjectName ${subjectName} -DnsNames ${join(dnsNames,'_')} -Force ${boolstring}'
scriptContent: loadTextContent('../scripts/key-vault-cert.ps1')
forceUpdateTag: now
Expand Down
85 changes: 85 additions & 0 deletions dev-infrastructure/modules/maestro/maestro-access-cert.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
Creates a certificate in Key Vault signed by the specified issuer.
For dev environments `Self` is used as issuer, for higher environments
OneCertV2 Private will be used.
The specified managed identity `certificateAccessManagedIdentityPrincipalId`
is granted access to the certificate in Key Vault. This will be leveraged
with CSI secret store to access the certificate from the maestro pods.
Execution scope: the resourcegroup of the Key Vault where the certificate will be stored
*/

@description('The Key Vault where the certificate for Event Grid access will be stored')
param keyVaultName string

@description('The managed identity that will be used to manage the certificate in Key Vault')
param kvCertOfficerManagedIdentityResourceId string

@description('The base domain name to be used for the certificates DNS name.')
param certDomain string

@description('The name of the client that will be created in the EventGrid Namespace')
param clientName string

@description('The name of the certificate in Key Vault.')
param keyVaultCertificateName string

@description('The issuer of the certificate.')
param certificateIssuer string = 'Self'

@description('Grant this managed identity access to the certificate in Key Vault.')
param certificateAccessManagedIdentityPrincipalId string

//
// C E R T I F I C A T E C R E A T I O N
//

var clientAuthenticationName = '${clientName}.${certDomain}'

module clientCertificate '../key-vault-cert.bicep' = {
name: '${clientName}-client-cert'
params: {
keyVaultName: keyVaultName
subjectName: 'CN=${clientName}'
certName: keyVaultCertificateName
keyVaultManagedIdentityId: kvCertOfficerManagedIdentityResourceId
dnsNames: [
clientAuthenticationName
]
issuerName: certificateIssuer
}
}

//
// C E R T I F I C A T E A C C E S S P E R M I S S I O N
//

var keyVaultSecretUserRoleId = subscriptionResourceId(
'Microsoft.Authorization/roleDefinitions/',
'4633458b-17de-408a-b874-0445c86b69e6'
)

resource kv 'Microsoft.KeyVault/vaults@2023-07-01' existing = {
name: keyVaultName
}

// grant permissions on the secret that contains the certificate

resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' existing = {
parent: kv
name: keyVaultCertificateName
}

resource secretAccessPermission 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
scope: secret
name: guid(certificateAccessManagedIdentityPrincipalId, keyVaultSecretUserRoleId, kv.id, keyVaultCertificateName)
properties: {
roleDefinitionId: keyVaultSecretUserRoleId
principalId: certificateAccessManagedIdentityPrincipalId
principalType: 'ServicePrincipal'
}
}

output certificateThumbprint string = clientCertificate.outputs.Thumbprint
output certificateSAN string = clientAuthenticationName
Loading

0 comments on commit 4f3c39b

Please sign in to comment.