Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chiac/svc #893

Closed
wants to merge 23 commits into from
14 changes: 7 additions & 7 deletions config/config.msft.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ defaults:

# General AKS config
kubernetesVersion: 1.30.5
istioVersion: "['asm-1-22']"
istioVersion: "asm-1-22"
vnetAddressPrefix: "10.128.0.0/14"
subnetPrefix: "10.128.8.0/21"
podSubnetPrefix: "10.128.64.0/18"
Expand Down Expand Up @@ -51,13 +51,13 @@ defaults:
maestro:
eventGrid:
name: arohcp-maestro-{{ .ctx.regionShort }}
maxClientSessionsPerAuthName: '4'
maxClientSessionsPerAuthName: 4
private: false
certDomain: 'selfsigned.maestro.keyvault.azure.com'
postgres:
name: arohcp-maestro-{{ .ctx.regionShort }}
serverVersion: '15'
serverStorageSizeGB: '32'
serverStorageSizeGB: 32
deploy: false
private: false
minTLSVersion: 'TLSV1.2'
Expand Down Expand Up @@ -89,7 +89,7 @@ defaults:

serviceKeyVault:
name: arohcp-svc-{{ .ctx.regionShort }}
rg: hcp-underlay-{{ .ctx.regionShort }}
rg: hcp-underlay-{{ .ctx.region }}-svc
region: {{ .ctx.region }}
softDelete: false
private: false
Expand Down Expand Up @@ -162,7 +162,7 @@ clouds:
osDiskSizeGB: 100
azCount: 3
# DNS
baseDnsZoneName: aro-hcp.azure-test.net'
baseDnsZoneName: aro-hcp.azure-test.net
regionalDNSSubdomain: '{{ .ctx.region }}'

# ACR
Expand Down Expand Up @@ -191,9 +191,9 @@ clouds:

# Grafana
monitoring:
grafanaAdminGroupPrincipalId: '??? the one to be used as Grafana Admin in grafana.bicep ???'
grafanaAdminGroupPrincipalId: '2fdb57d4-3fd3-415d-b604-1d0e37a188fe' # Azure Red Hat OpenShift MSFT Engineering

# DEVOPS MSI
# lets create this MSI manually for the time being and automate soon
# but we should use the MSI name as an input and not the resource ID of the MSI
aroDevopsMsiId: '??? the one for OIDC deployment script / lives in the global RG / needs to be created first thing on regional buildout ???'
aroDevopsMsiId: '/subscriptions/5299e6b7-b23b-46c8-8277-dc1147807117/resourcegroups/global-shared-resources/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aroint-int-public-oidc'
4 changes: 2 additions & 2 deletions dev-infrastructure/configurations/region.tmpl.bicepparam
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ param regionalDNSSubdomain = '{{ .regionalDNSSubdomain }}'

// maestro
param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}'
param maestroEventGridMaxClientSessionsPerAuthName = {{ .maestro.eventGrid.maxClientSessionsPerAuthName }}
param maestroEventGridPrivate = {{ .maestro.eventGrid.private }}
param maestroEventGridMaxClientSessionsPerAuthName = any('{{ .maestro.eventGrid.maxClientSessionsPerAuthName }}')
param maestroEventGridPrivate = any('{{ .maestro.eventGrid.private }}')
32 changes: 16 additions & 16 deletions dev-infrastructure/configurations/svc-cluster.tmpl.bicepparam
Original file line number Diff line number Diff line change
@@ -1,45 +1,45 @@
using '../templates/svc-cluster.bicep'

param kubernetesVersion = '{{ .kubernetesVersion }}'
param istioVersion = {{ .istioVersion }}
param istioVersion = ['{{ .istioVersion }}']
param vnetAddressPrefix = '{{ .vnetAddressPrefix }}'
param subnetPrefix = '{{ .subnetPrefix }}'
param podSubnetPrefix = '{{ .podSubnetPrefix }}'
param aksClusterName = '{{ .aksName }}'
param aksKeyVaultName = '{{ .svc.etcd.kvName }}'
param aksEtcdKVEnableSoftDelete = {{ .svc.etcd.kvSoftDelete }}
param aksEtcdKVEnableSoftDelete = any('{{ .svc.etcd.kvSoftDelete }}')

param userAgentMinCount = {{ .svc.userAgentPool.minCount }}
param userAgentMaxCount = {{ .svc.userAgentPool.maxCount }}
param userAgentMinCount = any('{{ .svc.userAgentPool.minCount }}')
param userAgentMaxCount = any('{{ .svc.userAgentPool.maxCount }}')
param userAgentVMSize = '{{ .svc.userAgentPool.vmSize }}'
param aksUserOsDiskSizeGB = {{ .svc.userAgentPool.osDiskSizeGB }}
param userAgentPoolAZCount = {{ .svc.userAgentPool.azCount }}
param aksUserOsDiskSizeGB = any('{{ .svc.userAgentPool.osDiskSizeGB }}')
param userAgentPoolAZCount = any('{{ .svc.userAgentPool.azCount }}')

param disableLocalAuth = {{ .frontend.cosmosDB.disableLocalAuth }}
param deployFrontendCosmos = {{ .frontend.cosmosDB.deploy }}
param disableLocalAuth = any('{{ .frontend.cosmosDB.disableLocalAuth }}')
param deployFrontendCosmos = any('{{ .frontend.cosmosDB.deploy }}')
param rpCosmosDbName = '{{ .frontend.cosmosDB.name }}'
param rpCosmosDbPrivate = {{ .frontend.cosmosDB.private }}
param rpCosmosDbPrivate = any('{{ .frontend.cosmosDB.private }}')

param maestroEventGridNamespacesName = '{{ .maestro.eventGrid.name }}'
param maestroServerMqttClientName = '{{ .maestro.serverMqttClientName }}'
param maestroCertDomain = '{{ .maestro.certDomain}}'
param maestroPostgresServerName = '{{ .maestro.postgres.name }}'
param maestroPostgresServerMinTLSVersion = '{{ .maestro.postgres.minTLSVersion }}'
param maestroPostgresServerVersion = '{{ .maestro.postgres.serverVersion }}'
param maestroPostgresServerStorageSizeGB = {{ .maestro.postgres.serverStorageSizeGB }}
param deployMaestroPostgres = {{ .maestro.postgres.deploy }}
param maestroPostgresPrivate = {{ .maestro.postgres.private }}
param maestroPostgresServerStorageSizeGB = any('{{ .maestro.postgres.serverStorageSizeGB }}')
param deployMaestroPostgres = any('{{ .maestro.postgres.deploy }}')
param maestroPostgresPrivate = any('{{ .maestro.postgres.private }}')

param deployCsInfra = {{ .clusterService.postgres.deploy }}
param deployCsInfra = any('{{ .clusterService.postgres.deploy }}')
param csPostgresServerName = '{{ .clusterService.postgres.name }}'
param csPostgresServerMinTLSVersion = '{{ .clusterService.postgres.minTLSVersion }}'
param clusterServicePostgresPrivate = {{ .clusterService.postgres.private }}
param clusterServicePostgresPrivate = any('{{ .clusterService.postgres.private }}')

param serviceKeyVaultName = '{{ .serviceKeyVault.name }}'
param serviceKeyVaultResourceGroup = '{{ .serviceKeyVault.rg }}'
param serviceKeyVaultLocation = '{{ .serviceKeyVault.region }}'
param serviceKeyVaultSoftDelete = {{ .serviceKeyVault.softDelete }}
param serviceKeyVaultPrivate = {{ .serviceKeyVault.private }}
param serviceKeyVaultSoftDelete = any('{{ .serviceKeyVault.softDelete }}')
param serviceKeyVaultPrivate = any('{{ .serviceKeyVault.private }}')

param acrPullResourceGroups = ['{{ .serviceComponentAcrResourceGroups }}']
param imageSyncAcrResourceGroupNames = ['{{ .imageSync.acrRG }}']
Expand Down
4 changes: 2 additions & 2 deletions dev-infrastructure/region-pipeline.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
serviceGroup: Microsoft.Azure.ARO.Test
serviceGroup: Microsoft.Azure.ARO.HCP.Region
rolloutName: Region Rollout
resourceGroups:
- name: {{ .regionRG }}
Expand All @@ -10,7 +10,7 @@ resourceGroups:
parameters: configurations/region.tmpl.bicepparam
- name: metrics-infra
action: ARM
template: modules/modules/metrics.bicep
template: modules/metrics/metrics.bicep
parameters: configurations/metrics.tmpl.bicepparam
dependsOn:
- region
2 changes: 1 addition & 1 deletion dev-infrastructure/svc-pipeline.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
serviceGroup: Microsoft.Azure.ARO.Test
serviceGroup: Microsoft.Azure.ARO.HCP.Service.Infra
rolloutName: Service Cluster Rollout
resourceGroups:
- name: {{ .svc.rg }}
Expand Down
2 changes: 1 addition & 1 deletion dev-infrastructure/templates/region.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ module regionalZoneDelegation '../modules/dns/zone-delegation.bicep' = {
params: {
childZoneName: regionalDNSSubdomain
childZoneNameservers: regionalZone.properties.nameServers
parentZoneName: baseDNSZoneName
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm. this would try to delegate the regional zone into the regional zone as child zone and parent zone would be the same

parentZoneName: regionalZone.name
}
}

Expand Down
182 changes: 91 additions & 91 deletions dev-infrastructure/templates/svc-cluster.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -231,42 +231,42 @@ module rpCosmosDb '../modules/rp-cosmos.bicep' = if (deployFrontendCosmos) {
output cosmosDBName string = deployFrontendCosmos ? rpCosmosDb.outputs.cosmosDBName : ''
output frontend_mi_client_id string = frontendMI.uamiClientID

//
// M A E S T R O
//

module maestroServer '../modules/maestro/maestro-server.bicep' = {
name: 'maestro-server'
params: {
maestroInfraResourceGroup: regionalResourceGroup
maestroEventGridNamespaceName: maestroEventGridNamespacesName
mqttClientName: maestroServerMqttClientName
certKeyVaultName: serviceKeyVaultName
certKeyVaultResourceGroup: serviceKeyVaultResourceGroup
keyVaultOfficerManagedIdentityName: aroDevopsMsiId
maestroCertificateDomain: maestroCertDomain
deployPostgres: deployMaestroPostgres
postgresServerName: maestroPostgresServerName
postgresServerVersion: maestroPostgresServerVersion
postgresServerMinTLSVersion: maestroPostgresServerMinTLSVersion
postgresServerStorageSizeGB: maestroPostgresServerStorageSizeGB
privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId
privateEndpointVnetId: svcCluster.outputs.aksVnetId
postgresServerPrivate: maestroPostgresPrivate
maestroServerManagedIdentityPrincipalId: filter(
svcCluster.outputs.userAssignedIdentities,
id => id.uamiName == 'maestro-server'
)[0].uamiPrincipalID
maestroServerManagedIdentityName: filter(
svcCluster.outputs.userAssignedIdentities,
id => id.uamiName == 'maestro-server'
)[0].uamiName
location: location
}
dependsOn: [
serviceKeyVault
]
}
// //
// // M A E S T R O
// //

// module maestroServer '../modules/maestro/maestro-server.bicep' = {
// name: 'maestro-server'
// params: {
// maestroInfraResourceGroup: regionalResourceGroup
// maestroEventGridNamespaceName: maestroEventGridNamespacesName
// mqttClientName: maestroServerMqttClientName
// certKeyVaultName: serviceKeyVaultName
// certKeyVaultResourceGroup: serviceKeyVaultResourceGroup
// keyVaultOfficerManagedIdentityName: aroDevopsMsiId
// maestroCertificateDomain: maestroCertDomain
// deployPostgres: deployMaestroPostgres
// postgresServerName: maestroPostgresServerName
// postgresServerVersion: maestroPostgresServerVersion
// postgresServerMinTLSVersion: maestroPostgresServerMinTLSVersion
// postgresServerStorageSizeGB: maestroPostgresServerStorageSizeGB
// privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId
// privateEndpointVnetId: svcCluster.outputs.aksVnetId
// postgresServerPrivate: maestroPostgresPrivate
// maestroServerManagedIdentityPrincipalId: filter(
// svcCluster.outputs.userAssignedIdentities,
// id => id.uamiName == 'maestro-server'
// )[0].uamiPrincipalID
// maestroServerManagedIdentityName: filter(
// svcCluster.outputs.userAssignedIdentities,
// id => id.uamiName == 'maestro-server'
// )[0].uamiName
// location: location
// }
// dependsOn: [
// serviceKeyVault
// ]
// }

//
// K E Y V A U L T S
Expand Down Expand Up @@ -320,7 +320,7 @@ module cs '../modules/cluster-service.bicep' = if (deployCsInfra) {
clusterServiceManagedIdentityName: clusterServiceMIName
}
dependsOn: [
maestroServer
// maestroServer
svcCluster
]
}
Expand Down Expand Up @@ -371,60 +371,60 @@ module imageServiceKeyVaultAccess '../modules/keyvault/keyvault-secret-access.bi
]
}

resource imageSyncAcrResourceGroups 'Microsoft.Resources/resourceGroups@2023-07-01' existing = [
for rg in imageSyncAcrResourceGroupNames: {
name: rg
scope: subscription()
}
]

module acrPushRole '../modules/acr-permissions.bicep' = [
for (_, i) in imageSyncAcrResourceGroupNames: {
name: guid(imageSyncAcrResourceGroups[i].id, resourceGroup().name, 'image-sync', 'push')
scope: imageSyncAcrResourceGroups[i]
params: {
principalId: imageSyncManagedIdentityPrincipalId
grantPushAccess: true
acrResourceGroupid: imageSyncAcrResourceGroups[i].id
}
}
]

resource clustersServiceAcrResourceGroups 'Microsoft.Resources/resourceGroups@2023-07-01' existing = [
for rg in clustersServiceAcrResourceGroupNames: {
name: rg
scope: subscription()
}
]

module acrManageTokenRole '../modules/acr-permissions.bicep' = [
for (_, i) in clustersServiceAcrResourceGroupNames: {
name: guid(clustersServiceAcrResourceGroups[i].id, resourceGroup().name, 'clusters-service', 'manage-tokens')
scope: clustersServiceAcrResourceGroups[i]
params: {
principalId: csManagedIdentityPrincipalId
grantManageTokenAccess: true
acrResourceGroupid: clustersServiceAcrResourceGroups[i].id
}
}
]

// oidc

module oidc '../modules/oidc/main.bicep' = {
name: '${deployment().name}-oidc'
params: {
location: location
storageAccountName: oidcStorageAccountName
rpMsiName: clusterServiceMIName
skuName: oidcStorageAccountSku
aroDevopsMsiId: aroDevopsMsiId
deploymentScriptLocation: location
}
dependsOn: [
svcCluster
]
}
// resource imageSyncAcrResourceGroups 'Microsoft.Resources/resourceGroups@2023-07-01' existing = [
// for rg in imageSyncAcrResourceGroupNames: {
// name: rg
// scope: subscription()
// }
// ]

// module acrPushRole '../modules/acr-permissions.bicep' = [
// for (_, i) in imageSyncAcrResourceGroupNames: {
// name: guid(imageSyncAcrResourceGroups[i].id, resourceGroup().name, 'image-sync', 'push')
// scope: imageSyncAcrResourceGroups[i]
// params: {
// principalId: imageSyncManagedIdentityPrincipalId
// grantPushAccess: true
// acrResourceGroupid: imageSyncAcrResourceGroups[i].id
// }
// }
// ]
Comment on lines +374 to +391
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this has been removed from main #896


// resource clustersServiceAcrResourceGroups 'Microsoft.Resources/resourceGroups@2023-07-01' existing = [
// for rg in clustersServiceAcrResourceGroupNames: {
// name: rg
// scope: subscription()
// }
// ]

// module acrManageTokenRole '../modules/acr-permissions.bicep' = [
// for (_, i) in clustersServiceAcrResourceGroupNames: {
// name: guid(clustersServiceAcrResourceGroups[i].id, resourceGroup().name, 'clusters-service', 'manage-tokens')
// scope: clustersServiceAcrResourceGroups[i]
// params: {
// principalId: csManagedIdentityPrincipalId
// grantManageTokenAccess: true
// acrResourceGroupid: clustersServiceAcrResourceGroups[i].id
// }
// }
// ]
Comment on lines +393 to +410
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this code has been moved to cluster-service.bicep and will not run in MSFT INT for the time being until we have the ACR token custom role figured out - #903


// // oidc

// module oidc '../modules/oidc/main.bicep' = {
// name: '${deployment().name}-oidc'
// params: {
// location: location
// storageAccountName: oidcStorageAccountName
// rpMsiName: clusterServiceMIName
// skuName: oidcStorageAccountSku
// aroDevopsMsiId: aroDevopsMsiId
// deploymentScriptLocation: location
// }
// dependsOn: [
// svcCluster
// ]
// }
Comment on lines +411 to +427
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the permission issue has been fixed in main - #897


//
// E V E N T G R I D P R I V A T E E N D P O I N T C O N N E C T I O N
Expand Down
2 changes: 1 addition & 1 deletion tooling/templatize/pkg/ev2/mapping.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func EV2Mapping(input config.Variables, prefix []string) (map[string]string, map
}
replaced[key] = replacement
} else {
placeholder := fmt.Sprintf("__%s__", strings.ToUpper(strings.Join(nestedKey, "_")))
placeholder := fmt.Sprintf("__%s__", strings.Join(nestedKey, "_"))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed in main including the tests - #901

output[placeholder] = strings.Join(nestedKey, ".")
replaced[key] = placeholder
}
Expand Down
Loading