From d189b77dd313f839e8d16d27d208a9f26ee0f07e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 1 Feb 2024 10:23:26 -0600 Subject: [PATCH] Vnet injection tutorial (#46) * Updated Solution with a vnet injection sample. * Added blob public storage feature enablement. * Updated deploy template. --- README.md | 20 ++ azuredeploy.json | 137 +++++++-- bicep/main.bicep | 54 +++- bicep/main.parameters.json | 65 ++--- bicep/modules/aks_agent_pool.bicep | 4 + bicep/modules/aks_cluster.bicep | 1 + bicep/modules/storage-account/main.bicep | 5 + .../storage-account/test/main.test.bicep | 2 +- docs/vnet-injection.md | 259 ++++++++++++++++++ 9 files changed, 480 insertions(+), 67 deletions(-) create mode 100644 docs/vnet-injection.md diff --git a/README.md b/README.md index 0a8f975b..9b3f0625 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,19 @@ azd env set _VPN_SHARED_KEY ``` +#### Feature: Public Blob Access + +__Purpose:__ Control public access to Blob Storage. + +__Details:__ The Storage accounts have public access points that can be enabled or disabled to enhance security. + +__How to Disable:__ + +```bash +azd env set ENABLE_BLOB_PUBLIC_ACCESS false +``` + + ### Deployment Commands Efficiently manage the resources with these Azure Developer CLI commands. They are designed to streamline the deployment process, allowing for a smooth setup and teardown of your environment. @@ -269,6 +282,13 @@ Our GitOps configuration resides in this Git repository and uses a customized [r Our GitOps approach simplifies the process of deploying and managing software, making it easier to maintain and update, as well as providing a configurable way of leveraging other software configurations by pointing to alternate repositories hosting other configurations. By leveraging this method, we ensure that our deployments can be extended to things that not only include the default software load. +## Customizations + +There are many things that can be done to customize the deployment. One example of this might be virtual network injection. + +See [this tutorial](docs/vnet-injection.md) for how a customization like this might be performed. + + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/azuredeploy.json b/azuredeploy.json index 3777b954..2e3e82cd 100644 --- a/azuredeploy.json +++ b/azuredeploy.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "3443129603583958142" + "templateHash": "5166673085691231756" } }, "parameters": { @@ -194,6 +194,40 @@ "description": "Resource group of the VNet (Optional: If exiting Network is selected)" } }, + "serviceCidr": { + "type": "string", + "defaultValue": "172.16.0.0/16", + "minLength": 9, + "maxLength": 18, + "metadata": { + "description": "The address range to use for services" + } + }, + "dockerBridgeCidr": { + "type": "string", + "defaultValue": "172.17.0.1/16", + "minLength": 9, + "maxLength": 18, + "metadata": { + "description": "The address range to use for the docker bridge" + } + }, + "dnsServiceIP": { + "type": "string", + "defaultValue": "172.16.0.10", + "minLength": 7, + "maxLength": 15, + "metadata": { + "description": "The IP address to reserve for DNS" + } + }, + "enableBlobPublicAccess": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false." + } + }, "vmAdminUsername": { "type": "string", "defaultValue": "[if(parameters('enableBastion'), 'azureUser', newGuid())]", @@ -698,7 +732,7 @@ "cluster": { "aksVersion": "1.28", "meshVersion": "asm-1-18", - "networkPlugin": "kubenet" + "networkPlugin": "[if(parameters('enablePodSubnet'), 'azure', 'kubenet')]" }, "gitops": { "name": "flux-system", @@ -3029,6 +3063,7 @@ ] }, { + "condition": "[equals(parameters('virtualNetworkNewOrExisting'), 'new')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)]", @@ -3715,7 +3750,7 @@ } }, { - "condition": "[parameters('enableBastion')]", + "condition": "[and(equals(parameters('virtualNetworkNewOrExisting'), 'new'), parameters('enableBastion'))]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)]", @@ -4402,7 +4437,7 @@ } }, { - "condition": "[parameters('enableBastion')]", + "condition": "[and(equals(parameters('virtualNetworkNewOrExisting'), 'new'), parameters('enableBastion'))]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('{0}-network-security-group-manage', variables('commonLayerConfig').name)]", @@ -5089,6 +5124,7 @@ } }, { + "condition": "[equals(parameters('virtualNetworkNewOrExisting'), 'new')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('{0}-virtual-network', variables('commonLayerConfig').name)]", @@ -5140,7 +5176,7 @@ ] }, "subnets": { - "value": "[union(array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).cluster), if(parameters('enableBastion'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).bastion), createArray()), if(parameters('enableBastion'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).machine), createArray()), if(parameters('enableVpnGateway'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).gateway), createArray()), if(parameters('enablePodSubnet'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).pods), createArray()))]" + "value": "[union(array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).cluster), if(parameters('enableBastion'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).bastion), createArray()), if(parameters('enableBastion'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).machine), createArray()), if(parameters('enableVpnGateway'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).gateway), createArray()), if(parameters('enablePodSubnet'), array(createObject('cluster', createObject('name', parameters('aksSubnetName'), 'addressPrefix', parameters('aksSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'pods', createObject('name', parameters('podSubnetName'), 'addressPrefix', parameters('podSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-cluster', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'roleAssignments', createArray(createObject('roleDefinitionIdOrName', 'Network Contributor', 'principalId', reference(resourceId('Microsoft.Resources/deployments', format('{0}-user-managed-identity', variables('commonLayerConfig').name)), '2022-09-01').outputs.principalId.value, 'principalType', 'ServicePrincipal'))), 'bastion', createObject('name', parameters('bastionSubnetName'), 'addressPrefix', parameters('bastionSubnetAddressPrefix'), 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-bastion', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null())), 'gateway', createObject('name', parameters('gatewaySubnetName'), 'addressPrefix', parameters('gatewaySubnetAddressPrefix')), 'machine', createObject('name', parameters('vmSubnetName'), 'addressPrefix', parameters('vmSubnetAddressPrefix'), 'serviceEndpoints', createArray(createObject('service', 'Microsoft.Storage'), createObject('service', 'Microsoft.KeyVault'), createObject('service', 'Microsoft.ContainerRegistry')), 'privateEndpointNetworkPolicies', 'Disabled', 'privateLinkServiceNetworkPolicies', 'Enabled', 'networkSecurityGroupResourceId', if(parameters('enableBastion'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-network-security-group-manage', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()))).pods), createArray()))]" } }, "template": { @@ -11006,6 +11042,9 @@ "tables": { "value": "[variables('commonLayerConfig').storage.tables]" }, + "allowBlobPublicAccess": { + "value": "[parameters('enableBlobPublicAccess')]" + }, "roleAssignments": { "value": [ { @@ -11040,7 +11079,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "15771443307137808627" + "templateHash": "12218145875057312705" } }, "parameters": { @@ -11249,6 +11288,13 @@ "description": "Amount of days the soft deleted data is stored and available for recovery. 0 is off." } }, + "allowBlobPublicAccess": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false." + } + }, "privateLinkSettings": { "type": "object", "defaultValue": { @@ -11365,6 +11411,7 @@ "accessTier": "[parameters('accessTier')]", "minimumTlsVersion": "TLS1_2", "encryption": "[if(variables('enableCMEK'), createObject('identity', createObject('userAssignedIdentity', parameters('cmekConfiguration').identityId), 'services', createObject('blob', createObject('enabled', true()), 'table', createObject('enabled', true()), 'file', createObject('enabled', true())), 'keySource', 'Microsoft.Keyvault', 'keyvaultproperties', createObject('keyname', parameters('cmekConfiguration').keyName, 'keyvaulturi', parameters('cmekConfiguration').kvUrl)), createObject('services', createObject('blob', createObject('enabled', true()), 'table', createObject('enabled', true()), 'file', createObject('enabled', true())), 'keySource', 'Microsoft.Storage'))]", + "allowBlobPublicAccess": "[parameters('allowBlobPublicAccess')]", "networkAcls": "[if(variables('enablePrivateLink'), createObject('bypass', 'AzureServices', 'defaultAction', 'Deny'), createObject())]" } }, @@ -14664,7 +14711,7 @@ "value": "[format('bh-{0}{1}', replace(variables('manageLayerConfig').name, '-', ''), uniqueString(resourceGroup().id, variables('manageLayerConfig').name))]" }, "vNetId": { - "value": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value]" + "value": "[createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')]]" }, "location": { "value": "[parameters('location')]" @@ -16065,6 +16112,9 @@ "diagnosticLogsRetentionInDays": { "value": 0 }, + "allowBlobPublicAccess": { + "value": "[parameters('enableBlobPublicAccess')]" + }, "sku": { "value": "[variables('partitionLayerConfig').storage.sku]" }, @@ -16108,7 +16158,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "15771443307137808627" + "templateHash": "12218145875057312705" } }, "parameters": { @@ -16317,6 +16367,13 @@ "description": "Amount of days the soft deleted data is stored and available for recovery. 0 is off." } }, + "allowBlobPublicAccess": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false." + } + }, "privateLinkSettings": { "type": "object", "defaultValue": { @@ -16433,6 +16490,7 @@ "accessTier": "[parameters('accessTier')]", "minimumTlsVersion": "TLS1_2", "encryption": "[if(variables('enableCMEK'), createObject('identity', createObject('userAssignedIdentity', parameters('cmekConfiguration').identityId), 'services', createObject('blob', createObject('enabled', true()), 'table', createObject('enabled', true()), 'file', createObject('enabled', true())), 'keySource', 'Microsoft.Keyvault', 'keyvaultproperties', createObject('keyname', parameters('cmekConfiguration').keyName, 'keyvaulturi', parameters('cmekConfiguration').kvUrl)), createObject('services', createObject('blob', createObject('enabled', true()), 'table', createObject('enabled', true()), 'file', createObject('enabled', true())), 'keySource', 'Microsoft.Storage'))]", + "allowBlobPublicAccess": "[parameters('allowBlobPublicAccess')]", "networkAcls": "[if(variables('enablePrivateLink'), createObject('bypass', 'AzureServices', 'defaultAction', 'Deny'), createObject())]" } }, @@ -19769,6 +19827,15 @@ "workspaceId": { "value": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-log-analytics', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value]" }, + "serviceCidr": { + "value": "[parameters('serviceCidr')]" + }, + "dnsServiceIP": { + "value": "[parameters('dnsServiceIP')]" + }, + "dockerBridgeCidr": { + "value": "[parameters('dockerBridgeCidr')]" + }, "serviceMeshProfile": { "value": "Istio" }, @@ -19814,7 +19881,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "125255005146302743" + "templateHash": "16374876761694977329" } }, "parameters": { @@ -20488,6 +20555,7 @@ "enableAutoScaling": true, "maxPods": "[variables('userPoolPresets')[parameters('clusterSize')].maxPods]", "vnetSubnetID": "[if(not(empty(parameters('aksSubnetId'))), parameters('aksSubnetId'), null())]", + "podSubnetID": "[if(not(empty(parameters('aksPodSubnetId'))), parameters('aksPodSubnetId'), null())]", "upgradeSettings": { "maxSurge": "33%" } @@ -20747,9 +20815,8 @@ "1" ] }, - "subnetId": { - "value": "" - }, + "subnetId": "[if(not(equals(parameters('virtualNetworkNewOrExisting'), 'new')), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('aksSubnetName'))), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('aksSubnetName'))))]", + "podSubnetId": "[if(and(not(equals(parameters('virtualNetworkNewOrExisting'), 'new')), parameters('enablePodSubnet')), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('podSubnetName'))), createObject('value', ''))]", "nodeTaints": { "value": [ "app=elasticsearch:NoSchedule" @@ -20768,7 +20835,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "2918991882660483195" + "templateHash": "1717696639715253483" } }, "parameters": { @@ -20847,6 +20914,12 @@ "description": "The subnet the node pool will use" } }, + "podSubnetId": { + "type": "string", + "metadata": { + "description": "The subnet the pods will use" + } + }, "osType": { "type": "string", "defaultValue": "Linux", @@ -20881,6 +20954,7 @@ "maxPods": "[parameters('maxPods')]", "type": "VirtualMachineScaleSets", "vnetSubnetID": "[if(not(empty(parameters('subnetId'))), parameters('subnetId'), null())]", + "podSubnetID": "[if(not(empty(parameters('podSubnetId'))), parameters('podSubnetId'), null())]", "upgradeSettings": { "maxSurge": "33%" }, @@ -20892,7 +20966,8 @@ } }, "dependsOn": [ - "[resourceId('Microsoft.Resources/deployments', format('{0}-aks-cluster', variables('serviceLayerConfig').name))]" + "[resourceId('Microsoft.Resources/deployments', format('{0}-aks-cluster', variables('serviceLayerConfig').name))]", + "[resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name))]" ] }, { @@ -20925,9 +21000,8 @@ "2" ] }, - "subnetId": { - "value": "" - }, + "subnetId": "[if(not(equals(parameters('virtualNetworkNewOrExisting'), 'new')), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('aksSubnetName'))), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('aksSubnetName'))))]", + "podSubnetId": "[if(and(not(equals(parameters('virtualNetworkNewOrExisting'), 'new')), parameters('enablePodSubnet')), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('podSubnetName'))), createObject('value', ''))]", "nodeTaints": { "value": [ "app=elasticsearch:NoSchedule" @@ -20946,7 +21020,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "2918991882660483195" + "templateHash": "1717696639715253483" } }, "parameters": { @@ -21025,6 +21099,12 @@ "description": "The subnet the node pool will use" } }, + "podSubnetId": { + "type": "string", + "metadata": { + "description": "The subnet the pods will use" + } + }, "osType": { "type": "string", "defaultValue": "Linux", @@ -21059,6 +21139,7 @@ "maxPods": "[parameters('maxPods')]", "type": "VirtualMachineScaleSets", "vnetSubnetID": "[if(not(empty(parameters('subnetId'))), parameters('subnetId'), null())]", + "podSubnetID": "[if(not(empty(parameters('podSubnetId'))), parameters('podSubnetId'), null())]", "upgradeSettings": { "maxSurge": "33%" }, @@ -21070,7 +21151,8 @@ } }, "dependsOn": [ - "[resourceId('Microsoft.Resources/deployments', format('{0}-aks-cluster', variables('serviceLayerConfig').name))]" + "[resourceId('Microsoft.Resources/deployments', format('{0}-aks-cluster', variables('serviceLayerConfig').name))]", + "[resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name))]" ] }, { @@ -21103,9 +21185,8 @@ "3" ] }, - "subnetId": { - "value": "" - }, + "subnetId": "[if(not(equals(parameters('virtualNetworkNewOrExisting'), 'new')), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('aksSubnetName'))), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('aksSubnetName'))))]", + "podSubnetId": "[if(and(not(equals(parameters('virtualNetworkNewOrExisting'), 'new')), parameters('enablePodSubnet')), createObject('value', format('{0}/subnets/{1}', createObject('new', if(equals(parameters('virtualNetworkNewOrExisting'), 'new'), reference(resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name)), '2022-09-01').outputs.resourceId.value, null()), 'existing', resourceId(parameters('virtualNetworkResourceGroup'), 'Microsoft.Network/virtualNetworks', parameters('virtualNetworkName')))[parameters('virtualNetworkNewOrExisting')], parameters('podSubnetName'))), createObject('value', ''))]", "nodeTaints": { "value": [ "app=elasticsearch:NoSchedule" @@ -21124,7 +21205,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "2918991882660483195" + "templateHash": "1717696639715253483" } }, "parameters": { @@ -21203,6 +21284,12 @@ "description": "The subnet the node pool will use" } }, + "podSubnetId": { + "type": "string", + "metadata": { + "description": "The subnet the pods will use" + } + }, "osType": { "type": "string", "defaultValue": "Linux", @@ -21237,6 +21324,7 @@ "maxPods": "[parameters('maxPods')]", "type": "VirtualMachineScaleSets", "vnetSubnetID": "[if(not(empty(parameters('subnetId'))), parameters('subnetId'), null())]", + "podSubnetID": "[if(not(empty(parameters('podSubnetId'))), parameters('podSubnetId'), null())]", "upgradeSettings": { "maxSurge": "33%" }, @@ -21248,7 +21336,8 @@ } }, "dependsOn": [ - "[resourceId('Microsoft.Resources/deployments', format('{0}-aks-cluster', variables('serviceLayerConfig').name))]" + "[resourceId('Microsoft.Resources/deployments', format('{0}-aks-cluster', variables('serviceLayerConfig').name))]", + "[resourceId('Microsoft.Resources/deployments', format('{0}-virtual-network', variables('commonLayerConfig').name))]" ] }, { diff --git a/bicep/main.bicep b/bicep/main.bicep index 91bdcd38..02bbd6aa 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -221,6 +221,21 @@ param virtualNetworkName string = 'osdu-network' @description('Resource group of the VNet (Optional: If exiting Network is selected)') param virtualNetworkResourceGroup string = 'osdu-network' +@minLength(9) +@maxLength(18) +@description('The address range to use for services') +param serviceCidr string = '172.16.0.0/16' + +@minLength(9) +@maxLength(18) +@description('The address range to use for the docker bridge') +param dockerBridgeCidr string = '172.17.0.1/16' + +@minLength(7) +@maxLength(15) +@description('The IP address to reserve for DNS') +param dnsServiceIP string = '172.16.0.10' + var nsgRules = { ssh_outbound: { name: 'AllowSshOutbound' @@ -390,7 +405,7 @@ var subnets = { service: 'Microsoft.ContainerRegistry' } ] - networkSecurityGroupResourceId: clusterNetworkSecurityGroup.outputs.resourceId + networkSecurityGroupResourceId: virtualNetworkNewOrExisting == 'new' ? clusterNetworkSecurityGroup.outputs.resourceId :null roleAssignments: [ { roleDefinitionIdOrName: 'Network Contributor' @@ -402,7 +417,7 @@ var subnets = { pods: { name: podSubnetName addressPrefix: podSubnetAddressPrefix - networkSecurityGroupResourceId: clusterNetworkSecurityGroup.outputs.resourceId + networkSecurityGroupResourceId: virtualNetworkNewOrExisting == 'new' ? clusterNetworkSecurityGroup.outputs.resourceId :null roleAssignments: [ { roleDefinitionIdOrName: 'Network Contributor' @@ -440,7 +455,7 @@ var subnets = { } } -module clusterNetworkSecurityGroup 'br/public:avm/res/network/network-security-group:0.1.0' = { +module clusterNetworkSecurityGroup 'br/public:avm/res/network/network-security-group:0.1.0' = if (virtualNetworkNewOrExisting == 'new') { name: '${commonLayerConfig.name}-network-security-group-cluster' params: { name: 'nsg-common${uniqueString(resourceGroup().id, 'common')}-aks' @@ -460,7 +475,7 @@ module clusterNetworkSecurityGroup 'br/public:avm/res/network/network-security-g } } -module bastionNetworkSecurityGroup 'br/public:avm/res/network/network-security-group:0.1.0' = if (enableBastion) { +module bastionNetworkSecurityGroup 'br/public:avm/res/network/network-security-group:0.1.0' = if (virtualNetworkNewOrExisting == 'new' && enableBastion) { name: '${commonLayerConfig.name}-network-security-group-bastion' params: { name: 'nsg-common${uniqueString(resourceGroup().id, 'common')}-bastion' @@ -485,7 +500,7 @@ module bastionNetworkSecurityGroup 'br/public:avm/res/network/network-security-g } } -module machineNetworkSecurityGroup 'br/public:avm/res/network/network-security-group:0.1.0' = if (enableBastion) { +module machineNetworkSecurityGroup 'br/public:avm/res/network/network-security-group:0.1.0' = if (virtualNetworkNewOrExisting == 'new' && enableBastion) { name: '${commonLayerConfig.name}-network-security-group-manage' params: { name: 'nsg-common${uniqueString(resourceGroup().id, 'common')}-vm' @@ -501,7 +516,7 @@ module machineNetworkSecurityGroup 'br/public:avm/res/network/network-security-g } } -module network 'br/public:avm/res/network/virtual-network:0.1.0' = { +module network 'br/public:avm/res/network/virtual-network:0.1.0' = if (virtualNetworkNewOrExisting == 'new') { name: '${commonLayerConfig.name}-virtual-network' params: { name: 'vnet-common${uniqueString(resourceGroup().id, 'common')}' @@ -781,6 +796,9 @@ module vaultEndpoint './modules/private-endpoint/main.bicep' = if (enablePrivate |_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| */ +@description('Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false.') +param enableBlobPublicAccess bool = true + var storageDNSZoneForwarder = 'blob.${environment().suffixes.storage}' var storageDnsZoneName = 'privatelink.${storageDNSZoneForwarder}' @@ -803,6 +821,9 @@ module configStorage './modules/storage-account/main.bicep' = { sku: commonLayerConfig.storage.sku tables: commonLayerConfig.storage.tables + // Apply Security + allowBlobPublicAccess: enableBlobPublicAccess + // Assign RBAC roleAssignments: [ { @@ -986,7 +1007,7 @@ module bastionHost 'br/public:avm/res/network/bastion-host:0.1.0' = if (enableBa name: '${manageLayerConfig.name}-bastion' params: { name: 'bh-${replace(manageLayerConfig.name, '-', '')}${uniqueString(resourceGroup().id, manageLayerConfig.name)}' - vNetId: network.outputs.resourceId + vNetId: vnetId[virtualNetworkNewOrExisting] location: location enableTelemetry: enableTelemetry } @@ -1290,6 +1311,9 @@ module partitionStorage './modules/storage-account/main.bicep' = [for (partition diagnosticWorkspaceId: logAnalytics.outputs.resourceId diagnosticLogsRetentionInDays: 0 + // Apply Security + allowBlobPublicAccess: enableBlobPublicAccess + // Configure Service sku: partitionLayerConfig.storage.sku containers: concat(partitionLayerConfig.storage.containers, [partition.name]) @@ -1452,7 +1476,7 @@ var serviceLayerConfig = { cluster: { aksVersion: '1.28' meshVersion: 'asm-1-18' - networkPlugin: 'kubenet' + networkPlugin: enablePodSubnet ? 'azure' : 'kubenet' } gitops: { name: 'flux-system' @@ -1508,6 +1532,11 @@ module cluster './modules/aks_cluster.bicep' = { identityId: stampIdentity.outputs.resourceId workspaceId: logAnalytics.outputs.resourceId + // Configure VNET Injection + serviceCidr: serviceCidr + dnsServiceIP: dnsServiceIP + dockerBridgeCidr: dockerBridgeCidr + // Configure Istio serviceMeshProfile: 'Istio' istioRevision: serviceLayerConfig.cluster.meshVersion @@ -1556,7 +1585,8 @@ module espool1 './modules/aks_agent_pool.bicep' = { availabilityZones: [ '1' ] - subnetId: '' + subnetId: virtualNetworkNewOrExisting != 'new' ? '${vnetId[virtualNetworkNewOrExisting]}/subnets/${aksSubnetName}' : '${vnetId[virtualNetworkNewOrExisting]}/subnets/${aksSubnetName}' + podSubnetId: virtualNetworkNewOrExisting != 'new' && enablePodSubnet ? '${vnetId[virtualNetworkNewOrExisting]}/subnets/${podSubnetName}' : '' nodeTaints: ['app=elasticsearch:NoSchedule'] nodeLabels: { app: 'elasticsearch' @@ -1575,7 +1605,8 @@ module espool2 './modules/aks_agent_pool.bicep' = { availabilityZones: [ '2' ] - subnetId: '' + subnetId: virtualNetworkNewOrExisting != 'new' ? '${vnetId[virtualNetworkNewOrExisting]}/subnets/${aksSubnetName}' : '${vnetId[virtualNetworkNewOrExisting]}/subnets/${aksSubnetName}' + podSubnetId: virtualNetworkNewOrExisting != 'new' && enablePodSubnet ? '${vnetId[virtualNetworkNewOrExisting]}/subnets/${podSubnetName}' : '' nodeTaints: ['app=elasticsearch:NoSchedule'] nodeLabels: { app: 'elasticsearch' @@ -1594,7 +1625,8 @@ module espool3 './modules/aks_agent_pool.bicep' = { availabilityZones: [ '3' ] - subnetId: '' + subnetId: virtualNetworkNewOrExisting != 'new' ? '${vnetId[virtualNetworkNewOrExisting]}/subnets/${aksSubnetName}' : '${vnetId[virtualNetworkNewOrExisting]}/subnets/${aksSubnetName}' + podSubnetId: virtualNetworkNewOrExisting != 'new' && enablePodSubnet ? '${vnetId[virtualNetworkNewOrExisting]}/subnets/${podSubnetName}' : '' nodeTaints: ['app=elasticsearch:NoSchedule'] nodeLabels: { app: 'elasticsearch' diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 16bbcd23..44adde38 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -1,33 +1,36 @@ { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "applicationClientId": { - "value": "${AZURE_CLIENT_ID}" - }, - "enablePodSubnet": { - "value": "${ENABLE_POD_SUBNET}" - }, - "enableVpnGateway": { - "value": "${ENABLE_VPN_GATEWAY}" - }, - "enableBastion": { - "value": "${ENABLE_BASTION}" - }, - "vpnSharedKey": { - "value": "${VPN_SHARED_KEY}" - }, - "remoteVpnAddress": { - "value": "${REMOTE_VPN_ADDRESS}" - }, - "remoteNetworkPrefix": { - "value": "${REMOTE_NETWORK_PREFIX}" - }, - "softwareRepository": { - "value": "${SOFTWARE_REPOSITORY}" - }, - "softwareBranch": { - "value": "${SOFTWARE_BRANCH}" - } + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "applicationClientId": { + "value": "${AZURE_CLIENT_ID}" + }, + "enablePodSubnet": { + "value": "${ENABLE_POD_SUBNET}" + }, + "enableVpnGateway": { + "value": "${ENABLE_VPN_GATEWAY}" + }, + "enableBastion": { + "value": "${ENABLE_BASTION}" + }, + "enableBlobPublicAccess": { + "value": "${ENABLE_BLOB_PUBLIC_ACCESS}" + }, + "vpnSharedKey": { + "value": "${VPN_SHARED_KEY}" + }, + "remoteVpnAddress": { + "value": "${REMOTE_VPN_ADDRESS}" + }, + "remoteNetworkPrefix": { + "value": "${REMOTE_NETWORK_PREFIX}" + }, + "softwareRepository": { + "value": "${SOFTWARE_REPOSITORY}" + }, + "softwareBranch": { + "value": "${SOFTWARE_BRANCH}" } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/bicep/modules/aks_agent_pool.bicep b/bicep/modules/aks_agent_pool.bicep index bf19810f..b8834b56 100644 --- a/bicep/modules/aks_agent_pool.bicep +++ b/bicep/modules/aks_agent_pool.bicep @@ -33,6 +33,9 @@ param nodeLabels object = {} @description('The subnet the node pool will use') param subnetId string +@description('The subnet the pods will use') +param podSubnetId string + @description('OS Type for the node pool') @allowed([ 'Linux' @@ -61,6 +64,7 @@ resource nodepool 'Microsoft.ContainerService/managedClusters/agentPools@2023-10 maxPods: maxPods type: 'VirtualMachineScaleSets' vnetSubnetID: !empty(subnetId) ? subnetId : null + podSubnetID: !empty(podSubnetId) ? podSubnetId : null upgradeSettings: { maxSurge: '33%' } diff --git a/bicep/modules/aks_cluster.bicep b/bicep/modules/aks_cluster.bicep index 3493e71e..908e1ee2 100644 --- a/bicep/modules/aks_cluster.bicep +++ b/bicep/modules/aks_cluster.bicep @@ -476,6 +476,7 @@ var userPoolProfile = { enableAutoScaling: true maxPods: userPoolPresets[clusterSize].maxPods vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : null + podSubnetID: !empty(aksPodSubnetId) ? aksPodSubnetId : null upgradeSettings: { maxSurge: '33%' } diff --git a/bicep/modules/storage-account/main.bicep b/bicep/modules/storage-account/main.bicep index 0fe1d80f..d52f08b1 100644 --- a/bicep/modules/storage-account/main.bicep +++ b/bicep/modules/storage-account/main.bicep @@ -143,6 +143,9 @@ param cmekConfiguration object = { @maxValue(365) param deleteRetention int = 0 +@description('Optional. Indicates whether public access is enabled for all blobs or containers in the storage account. For security reasons, it is recommended to set it to false.') +param allowBlobPublicAccess bool = false + var enableCMEK = !empty(cmekConfiguration.kvUrl) && !empty(cmekConfiguration.keyName) && !empty(cmekConfiguration.identityId) ? true : false var diagnosticsLogs = [for log in logsToEnable: { @@ -222,6 +225,8 @@ resource storage 'Microsoft.Storage/storageAccounts@2022-05-01' = { keySource: 'Microsoft.Storage' } + allowBlobPublicAccess: allowBlobPublicAccess + networkAcls: enablePrivateLink ? { bypass: 'AzureServices' defaultAction: 'Deny' diff --git a/bicep/modules/storage-account/test/main.test.bicep b/bicep/modules/storage-account/test/main.test.bicep index 83b93476..804c4e7e 100644 --- a/bicep/modules/storage-account/test/main.test.bicep +++ b/bicep/modules/storage-account/test/main.test.bicep @@ -13,7 +13,7 @@ param location string = resourceGroup().location module storage '../main.bicep' = { name: 'storage_account' params: { - resourceName: resourceName + name: 'sa${replace(resourceName, '-', '')}${uniqueString(resourceGroup().id, resourceName)}' location: location sku: 'Standard_LRS' } diff --git a/docs/vnet-injection.md b/docs/vnet-injection.md new file mode 100644 index 00000000..bd2db2f2 --- /dev/null +++ b/docs/vnet-injection.md @@ -0,0 +1,259 @@ +# Virtual Network Injection + +The provided custom deployment solution is a sample of how to enable and use virtual network (VNet) injection, allowing the creation of a network prior to deploying the solution. This is particularly beneficial in organizations with specific network security policies. Below is a tutorial that shows an example of a simple deployment leveraging network injection. + +## Planning + +Network planning is crucial when working with AKS and a workload for AKS. This is an advanced topic and the assumption when bringing your own network is that it has been planned properly. + +Several resources exist that can help on planning networks for AKS and to understand the networking concepts of AKS. + +- [AKS Network Topology and Connectivity](https://learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/app-platform/aks/network-topology-and-connectivity) + +- [Azure CNI Advanced Networking](https://learn.microsoft.com/en-us/azure/aks/concepts-network#azure-cni-advanced-networking) + +- [AKS Network Plugin Overviews](https://techcommunity.microsoft.com/t5/apps-on-azure-blog/how-to-choose-the-right-network-plugin-for-your-aks-cluster-a/ba-p/3969292) + +__Default Solution__ + +The default implementation uses a simple Virtual Network with a Kubenet plugin. One subnet which is provided to the AKS cluster is required, and additional subnets can be enabled for Bastion, VPN Gateway etc. + +- Virtual Network CIDR: `10.1.0.0/16` + +- Cluster Nodes Subnet CIDR: `10.1.0.0/20` + +- Gateway Subnet CIDR: `10.1.17.0/24` + +- Bastion Subnet CIDR: `10.1.16.0/24` + +- Virtual Machine Subnet CIDR: `10.1.18.0/24` + +- AKS Service CIDR: `172.16.0.0/16` + +- AKS DNS Service IP: `172.16.0.10` + +- Docker Bridge CIDR: `172.17.0.1/16` + +__Custom Solution__ + +This custom configuration tutorial will use a pre-created network along with a dedicated Pod Subnet which activates the `Azure CNI` network plugin. + +Things to considered when planning. + +- Virtual network + - A network can be as large as /8, but has a limit of 65,536 IP Address + +- Subnet + - A minimum subnet size: (number of nodes + 1) + ((number of nodes + 1) * maximum pods per node that you configure) + - Example 8 node cluster: (9) + (9 * 30 (default, 30 pods per node)) = 270 (/23 or larger) + +- Kubernetes Service Address + - Must be smaller then /12 + + +__Network Details__ + +For this example the following network details will be used. + +- Virtual Network CIDR: `172.18.0.0/22` + +- Cluster Nodes Subnet CIDR: `172.18.0.0/24` + +- Pod Subnet CIDR: `172.18.1.0/24` + +- AKS Service CIDR: `172.16.0.0/16` + +- AKS DNS Service IP: `172.16.0.10` + +- Docker Bridge CIDR: `172.17.0.1/16` + + +## Prepare a virtual network + +This section outlines the steps for manually creating a virtual network outside of the solution. + +__Resource Group__ + +Use the following command to create a new resource group: + +```bash +NETWORK_GROUP='operations' +AZURE_LOCATION='eastus' + +# resource_group +az group create --name $NETWORK_GROUP \ +--location $AZURE_LOCATION +``` + +__Network Security Group__ + +Network Security Groups (NSGs) are essential for securing virtual network resources. NSGs control inbound and outbound traffic to network interfaces (NIC), VMs, and subnets. + +Use the following commands set up an NSG with rules to allow HTTP and HTTPS traffic. + + +```bash +NSG_NAME='custom-vnet-nsg' + +# network_security_group +az network nsg create --name $NSG_NAME \ +--resource-group $NETWORK_GROUP \ +--location $AZURE_LOCATION + + +# http_inbound_rule +az network nsg rule create --name AllowHttpInbound \ +--nsg-name $NSG_NAME --resource-group $NETWORK_GROUP \ +--priority 200 --access Allow --direction Inbound \ +--protocol 'Tcp' --source-address-prefixes 'Internet' --source-port-ranges '*' \ +--destination-address-prefixes '*' --destination-port-ranges '80' + +# https_inbound_rule +az network nsg rule create --name AllowHttpsInbound \ +--nsg-name $NSG_NAME --resource-group $NETWORK_GROUP \ +--priority 210 --access Allow --direction Inbound \ +--protocol 'Tcp' --source-address-prefixes 'Internet' --source-port-ranges '*' \ +--destination-address-prefixes '*' --destination-port-ranges '443' +``` + + +__Virtual Network__ + +The virtual network is a critical component that enables Azure resources like AKS to communicate effectively. This step involves setting up the required 'ClusterSubnet' and an optional 'PodSubnet'. + +Use the following commands set up the network with a required subnet for the cluster and an optional subnet for the pods. + +```bash +NETWORK_NAME='custom-vnet' +VNET_PREFIX='172.18.0.0/22' + +CLUSTER_SUBNET_NAME='subnet1' +CLUSTER_SUBNET_PREFIX='172.18.0.0/24' + +POD_SUBNET_NAME='subnet2' +POD_SUBNET_PREFIX='172.18.1.0/24' + +# virtual_network +az network vnet create --name $NETWORK_NAME \ +--resource-group $NETWORK_GROUP \ +--location $AZURE_LOCATION \ +--address-prefix $VNET_PREFIX + +# virtual_network_subnet_cluster +az network vnet subnet create --name $CLUSTER_SUBNET_NAME \ +--resource-group $NETWORK_GROUP \ +--vnet-name $NETWORK_NAME \ +--address-prefix $CLUSTER_SUBNET_PREFIX \ +--network-security-group $NSG_NAME + +# virtual_network_subnet_pods +az network vnet subnet create --name $POD_SUBNET_NAME \ +--resource-group $NETWORK_GROUP \ +--vnet-name $NETWORK_NAME \ +--address-prefix $POD_SUBNET_PREFIX \ +--network-security-group $NSG_NAME +``` + + +## Initialize and Configure Solution + +This section provides the steps to authenticate your session then initialize a custom environment using Azure Developer CLI (azd). + + +__Authenticate and Initialize__ + +First, authenticate your session and then initialize a custom environment: + +```bash +# authenticate_session +azd auth login + +# initialize_environment +azd init -e custom +``` + +__Create Parameters__ + +```json +cat < bicep/main.parameters.json +{ + "\$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "applicationClientId": { + "value": "\${AZURE_CLIENT_ID}" + }, + "enablePodSubnet": { + "value": "\${ENABLE_POD_SUBNET}" + }, + "softwareRepository": { + "value": "\${SOFTWARE_REPOSITORY}" + }, + "softwareBranch": { + "value": "\${SOFTWARE_BRANCH}" + }, + "virtualNetworkNewOrExisting": { + "value": "Existing" + }, + "virtualNetworkResourceGroup": { + "value": "\${VIRTUAL_NETWORK_GROUP}" + }, + "virtualNetworkName": { + "value": "\${VIRTUAL_NETWORK_NAME}" + }, + "virtualNetworkAddressPrefix": { + "value": "\${VIRTUAL_NETWORK_PREFIX}" + }, + "aksSubnetName": { + "value": "\${AKS_SUBNET_NAME}" + }, + "aksSubnetAddressPrefix": { + "value": "\${AKS_SUBNET_PREFIX}" + }, + "podSubnetName": { + "value": "\${POD_SUBNET_NAME}" + }, + "podSubnetAddressPrefix": { + "value": "\${POD_SUBNET_PREFIX}" + } + } +} +EOF + +``` + +__Configure Environment Variables__ + +Set the necessary environment variables for your deployment: + +```bash +# define_application_id +APP_NAME= +azd env set AZURE_CLIENT_ID $(az ad app list --display-name $APP_NAME --query "[].appId" -otsv) + +# identify_software_repository +azd env set SOFTWARE_REPOSITORY https://github.com/azure/osdu-developer +azd env set SOFTWARE_BRANCH main + +# enable_pod_subnet +azd env set ENABLE_POD_SUBNET true + +# define_network_configuration +azd env set VIRTUAL_NETWORK_GROUP $NETWORK_GROUP +azd env set VIRTUAL_NETWORK_NAME $NETWORK_NAME +azd env set VIRTUAL_NETWORK_PREFIX $VNET_PREFIX +azd env set AKS_SUBNET_NAME $CLUSTER_SUBNET_NAME +azd env set AKS_SUBNET_PREFIX $CLUSTER_SUBNET_PREFIX +azd env set POD_SUBNET_NAME $POD_SUBNET_NAME +azd env set POD_SUBNET_PREFIX $POD_SUBNET_PREFIX +``` + +__Start the Deployment__ + +Initiate the deployment using the following command: + +```bash +# provision_solution +azd provision +``` +