diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aec9907..63fddfa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,16 +34,21 @@ env: ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} # other - # prod or staging - CERT_API_ENVIRONMENT: prod + # prod or staging. "" disables cert-manager annotations + CERT_API_ENVIRONMENT: "" DEMO_USER_USERNAME: demo_user # DEMO_USER_PASSWORD: ${{ secrets.DEMO_USER_PASSWORD }} DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk + DNS_RG_NAME: rg-dns DOCKER_FQDN: docker.thehypepipe.co.uk EMAIL_ADDRESS: certadmin@domain.com ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true + K8S_TLS_SECRET_NAME: tls-secret + KEY_VAULT_NAME: kv-rush-iz6y + KEY_VAULT_CERT_NAME: wildcard-thehypepipe-co-uk + KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} @@ -81,17 +86,16 @@ jobs: # ref: develop # Init tasks - inc Env var concatenation - # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - name: Init tasks - inc Env var concatenation (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - echo ::set-env name=DNS_RG_NAME::${{ env.PREFIX }}-rg-externaldns - echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001 - echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001 - echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001 - echo ::set-env name=VELERO_STORAGE_ACCOUNT::${{ env.PREFIX }}stbckuksouth001 + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV # Show event info - name: Show triggered event data @@ -257,8 +261,8 @@ jobs: # Init tasks - inc Env var concatenation - name: Init tasks - inc Env var concatenation (Workaround) run: | - echo "::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001" - echo "::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001" + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV # Login - name: Login to Azure diff --git a/.github/workflows/deallocate_aks_vmss.yml b/.github/workflows/deallocate_aks_vmss.yml index 01195d9..161e4d3 100644 --- a/.github/workflows/deallocate_aks_vmss.yml +++ b/.github/workflows/deallocate_aks_vmss.yml @@ -43,9 +43,8 @@ jobs: - name: Init tasks - inc Env var concatenation (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV # Show event info - name: Show triggered event data run: pwsh -command "./scripts/Get-EventData.ps1" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 027cdee..9ed843a 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -33,13 +33,19 @@ env: # prod or staging CERT_API_ENVIRONMENT: staging DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk + DNS_RG_NAME: rg-dns EMAIL_ADDRESS: certadmin@domain.com ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true + K8S_TLS_SECRET_NAME: tls-secret + KEY_VAULT_NAME: kv-rush-iz6y + KEY_VAULT_CERT_NAME: wildcard-thehypepipe-co-uk + KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot LOCATION: uksouth + ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: false + VELERO_ENABLED: true # terraform TF_IN_AUTOMATION: "true" @@ -69,16 +75,16 @@ jobs: # ref: develop # Env var concatenation - # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - name: Concatenate env vars (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001 - echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001 - echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001 - echo ::set-env name=VELERO_STORAGE_ACCOUNT::${{ env.PREFIX }}stbckuksouth001 + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV # Show event info - name: Show triggered event data diff --git a/.github/workflows/start_aks_vmss.yml b/.github/workflows/start_aks_vmss.yml index a64dafb..04d7d12 100644 --- a/.github/workflows/start_aks_vmss.yml +++ b/.github/workflows/start_aks_vmss.yml @@ -43,8 +43,8 @@ jobs: - name: Init tasks - inc Env var concatenation (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV # Show event info - name: Show triggered event data diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1c9790..924fc7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 - name: BASH - Concatenate env vars (Workaround) run: | - echo ::set-env name=MY_CONCATENATED_BASH_VAR::${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-bash-stepvalue01 + echo "MY_CONCATENATED_BASH_VAR=${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-bash-stepvalue01 >> $GITHUB_ENV - name: PWSH - Concatenate env vars (Workaround) # override the default bash shell, as running on ubuntu @@ -66,7 +66,7 @@ jobs: shell: pwsh # pwsh requires double quotes when setting env vars run: | - echo "::set-env name=MY_CONCATENATED_PWSH_VAR::${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-pwsh-stepvalue01" + echo ""MY_CONCATENATED_PWSH_VAR=${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-pwsh-stepvalue01" >> $GITHUB_ENV - name: Output event data shell: pwsh diff --git a/aad-pod-identity/README.md b/aad-pod-identity/README.md index 0ce7bd9..8d134b1 100644 --- a/aad-pod-identity/README.md +++ b/aad-pod-identity/README.md @@ -25,7 +25,7 @@ Before automating the installation and configuration of aad-pod-identity, follow ### Deploy aad-pod-identity using Helm 3 -Repo: [https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity](https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity) +Repo: [aad-pod-identity chart](https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity) ```bash # Navigate to aad-pod-identity folder @@ -44,7 +44,7 @@ helm search repo aad-pod-identity --version ^2.0.0 kubectl create namespace aad-pod-identity # Install aad-pod-identity -helm upgrade aad-pod-identity aad-pod-identity/aad-pod-identity --version 2.0.1 --values aad_pod_identity_values.yaml --install --atomic --namespace aad-pod-identity --debug +helm upgrade aad-pod-identity aad-pod-identity/aad-pod-identity --version 2.0.2 --values aad_pod_identity_values.yaml --set=installCRDs=true --install --atomic --namespace aad-pod-identity --debug ``` ### Configure aad-pod-identity @@ -70,9 +70,20 @@ export IDENTITY_RESOURCE_ID="$(az identity show -g "$AKS_NODE_RESOURCE_GROUP_NAM echo "IDENTITY_RESOURCE_ID: $IDENTITY_RESOURCE_ID" echo "IDENTITY_CLIENT_ID: $IDENTITY_CLIENT_ID" +# [OPTIONAL] Check assigned VMSS identity +az vmss identity show -g "$AKS_NODE_RESOURCE_GROUP_NAME" -n "aks-default-39636823-vmss" + # Assign the identity a role export IDENTITY_ASSIGNMENT_ID="$(az role assignment create --role Contributor --assignee "$IDENTITY_CLIENT_ID" --scope "$AKS_NODE_RESOURCE_GROUP_ID" --query id -o tsv)" +# Describe AzureIdentity CRDs (they dont have metadata) +kubectl describe AzureIdentity velero +kubectl explain --recursive AzureIdentity +kubectl explain --recursive backups +kubectl explain --recursive AzureIdentity.spec +kubectl explain --recursive AzureIdentityBinding +kubectl explain --recursive AzureIdentityBinding.spec + # Create an AzureIdentity cat < # vmType: <`standard` for normal virtual machine nodes, and `vmss` for cluster deployed with a virtual machine scale set> # tenantID: -# clientID: -# clientSecret: - +# clientID: +# clientSecret: +# useMSI: +# userAssignedMSIClientID: # Operation mode for pod-identity. Default is standard mode that has MIC doing identity assignment # Allowed values: "standard", "managed" operationMode: "standard" mic: image: mic - tag: 1.6.2 + tag: v1.6.3 priorityClassName: "" @@ -111,9 +118,13 @@ mic: # Default value is 1s updateUserMSIRetryInterval: "" + # The interval between reconciling identity assignment on Azure based on an existing list of AzureAssignedIdentities + # Default value is 3m + identityAssignmentReconcileInterval: "" + nmi: image: nmi - tag: 1.6.2 + tag: v1.6.3 priorityClassName: "" diff --git a/scripts/Replace-Tokens.ps1 b/scripts/Replace-Tokens.ps1 index 11db016..11fb741 100755 --- a/scripts/Replace-Tokens.ps1 +++ b/scripts/Replace-Tokens.ps1 @@ -1,6 +1,6 @@ # Replace tokens param ( - $TargetFilePattern = './terraform/*.tf', + $TargetFilePattern = @('./terraform/*.tf', './terraform/files/*.yaml'), $TokenPrefix = '__', $TokenSuffix = '__', diff --git a/scripts/storage_key.sh b/scripts/storage_key.sh index 558a4e1..1b2a20a 100644 --- a/scripts/storage_key.sh +++ b/scripts/storage_key.sh @@ -17,9 +17,9 @@ echo "FINISHED: $taskMessage." taskMessage="Updating workflow env vars" echo "STARTED: $taskMessage..." -# https://help.github.com/en/actions/reference/development-tools-for-github-actions#set-an-environment-variable-set-env -# ::set-env name={name}::{value} -echo "::set-env name=STORAGE_KEY::$storage_key" +# # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files +# echo "{name}={value}" >> $GITHUB_ENV +echo "STORAGE_KEY=$storage_key" >> "$GITHUB_ENV" # Mask sensitive env var # https://help.github.com/en/actions/reference/development-tools-for-github-actions#example-masking-an-environment-variable diff --git a/terraform/aks.tf b/terraform/aks.tf index 2bc5f21..a4ec96a 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -82,36 +82,28 @@ resource "azurerm_kubernetes_cluster" "aks" { } } - - # TODO DELETE SECTION - # service_principal block: https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#client_id - # service_principal { - # client_id = var.service_principal_client_id - # client_secret = var.service_principal_client_secret - # } - # TODO DELETE SECTION - - # managed identity block: https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#type-1 identity { type = "SystemAssigned" } # TODO Enable RBAC and AAD auth: https://app.zenhub.com/workspaces/aks-nexus-velero-5e602702ee332f0fc76d35dd/issues/adamrushuk/aks-nexus-velero/105 - # role_based_access_control { - # enabled = true - - # azure_active_directory { - # managed = true - # admin_group_object_ids = [ - # data.azuread_group.aks.id - # ] - # } - # } + role_based_access_control { + enabled = true + + # azure_active_directory { + # managed = true + # admin_group_object_ids = [ + # data.azuread_group.aks.id + # ] + # } + } addon_profile { + # cannot remove this deprecated block yet, due to this issue: + # https://github.com/terraform-providers/terraform-provider-azurerm/issues/7716 kube_dashboard { - enabled = var.aks_dashboard_enabled + enabled = false } oms_agent { diff --git a/terraform/data.tf b/terraform/data.tf index 56510ac..6e06f9a 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -4,3 +4,7 @@ data "azurerm_subscription" "current" {} data "azuread_group" "aks" { name = var.aad_group_name } + +data "azurerm_resource_group" "aks_node_rg" { + name = azurerm_kubernetes_cluster.aks.node_resource_group +} diff --git a/terraform/dns.tf b/terraform/dns.tf index 5b7b801..a5ebed0 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -1,17 +1,11 @@ # DNS -resource "azurerm_resource_group" "dns" { +data "azurerm_resource_group" "dns" { name = var.dns_resource_group_name - location = var.location - tags = var.tags - - lifecycle { - ignore_changes = [tags] - } } -resource "azurerm_dns_zone" "dns" { +data "azurerm_dns_zone" "dns" { name = var.dns_zone_name - resource_group_name = azurerm_resource_group.dns.name + resource_group_name = data.azurerm_resource_group.dns.name } @@ -48,7 +42,7 @@ resource "azuread_service_principal_password" "aks_dns_sp" { resource "azurerm_role_assignment" "aks_dns_sp_to_rg" { principal_id = azuread_service_principal.aks_dns_sp.id role_definition_name = "Reader" - scope = azurerm_dns_zone.dns.id + scope = data.azurerm_dns_zone.dns.id skip_service_principal_aad_check = true depends_on = [azuread_service_principal_password.aks_dns_sp] } @@ -57,7 +51,7 @@ resource "azurerm_role_assignment" "aks_dns_sp_to_rg" { resource "azurerm_role_assignment" "aks_dns_sp_to_zone" { principal_id = azuread_service_principal.aks_dns_sp.id role_definition_name = "Contributor" - scope = azurerm_resource_group.dns.id + scope = data.azurerm_resource_group.dns.id skip_service_principal_aad_check = true depends_on = [azuread_service_principal_password.aks_dns_sp] } @@ -77,7 +71,7 @@ resource "kubernetes_secret" "external_dns" { "aadClientSecret": "${random_string.aks_dns_sp.result}", "tenantId": "${data.azurerm_subscription.current.tenant_id}", "subscriptionId": "${data.azurerm_subscription.current.subscription_id}", - "resourceGroup": "${azurerm_resource_group.dns.name}" + "resourceGroup": "${data.azurerm_resource_group.dns.name}" } EOT } diff --git a/terraform/files/AzureKeyVaultSecret.yaml b/terraform/files/AzureKeyVaultSecret.yaml new file mode 100644 index 0000000..b3f5b40 --- /dev/null +++ b/terraform/files/AzureKeyVaultSecret.yaml @@ -0,0 +1,103 @@ +# https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml +# +# https://akv2k8s.io/reference/azure-key-vault-secret/ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: azurekeyvaultsecrets.spv.no + labels: + app.kubernetes.io/name: akv2k8s + annotations: + "helm.sh/resource-policy": keep +spec: + group: spv.no + names: + kind: AzureKeyVaultSecret + listKind: AzureKeyVaultSecretList + plural: azurekeyvaultsecrets + singular: azurekeyvaultsecret + shortNames: + - akvs + categories: + - all + additionalPrinterColumns: + - name: Vault + type: string + description: Which Azure Key Vault this resource is asosiated with + JSONPath: .spec.vault.name + - name: Vault Object + type: string + description: Which Azure Key Vault object this resource is asosiated with + JSONPath: .spec.vault.object.name + - name: Secret Name + type: string + description: Which Kubernetes Secret this resource is synched with, if any + JSONPath: .status.secretName + - name: Synched + type: string + description: When this resource was last synched with Azure Key Vault + JSONPath: .status.lastAzureUpdate + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: false + - name: v1 + served: true + storage: true + + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + required: ['vault'] + properties: + vault: + required: ['name', 'object'] + properties: + name: + type: string + description: Name of the Azure Key Vault + object: + required: ['name', 'type'] + properties: + name: + type: string + description: The object name in Azure Key Vault + type: + type: string + description: The type of object in Azure Key Vault + enum: + - secret + - certificate + - key + - multi-key-value-secret + version: + type: string + description: The object version in Azure Key Vault + contentType: + type: string + description: Only used when type is multi-key-value-secret. + enum: + - application/x-json + - application/x-yaml + output: + properties: + transform: + type: array + items: + type: string + secret: + required: ['name'] + properties: + name: + type: string + description: Name for Kubernetes secret + type: + type: string + description: Type of Secret in Kubernetes + dataKey: + type: string + description: The key to use in Kubernetes secret when setting the value from Azure Keyv Vault object data diff --git a/terraform/files/akv2k8s-exception.yaml b/terraform/files/akv2k8s-exception.yaml new file mode 100644 index 0000000..b5dee1c --- /dev/null +++ b/terraform/files/akv2k8s-exception.yaml @@ -0,0 +1,9 @@ +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzurePodIdentityException +metadata: + name: akv2k8s-exception + namespace: akv2k8s # change if akv2k8s is installed in a different ns +spec: + podLabels: + app.kubernetes.io/instance: akv2k8s + app.kubernetes.io/name: controller diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/akvs-certificate-sync.yaml new file mode 100644 index 0000000..33b5682 --- /dev/null +++ b/terraform/files/akvs-certificate-sync.yaml @@ -0,0 +1,17 @@ +# https://akv2k8s.io/tutorials/sync/2-certificate/ +apiVersion: spv.no/v1alpha1 +kind: AzureKeyVaultSecret +metadata: + name: certificate-sync + namespace: nexus +spec: + vault: + name: __KEY_VAULT_NAME__ # name of key vault + object: + name: __KEY_VAULT_CERT_NAME__ # key vault certificate name + type: certificate + output: + secret: + name: __K8S_TLS_SECRET_NAME__ # kubernetes secret name + type: kubernetes.io/tls # kubernetes secret type + chainOrder: ensureserverfirst diff --git a/terraform/files/azureIdentities.yaml.tpl b/terraform/files/azureIdentities.yaml.tpl new file mode 100644 index 0000000..a37dcce --- /dev/null +++ b/terraform/files/azureIdentities.yaml.tpl @@ -0,0 +1,13 @@ +azureIdentities: + - name: "velero" + # if not defined, then the azure identity will be deployed in the same namespace as the chart + namespace: "" + # type 0: MSI, type 1: Service Principal + type: 0 + # /subscriptions/subscription-id/resourcegroups/resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/identity-name + resourceID: "${resourceID}" + clientID: "${clientID}" + binding: + name: "velero-binding" + # The selector will also need to be included in labels for app deployment + selector: "velero" diff --git a/terraform/function_app.tf b/terraform/function_app.tf index 3145930..e2d625f 100644 --- a/terraform/function_app.tf +++ b/terraform/function_app.tf @@ -120,12 +120,8 @@ resource "azurerm_function_app" "func_app" { # Give Function App Reader role for the AKS cluster node resource group -data "azurerm_resource_group" "aks" { - name = azurerm_kubernetes_cluster.aks.node_resource_group -} - resource "azurerm_role_assignment" "func_app" { - scope = data.azurerm_resource_group.aks.id + scope = data.azurerm_resource_group.aks_node_rg.id role_definition_name = "Reader" principal_id = azurerm_function_app.func_app.identity.0.principal_id } diff --git a/terraform/helm/aad_pod_identity_values.yaml b/terraform/helm/aad_pod_identity_values.yaml new file mode 100644 index 0000000..cbc226f --- /dev/null +++ b/terraform/helm/aad_pod_identity_values.yaml @@ -0,0 +1,206 @@ +# source: https://github.com/Azure/aad-pod-identity/blob/v1.6.3/charts/aad-pod-identity/values.yaml + +# Default values for aad-pod-identity-helm. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nameOverride: "" +fullnameOverride: "" + +image: + repository: mcr.microsoft.com/oss/azure/aad-pod-identity + imagePullPolicy: Always + +# One or more secrets to be used when pulling images +# imagePullSecrets: +# - name: myRegistryKeySecretName + +# https://github.com/Azure/aad-pod-identity#4-optional-match-pods-in-the-namespace +# By default, AAD Pod Identity matches pods to identities across namespaces. +# To match only pods in the namespace containing AzureIdentity set this to true. +forceNameSpaced: "false" + +# When NMI runs on a node where MIC is running, then MIC token request call is also +# intercepted by NMI. MIC can't get a valid token as to initialize and then +# assign the identity. Installing an exception for MIC would ensure all token requests +# for MIC pods directly go to IMDS and not go through the pod-identity validation +# https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.app-exception.md +installMICException: "true" + +## If using a separate service principal for aad-pod-identity instead of cluster service principal specify the following +## (The chart will perform the base64 encoding for you for values that are stored in secrets.) +adminsecret: {} +# cloud: +# subscriptionID: +# resourceGroup: +# vmType: <`standard` for normal virtual machine nodes, and `vmss` for cluster deployed with a virtual machine scale set> +# tenantID: +# clientID: +# clientSecret: +# useMSI: +# userAssignedMSIClientID: +# Operation mode for pod-identity. Default is standard mode that has MIC doing identity assignment +# Allowed values: "standard", "managed" +operationMode: "standard" + +mic: + image: mic + tag: v1.6.3 + + priorityClassName: "" + + # log level. Uses V logs (glog) + logVerbosity: 0 + + resources: + limits: + cpu: 200m + memory: 1024Mi + requests: + cpu: 100m + memory: 256Mi + + podAnnotations: {} + + ## Node labels for pod assignment + ## aad-pod-identity is currently only supported on linux + nodeSelector: + kubernetes.io/os: linux + + tolerations: [] + + affinity: {} + + leaderElection: + # Override leader election instance name (default is 'hostname') + instance: "" + # Override the namespace to create leader election objects (default is default namespace) + namespace: "" + # Override leader election name (default is aad-pod-identity-mic) + name: "" + # Override leader election duration (default is 15s) + duration: "" + + # Override http liveliness probe port (default is 8080) + probePort: "" + + # Override interval in seconds at which sync loop should periodically check for errors and reconcile (default is 3600s) + syncRetryDuration: "" + + # Override the defult value of immutable identities. + immutableUserMSIs: [] + # Example of MSIs (should be replaced with the real client ids) + #- "00000000-0000-0000-0000-000000000000" + #- "11111111-1111-1111-1111-111111111111" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#batch-create-delete-flag + # default value is 20 + createDeleteBatch: "" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#client-qps-flag + # default value is 5 + clientQps: "" + + # default value is 8888 + # prometheus port for metrics + prometheusPort: "" + + # cloud configuration used to authenticate with Azure + cloudConfig: "/etc/kubernetes/azure.json" + + # The maximum retry of UpdateUserMSI call. MIC updates all the identities in a batch. If a single identity contains an error + # or is invalid, then the entire operation fails. Configuring this flag will make MIC retry by removing the erroneous identities + # returned in the error + # Default value is 2. + updateUserMSIMaxRetry: "" + + # The duration to wait before retrying UpdateUserMSI (batch assigning/un-assigning identity from VM/VMSS) in case of errors + # Default value is 1s + updateUserMSIRetryInterval: "" + + # The interval between reconciling identity assignment on Azure based on an existing list of AzureAssignedIdentities + # Default value is 3m + identityAssignmentReconcileInterval: "" + +nmi: + image: nmi + tag: v1.6.3 + + priorityClassName: "" + + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + + podAnnotations: {} + + ## Node labels for pod assignment + ## aad-pod-identity is currently only supported on linux + nodeSelector: + kubernetes.io/os: linux + + tolerations: [] + + affinity: {} + + # Override iptables update interval in seconds (default is 60) + ipTableUpdateTimeIntervalInSeconds: "" + + # Override mic namespace to short circuit MIC token requests (default is default namespace) + micNamespace: "" + + # Override http liveliness probe port (default is 8080) + probePort: "8085" + + # Override number of retries in NMI to find assigned identity in CREATED state (default is 16) + retryAttemptsForCreated: "" + + # Override number of retries in NMI to find assigned identity in ASSIGNED state (default is 4) + retryAttemptsForAssigned: "" + + # Override retry interval to find assigned identities in seconds (default is 5) + findIdentityRetryIntervalInSeconds: "" + + # Enable scale features - https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#enable-scale-features-flag + # Accepted values are true/false. Default is false. + enableScaleFeatures: "" + + # default value is 9090 + # prometheus port for metrics + prometheusPort: "" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#block-instance-metadata-flag + # default is false + blockInstanceMetadata: "" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#metadata-header-required-flag + # default is false + metadataHeaderRequired: "" + +rbac: + enabled: true + # NMI requires permissions to get secrets when service principal (type: 1) is used in AzureIdentity. + # If using only MSI (type: 0) in AzureIdentity, secret get permission can be disabled by setting this to false. + allowAccessToSecrets: true + +# Create azure identities and bindings +azureIdentities: [] + # - name: "velero" + # # if not defined, then the azure identity will be deployed in the same namespace as the chart + # namespace: "" + # # type 0: MSI, type 1: Service Principal + # type: 0 + # # /subscriptions/subscription-id/resourcegroups/resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/identity-name + # resourceID: "" + # clientID: "" + # binding: + # name: "velero-binding" + # # The selector will also need to be included in labels for app deployment + # selector: "velero" + +# If true, install necessary custom resources. +installCRDs: false diff --git a/terraform/helm/velero_values.yaml b/terraform/helm/velero_values.yaml index 81acdf8..5f7a0e5 100644 --- a/terraform/helm/velero_values.yaml +++ b/terraform/helm/velero_values.yaml @@ -1,4 +1,5 @@ -# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.12.17/charts/velero/values.yaml +# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.13.6/charts/velero/values.yaml + ## ## Configuration settings that directly affect the Velero deployment YAML. ## @@ -8,7 +9,7 @@ image: # https://hub.docker.com/r/velero/velero/tags repository: velero/velero - tag: v1.5.1 + tag: v1.5.2 # Digest value example: sha256:d238835e151cec91c6a811fe3a89a66d3231d9f64d09e5f3c49552672d271f38. If used, it will # take precedence over the image.tag. # digest: @@ -21,14 +22,13 @@ image: # # If using kube2iam or kiam, use the following annotation with your AWS_ACCOUNT_ID # and VELERO_ROLE_NAME filled in: -# iam.amazonaws.com/role: arn:aws:iam:::role/ podAnnotations: {} + # iam.amazonaws.com/role: "arn:aws:iam:::role/" # Additional pod labels for Velero deployment's template. Optional # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ podLabels: {} - # Resource requests/limits to specify for the Velero deployment. Optional. resources: {} @@ -36,10 +36,8 @@ resources: {} # https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/releases initContainers: - name: velero-plugin-for-microsoft-azure - # use "main" for development build # https://hub.docker.com/r/velero/velero-plugin-for-microsoft-azure/tags - image: velero/velero-plugin-for-microsoft-azure:v1.1.0 - # image: velero/velero-plugin-for-microsoft-azure:main + image: velero/velero-plugin-for-microsoft-azure:v1.1.1 imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /target @@ -51,6 +49,9 @@ initContainers: securityContext: {} # fsGroup: 1337 +# Pod priority class name to use for the Velero deployment. Optional. +priorityClassName: "" + # Tolerations to use for the Velero deployment. Optional. tolerations: [] @@ -93,7 +94,7 @@ configuration: provider: azure # Parameters for the `default` BackupStorageLocation. See - # https://velero.io/docs/v1.4/api-types/backupstoragelocation/ + # https://velero.io/docs/v1.5/api-types/backupstoragelocation/ backupStorageLocation: # name is the name of the backup storage location where backups should be stored. If a name is not provided, # a backup storage location will be created with the name "default". Optional. @@ -124,7 +125,7 @@ configuration: # serviceAccount: # Parameters for the `default` VolumeSnapshotLocation. See - # https://velero.io/docs/v1.4/api-types/volumesnapshotlocation/ + # https://velero.io/docs/v1.5/api-types/volumesnapshotlocation/ volumeSnapshotLocation: # name is the name of the volume snapshot location where snapshots are being taken. Required. name: default @@ -168,6 +169,9 @@ configuration: # Set log-format for Velero pod. Default: text. Other option: json. logFormat: + # Set true for backup all pod volumes without having to apply annotation on the pod when used restic Default: false. Other option: false. + defaultVolumesToRestic: + ## ## End of backup/snapshot location settings. ## @@ -220,7 +224,7 @@ restic: podVolumePath: /var/lib/kubelet/pods privileged: false # Pod priority class name to use for the Restic daemonset. Optional. - priorityClassName: {} + priorityClassName: "" # Resource requests/limits to specify for the Restic daemonset deployment. Optional. resources: {} # Tolerations to use for the Restic daemonset. Optional. @@ -242,13 +246,13 @@ restic: # Eg: # schedules: # mybackup: -# labels: -# myenv: foo +# labels: +# myenv: foo # schedule: "0 0 * * *" # template: # ttl: "240h" # includedNamespaces: -# - foo +# - foo schedules: {} # Velero ConfigMaps. diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf new file mode 100644 index 0000000..d36e336 --- /dev/null +++ b/terraform/helm_aad_pod_identity.tf @@ -0,0 +1,73 @@ +# aad-pod-identity helm chart + +# role assignment for aad-pod-identity +# https://azure.github.io/aad-pod-identity/docs/getting-started/role-assignment/#performing-role-assignments +resource "azurerm_role_assignment" "aks_mi_aks_node_rg_vm_contributor" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "Virtual Machine Contributor" + scope = data.azurerm_resource_group.aks_node_rg.id + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "aks_mi_aks_node_rg_mi_operator" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "Managed Identity Operator" + scope = data.azurerm_resource_group.aks_node_rg.id + skip_service_principal_aad_check = true +} + +# velero user MI in different RG, so assign role there too +resource "azurerm_role_assignment" "aks_mi_velero_rg_mi_operator" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "Managed Identity Operator" + scope = azurerm_resource_group.velero[0].id + skip_service_principal_aad_check = true +} + +data "template_file" "azureIdentities" { + count = var.velero_enabled ? 1 : 0 + template = file("${path.module}/files/azureIdentities.yaml.tpl") + vars = { + resourceID = azurerm_user_assigned_identity.velero[0].id + clientID = azurerm_user_assigned_identity.velero[0].client_id + } +} + +# https://www.terraform.io/docs/providers/kubernetes/r/namespace.html +resource "kubernetes_namespace" "aad_pod_identity" { + count = var.velero_enabled ? 1 : 0 + metadata { + name = "aad-pod-identity" + } + timeouts { + delete = "15m" + } + + depends_on = [azurerm_kubernetes_cluster.aks] +} + +# https://www.terraform.io/docs/providers/helm/r/release.html +resource "helm_release" "aad_pod_identity" { + count = var.velero_enabled ? 1 : 0 + chart = "aad-pod-identity" + name = "aad-pod-identity" + namespace = "aad-pod-identity" + repository = "https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts" + version = var.aad_pod_identity_chart_version + + values = [ + file("helm/aad_pod_identity_values.yaml"), + data.template_file.azureIdentities[0].rendered + ] + + set { + name = "installCRDs" + value = "true" + } + + timeout = 600 + depends_on = [kubernetes_namespace.aad_pod_identity[0]] +} diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf new file mode 100644 index 0000000..96dbf36 --- /dev/null +++ b/terraform/helm_akv2k8s.tf @@ -0,0 +1,107 @@ +# Azure Key Vault to Kubernetes (akv2k8s) makes Azure Key Vault secrets, certificates and keys available in +# Kubernetes and/or your application - in a simple and secure way +# +# https://akv2k8s.io/ +# https://github.com/SparebankenVest/azure-key-vault-to-kubernetes + +# Key vault access policy for AKS / akv2k8s +data "azurerm_key_vault" "kv" { + name = var.key_vault_name + resource_group_name = var.key_vault_resource_group_name +} + +resource "azurerm_key_vault_access_policy" "aks" { + key_vault_id = data.azurerm_key_vault.kv.id + + tenant_id = data.azurerm_subscription.current.tenant_id + object_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + + certificate_permissions = [ + "get" + ] + + key_permissions = [ + "get" + ] + + secret_permissions = [ + "get" + ] +} + + +resource "local_file" "kubeconfig" { + sensitive_content = azurerm_kubernetes_cluster.aks.kube_config_raw + filename = var.aks_config_path + + depends_on = [azurerm_kubernetes_cluster.aks] +} + +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "akv2k8s_crds" { + triggers = { + # always_run = "${timestamp()}" + akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + cert_sync_yaml_contents = filemd5(var.cert_sync_yaml_path) + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = <