From 56e15184ef97a9f7aceb579ed2694f6fe35f272b Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 08:18:54 -0500 Subject: [PATCH 01/55] airflow adjustments. --- bicep/modules/aks-config-map/main.bicep | 4 +- bicep/modules/aks-config-map/main.json | 231 --- .../aks-config-map/test/prereq.test.bicep | 2 +- bicep/modules/aks-run-command/main.bicep | 6 +- bicep/modules/aks-run-command/main.json | 211 --- bicep/modules/cosmos-db/main.json | 1645 ----------------- bicep/modules/private-endpoint/main.json | 438 ----- bicep/modules/script-blob-upload/main.bicep | 6 +- .../modules/script-kv-certificate/main.bicep | 6 +- bicep/modules/script-kv-certificate/main.json | 345 ---- bicep/modules/script-share-upload/main.bicep | 6 +- bicep/modules/script-sshkeypair/main.bicep | 6 +- bicep/modules/script-sshkeypair/main.json | 198 -- software/components/airflow/release.yaml | 41 +- 14 files changed, 46 insertions(+), 3099 deletions(-) delete mode 100644 bicep/modules/aks-config-map/main.json delete mode 100644 bicep/modules/aks-run-command/main.json delete mode 100644 bicep/modules/cosmos-db/main.json delete mode 100644 bicep/modules/private-endpoint/main.json delete mode 100644 bicep/modules/script-kv-certificate/main.json delete mode 100644 bicep/modules/script-sshkeypair/main.json diff --git a/bicep/modules/aks-config-map/main.bicep b/bicep/modules/aks-config-map/main.bicep index 2904722f..1cc7d923 100644 --- a/bicep/modules/aks-config-map/main.bicep +++ b/bicep/modules/aks-config-map/main.bicep @@ -103,8 +103,8 @@ resource runAksCommand 'Microsoft.Resources/deploymentScripts@2023-08-01' = { ] properties: { forceUpdateTag: forceUpdateTag - azCliVersion: '2.61.0' - timeout: 'PT15M' + azCliVersion: '2.63.0' + timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ { name: 'RG', value: resourceGroup().name } diff --git a/bicep/modules/aks-config-map/main.json b/bicep/modules/aks-config-map/main.json deleted file mode 100644 index 2a068654..00000000 --- a/bicep/modules/aks-config-map/main.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.24.24.22086", - "templateHash": "4801579897722142587" - }, - "name": "AKS Config Map", - "description": "An Azure CLI Deployment Script that allows you to create a helm char on a Kubernetes cluster.", - "owner": "Daniel Scholl" - }, - "parameters": { - "aksName": { - "type": "string", - "metadata": { - "description": "The name of the Azure Kubernetes Service" - } - }, - "location": { - "type": "string", - "metadata": { - "description": "The location to deploy the resources to" - } - }, - "forceUpdateTag": { - "type": "string", - "defaultValue": "[utcNow()]", - "metadata": { - "description": "How the deployment script should be forced to execute" - } - }, - "rbacRolesNeeded": { - "type": "array", - "defaultValue": [ - "b24988ac-6180-42a0-ab88-20f7382dd24c", - "7f6c6a51-bcf8-42ba-9220-52d62157d7db" - ], - "metadata": { - "description": "An array of Azure RoleIds that are required for the DeploymentScript resource" - } - }, - "newOrExistingManagedIdentity": { - "type": "string", - "defaultValue": "new", - "allowedValues": [ - "new", - "existing" - ], - "metadata": { - "description": "Create \"new\" or use \"existing\" Managed Identity. Default: new" - } - }, - "managedIdentityName": { - "type": "string", - "defaultValue": "[format('id-AksConfigMap-{0}', parameters('location'))]", - "metadata": { - "description": "Name of the Managed Identity resource" - } - }, - "existingManagedIdentitySubId": { - "type": "string", - "defaultValue": "[subscription().subscriptionId]", - "metadata": { - "description": "For an existing Managed Identity, the Subscription Id it is located in" - } - }, - "existingManagedIdentityResourceGroupName": { - "type": "string", - "defaultValue": "[resourceGroup().name]", - "metadata": { - "description": "For an existing Managed Identity, the Resource Group it is located in" - } - }, - "name": { - "type": "string", - "defaultValue": "configuration", - "metadata": { - "description": "Specify the config map name." - } - }, - "namespace": { - "type": "string", - "defaultValue": "default", - "metadata": { - "description": "Specify the namespace for the config mapl" - } - }, - "propertyData": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Specify the config map single property data. (e.g. \"player_initial_lives=3\")" - } - }, - "fileData": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Specify the config map file data. (e.g. \"game.properties: |enemy.types=aliens,monsters\nplayer.maximum-lives=5\")" - } - }, - "initialScriptDelay": { - "type": "string", - "defaultValue": "120s", - "metadata": { - "description": "A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate" - } - }, - "cleanupPreference": { - "type": "string", - "defaultValue": "OnSuccess", - "allowedValues": [ - "OnSuccess", - "OnExpiration", - "Always" - ], - "metadata": { - "description": "When the script resource is cleaned up" - } - }, - "isCrossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Set to true when deploying template across tenants" - } - } - }, - "variables": { - "$fxv#0": "\nset -e +H\n# -e to exit on error\n# +H to prevent history expansion\n\n# This script is used to apply a ConfigMap to an AKS cluster using the az aks command invoke command.\n\nif [ \"$initialDelay\" != \"0\" ]\nthen\n echo \"Waiting on RBAC replication ($initialDelay)\"\n sleep $initialDelay\n\n #Force RBAC refresh\n az logout\n az login --identity\nfi\n\n# Function to convert semi-colon-separated key-value pairs in $dataPropertyLike to YAML format\nconvert_properties_to_yaml() {\n local IFS=\";\"\n for pair in $dataPropertyLike; do\n IFS='=' read -r key value <<< \"$pair\"\n echo \" $key: \\\"$value\\\"\"\n done\n echo \"\" # Add an empty line for separation\n}\n\n# Function to append file-like data in $dataFileLike to YAML format, converting \\t to spaces\nappend_files_to_yaml() {\n local IFS=\";\"\n for file in $dataFileLike; do\n local name=\"${file%%: *}\"\n local content=\"${file#*: |}\"\n # Process content to ensure correct new line handling and indentation\n content=$(echo \"$content\" | sed 's/\\\\n/\\n/g' | sed 's/^/ /') # Adjust for actual new lines and indent\n echo \" $name: |\"\n echo \"$content\"\n done\n}\n\n\necho \"Checking and updating configmap $configMap in AKS Cluster $aksName in $RG\"\n\n\n# Combining property-like and file-like data into the ConfigMap\ncombinedYaml=$(cat < $AZ_SCRIPTS_OUTPUT_PATH\n", - "useExistingManagedIdentity": "[equals(parameters('newOrExistingManagedIdentity'), 'existing')]", - "delegatedManagedIdentityResourceId": "[if(variables('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')))]" - }, - "resources": [ - { - "condition": "[not(variables('useExistingManagedIdentity'))]", - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2023-01-31", - "name": "[parameters('managedIdentityName')]", - "location": "[parameters('location')]" - }, - { - "copy": { - "name": "rbac", - "count": "[length(parameters('rbacRolesNeeded'))]" - }, - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('aksName'))]", - "name": "[guid(resourceId('Microsoft.ContainerService/managedClusters', parameters('aksName')), parameters('rbacRolesNeeded')[copyIndex()], if(variables('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]", - "properties": { - "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', parameters('rbacRolesNeeded')[copyIndex()])]", - "principalId": "[if(variables('useExistingManagedIdentity'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2023-01-31').principalId, reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2023-01-31').principalId)]", - "principalType": "ServicePrincipal", - "delegatedManagedIdentityResourceId": "[if(parameters('isCrossTenant'), variables('delegatedManagedIdentityResourceId'), null())]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]" - ] - }, - { - "type": "Microsoft.Resources/deploymentScripts", - "apiVersion": "2020-10-01", - "name": "[format('script-{0}-{1}', parameters('aksName'), deployment().name)]", - "location": "[parameters('location')]", - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[format('{0}', if(variables('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]": {} - } - }, - "kind": "AzureCLI", - "properties": { - "forceUpdateTag": "[parameters('forceUpdateTag')]", - "azCliVersion": "2.54.0", - "timeout": "PT10M", - "retentionInterval": "P1D", - "environmentVariables": [ - { - "name": "RG", - "value": "[resourceGroup().name]" - }, - { - "name": "aksName", - "value": "[parameters('aksName')]" - }, - { - "name": "configMap", - "value": "[parameters('name')]" - }, - { - "name": "namespace", - "value": "[parameters('namespace')]" - }, - { - "name": "dataPropertyLike", - "value": "[join(parameters('propertyData'), ';')]" - }, - { - "name": "dataFileLike", - "value": "[join(parameters('fileData'), ';')]" - }, - { - "name": "initialDelay", - "value": "[parameters('initialScriptDelay')]" - } - ], - "scriptContent": "[variables('$fxv#0')]", - "cleanupPreference": "[parameters('cleanupPreference')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]", - "rbac" - ] - } - ], - "outputs": { - "commandOutput": { - "type": "object", - "metadata": { - "description": "Array of command output from each Deployment Script AKS run command" - }, - "value": { - "Name": "[format('script-{0}-{1}', parameters('aksName'), deployment().name)]", - "CommandOutput": "[reference(resourceId('Microsoft.Resources/deploymentScripts', format('script-{0}-{1}', parameters('aksName'), deployment().name)), '2020-10-01').outputs]" - } - } - } -} \ No newline at end of file diff --git a/bicep/modules/aks-config-map/test/prereq.test.bicep b/bicep/modules/aks-config-map/test/prereq.test.bicep index 363131c1..53305fd2 100644 --- a/bicep/modules/aks-config-map/test/prereq.test.bicep +++ b/bicep/modules/aks-config-map/test/prereq.test.bicep @@ -39,7 +39,7 @@ output aksName string = aks.name // ] // properties: { // forceUpdateTag: forceUpdateTag -// azCliVersion: '2.35.0' +// azCliVersion: '2.63.0' // timeout: 'PT30M' // retentionInterval: 'P1D' // scriptContent: 'sleep 120s' diff --git a/bicep/modules/aks-run-command/main.bicep b/bicep/modules/aks-run-command/main.bicep index 74412d5e..4127ffc0 100644 --- a/bicep/modules/aks-run-command/main.bicep +++ b/bicep/modules/aks-run-command/main.bicep @@ -73,7 +73,7 @@ resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for roleDe }] @batchSize(1) -resource runAksCommand 'Microsoft.Resources/deploymentScripts@2020-10-01' = [for (command, i) in commands: { +resource runAksCommand 'Microsoft.Resources/deploymentScripts@2023-08-01' = [for (command, i) in commands: { name: 'script-${aks.name}-${deployment().name}-${i}' location: location identity: { @@ -88,8 +88,8 @@ resource runAksCommand 'Microsoft.Resources/deploymentScripts@2020-10-01' = [for ] properties: { forceUpdateTag: forceUpdateTag - azCliVersion: '2.61.0' - timeout: 'PT15M' + azCliVersion: '2.63.0' + timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ { name: 'RG', value: resourceGroup().name } diff --git a/bicep/modules/aks-run-command/main.json b/bicep/modules/aks-run-command/main.json deleted file mode 100644 index 57809185..00000000 --- a/bicep/modules/aks-run-command/main.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.19.5.34762", - "templateHash": "14134297598922776972" - }, - "name": "AKS Run Command Script", - "description": "An Azure CLI Deployment Script that allows you to run a command on a Kubernetes cluster.", - "owner": "Aks-Bicep-Accelerator-Maintainers" - }, - "parameters": { - "aksName": { - "type": "string", - "metadata": { - "description": "The name of the Azure Kubernetes Service" - } - }, - "location": { - "type": "string", - "metadata": { - "description": "The location to deploy the resources to" - } - }, - "forceUpdateTag": { - "type": "string", - "defaultValue": "[utcNow()]", - "metadata": { - "description": "How the deployment script should be forced to execute" - } - }, - "rbacRolesNeeded": { - "type": "array", - "defaultValue": [ - "b24988ac-6180-42a0-ab88-20f7382dd24c", - "7f6c6a51-bcf8-42ba-9220-52d62157d7db" - ], - "metadata": { - "description": "An array of Azure RoleIds that are required for the DeploymentScript resource" - } - }, - "newOrExistingManagedIdentity": { - "type": "string", - "defaultValue": "new", - "allowedValues": [ - "new", - "existing" - ], - "metadata": { - "description": "Create \"new\" or use \"existing\" Managed Identity. Default: new" - } - }, - "managedIdentityName": { - "type": "string", - "defaultValue": "[format('id-AksRunCommandProxy-{0}', parameters('location'))]", - "metadata": { - "description": "Name of the Managed Identity resource" - } - }, - "existingManagedIdentitySubId": { - "type": "string", - "defaultValue": "[subscription().subscriptionId]", - "metadata": { - "description": "For an existing Managed Identity, the Subscription Id it is located in" - } - }, - "existingManagedIdentityResourceGroupName": { - "type": "string", - "defaultValue": "[resourceGroup().name]", - "metadata": { - "description": "For an existing Managed Identity, the Resource Group it is located in" - } - }, - "commands": { - "type": "array", - "metadata": { - "description": "An array of commands to run" - } - }, - "initialScriptDelay": { - "type": "string", - "defaultValue": "120s", - "metadata": { - "description": "A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate" - } - }, - "cleanupPreference": { - "type": "string", - "defaultValue": "OnSuccess", - "metadata": { - "description": "When the script resource is cleaned up" - }, - "allowedValues": [ - "OnSuccess", - "OnExpiration", - "Always" - ] - }, - "isCrossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Set to true when deploying template across tenants" - } - } - }, - "variables": { - "$fxv#0": "#!/bin/bash\n\nset -e +H\n# -e to exit on error\n# +H to prevent history expansion\n\nif [ \"$loopIndex\" == \"0\" ] && [ \"$initialDelay\" != \"0\" ]\nthen\n echo \"Waiting on RBAC replication ($initialDelay)\"\n sleep $initialDelay\n\n #Force RBAC refresh\n az logout\n az login --identity\nfi\n\necho \"Sending command $command to AKS Cluster $aksName in $RG\"\ncmdOut=$(az aks command invoke -g $RG -n $aksName -o json --command \"${command}\")\necho $cmdOut\n\njsonOutputString=$cmdOut\necho $jsonOutputString > $AZ_SCRIPTS_OUTPUT_PATH\n", - "useExistingManagedIdentity": "[equals(parameters('newOrExistingManagedIdentity'), 'existing')]", - "delegatedManagedIdentityResourceId": "[if(variables('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')))]" - }, - "resources": [ - { - "condition": "[not(variables('useExistingManagedIdentity'))]", - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2023-01-31", - "name": "[parameters('managedIdentityName')]", - "location": "[parameters('location')]" - }, - { - "copy": { - "name": "rbac", - "count": "[length(parameters('rbacRolesNeeded'))]" - }, - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('aksName'))]", - "name": "[guid(resourceId('Microsoft.ContainerService/managedClusters', parameters('aksName')), parameters('rbacRolesNeeded')[copyIndex()], if(variables('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]", - "properties": { - "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', parameters('rbacRolesNeeded')[copyIndex()])]", - "principalId": "[if(variables('useExistingManagedIdentity'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2023-01-31').principalId, reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2023-01-31').principalId)]", - "principalType": "ServicePrincipal", - "delegatedManagedIdentityResourceId": "[if(parameters('isCrossTenant'), variables('delegatedManagedIdentityResourceId'), null())]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]" - ] - }, - { - "copy": { - "name": "runAksCommand", - "count": "[length(parameters('commands'))]", - "mode": "serial", - "batchSize": 1 - }, - "type": "Microsoft.Resources/deploymentScripts", - "apiVersion": "2020-10-01", - "name": "[format('AKS-Run-{0}-{1}-{2}', parameters('aksName'), deployment().name, copyIndex())]", - "location": "[parameters('location')]", - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[format('{0}', if(variables('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]": {} - } - }, - "kind": "AzureCLI", - "properties": { - "forceUpdateTag": "[parameters('forceUpdateTag')]", - "azCliVersion": "2.35.0", - "timeout": "PT10M", - "retentionInterval": "P1D", - "environmentVariables": [ - { - "name": "RG", - "value": "[resourceGroup().name]" - }, - { - "name": "aksName", - "value": "[parameters('aksName')]" - }, - { - "name": "command", - "value": "[parameters('commands')[copyIndex()]]" - }, - { - "name": "initialDelay", - "value": "[parameters('initialScriptDelay')]" - }, - { - "name": "loopIndex", - "value": "[string(copyIndex())]" - } - ], - "scriptContent": "[variables('$fxv#0')]", - "cleanupPreference": "[parameters('cleanupPreference')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]", - "rbac" - ] - } - ], - "outputs": { - "commandOutput": { - "type": "array", - "metadata": { - "description": "Array of command output from each Deployment Script AKS run command" - }, - "copy": { - "count": "[length(parameters('commands'))]", - "input": { - "Index": "[copyIndex()]", - "Name": "[format('AKS-Run-{0}-{1}-{2}', parameters('aksName'), deployment().name, copyIndex())]", - "CommandOutput": "[reference(resourceId('Microsoft.Resources/deploymentScripts', format('AKS-Run-{0}-{1}-{2}', parameters('aksName'), deployment().name, copyIndex())), '2020-10-01').outputs]" - } - } - } - } -} \ No newline at end of file diff --git a/bicep/modules/cosmos-db/main.json b/bicep/modules/cosmos-db/main.json deleted file mode 100644 index 7e289f41..00000000 --- a/bicep/modules/cosmos-db/main.json +++ /dev/null @@ -1,1645 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "692756270669382771" - } - }, - "parameters": { - "resourceName": { - "type": "string", - "metadata": { - "description": "Used to name all resources" - }, - "maxLength": 20, - "minLength": 3 - }, - "resourceLocation": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "Optional: Resource Location." - } - }, - "tags": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Tags." - } - }, - "enableDeleteLock": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Enable lock to prevent accidental deletion" - } - }, - "multiwriteRegions": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Locations enabled for the Cosmos DB account." - } - }, - "maxThroughput": { - "type": "int", - "defaultValue": 4000, - "metadata": { - "description": "Optional. Represents maximum throughput, the resource can scale up to. Cannot be set together with `throughput`. If `throughput` is set to something else than -1, this autoscale setting is ignored." - } - }, - "throughput": { - "type": "int", - "defaultValue": -1, - "metadata": { - "description": "Optional. Request Units per second (for example 10000). Cannot be set together with `maxThroughput`." - } - }, - "systemAssignedIdentity": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Enables system assigned managed identity on the resource." - } - }, - "userAssignedIdentities": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Optional. The ID(s) to assign to the resource." - } - }, - "defaultIdentity": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The default identity to be used." - } - }, - "databaseAccountOfferType": { - "type": "string", - "defaultValue": "Standard", - "allowedValues": [ - "Standard" - ], - "metadata": { - "description": "Optional. The offer type for the Cosmos DB database account." - } - }, - "defaultConsistencyLevel": { - "type": "string", - "defaultValue": "Session", - "metadata": { - "description": "Optional. The default consistency level of the Cosmos DB account." - }, - "allowedValues": [ - "Eventual", - "ConsistentPrefix", - "Session", - "BoundedStaleness", - "Strong" - ] - }, - "automaticFailover": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Optional. Enable automatic failover for regions." - } - }, - "maxStalenessPrefix": { - "type": "int", - "defaultValue": 100000, - "metadata": { - "description": "Optional. Max stale requests. Required for BoundedStaleness. Valid ranges, Single Region: 10 to 1000000. Multi Region: 100000 to 1000000." - }, - "maxValue": 2147483647, - "minValue": 10 - }, - "maxIntervalInSeconds": { - "type": "int", - "defaultValue": 300, - "metadata": { - "description": "Optional. Max lag time (minutes). Required for BoundedStaleness. Valid ranges, Single Region: 5 to 84600. Multi Region: 300 to 86400." - }, - "maxValue": 86400, - "minValue": 5 - }, - "serverVersion": { - "type": "string", - "defaultValue": "4.2", - "allowedValues": [ - "3.2", - "3.6", - "4.0", - "4.2" - ], - "metadata": { - "description": "Optional. Specifies the MongoDB server version to use." - } - }, - "sqlDatabases": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. SQL Databases configurations." - } - }, - "gremlinDatabases": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Gremlin Databases configurations." - } - }, - "mongodbDatabases": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. MongoDB Databases configurations." - } - }, - "capabilitiesToAdd": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. List of Cosmos DB capabilities for the account." - }, - "allowedValues": [ - "EnableCassandra", - "EnableTable", - "EnableGremlin", - "EnableMongo", - "DisableRateLimitingResponses", - "EnableServerless" - ] - }, - "backupPolicyType": { - "type": "string", - "defaultValue": "Periodic", - "metadata": { - "description": "Optional. Describes the mode of backups." - }, - "allowedValues": [ - "Periodic", - "Continuous" - ] - }, - "backupPolicyContinuousTier": { - "type": "string", - "defaultValue": "Continuous30Days", - "metadata": { - "description": "Optional. Configuration values for continuous mode backup." - }, - "allowedValues": [ - "Continuous30Days", - "Continuous7Days" - ] - }, - "backupIntervalInMinutes": { - "type": "int", - "defaultValue": 240, - "metadata": { - "description": "Optional. An integer representing the interval in minutes between two backups. Only applies to periodic backup type." - }, - "maxValue": 1440, - "minValue": 60 - }, - "backupRetentionIntervalInHours": { - "type": "int", - "defaultValue": 8, - "metadata": { - "description": "Optional. An integer representing the time (in hours) that each backup is retained. Only applies to periodic backup type." - }, - "maxValue": 720, - "minValue": 2 - }, - "backupStorageRedundancy": { - "type": "string", - "defaultValue": "Local", - "metadata": { - "description": "Optional. Enum to indicate type of backup residency. Only applies to periodic backup type." - }, - "allowedValues": [ - "Geo", - "Local", - "Zone" - ] - }, - "roleAssignments": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Array of objects that describe RBAC permissions, format { roleDefinitionResourceId (string), principalId (string), principalType (enum), enabled (bool) }. Ref: https://docs.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments?tabs=bicep" - } - }, - "diagnosticWorkspaceId": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. Resource ID of the diagnostic log analytics workspace." - } - }, - "diagnosticStorageAccountId": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. Resource ID of the diagnostic storage account." - } - }, - "diagnosticEventHubAuthorizationRuleId": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. Resource ID of the diagnostic event hub authorization rule for the Event Hubs namespace in which the event hub should be created or streamed to." - } - }, - "diagnosticEventHubName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. Name of the diagnostic event hub within the namespace to which logs are streamed. Without this, an event hub is created for each log category." - } - }, - "diagnosticLogsRetentionInDays": { - "type": "int", - "defaultValue": 365, - "maxValue": 365, - "minValue": 0, - "metadata": { - "description": "Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely." - } - }, - "logsToEnable": { - "type": "array", - "defaultValue": [ - "DataPlaneRequests", - "MongoRequests", - "QueryRuntimeStatistics", - "PartitionKeyStatistics", - "PartitionKeyRUConsumption", - "ControlPlaneRequests", - "CassandraRequests", - "GremlinRequests", - "TableApiRequests" - ], - "allowedValues": [ - "DataPlaneRequests", - "MongoRequests", - "QueryRuntimeStatistics", - "PartitionKeyStatistics", - "PartitionKeyRUConsumption", - "ControlPlaneRequests", - "CassandraRequests", - "GremlinRequests", - "TableApiRequests" - ], - "metadata": { - "description": "Optional. The name of logs that will be streamed." - } - }, - "metricsToEnable": { - "type": "array", - "defaultValue": [ - "Requests" - ], - "allowedValues": [ - "Requests" - ], - "metadata": { - "description": "Optional. The name of metrics that will be streamed." - } - }, - "kvKeyUri": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. Customer Managed Encryption Key." - } - }, - "crossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment's principal object." - } - }, - "privateLinkSettings": { - "type": "object", - "defaultValue": { - "subnetId": "1", - "vnetId": "1" - }, - "metadata": { - "description": "Settings Required to Enable Private Link" - } - }, - "keyVaultName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional: Key Vault Name to store secrets into" - } - }, - "databaseEndpointSecretName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional: To save storage account name into vault set the secret hame." - } - }, - "databasePrimaryKeySecretName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional: To save storage account key into vault set the secret hame." - } - }, - "databaseConnectionStringSecretName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional: To save storage account connectionstring into vault set the secret hame." - } - } - }, - "variables": { - "copy": [ - { - "name": "diagnosticsLogs", - "count": "[length(parameters('logsToEnable'))]", - "input": { - "category": "[parameters('logsToEnable')[copyIndex('diagnosticsLogs')]]", - "enabled": true, - "retentionPolicy": { - "enabled": true, - "days": "[parameters('diagnosticLogsRetentionInDays')]" - } - } - }, - { - "name": "diagnosticsMetrics", - "count": "[length(parameters('metricsToEnable'))]", - "input": { - "category": "[parameters('metricsToEnable')[copyIndex('diagnosticsMetrics')]]", - "timeGrain": null, - "enabled": true, - "retentionPolicy": { - "enabled": true, - "days": "[parameters('diagnosticLogsRetentionInDays')]" - } - } - }, - { - "name": "databaseAccount_locations", - "count": "[length(parameters('multiwriteRegions'))]", - "input": { - "failoverPriority": "[parameters('multiwriteRegions')[copyIndex('databaseAccount_locations')].failoverPriority]", - "isZoneRedundant": "[parameters('multiwriteRegions')[copyIndex('databaseAccount_locations')].isZoneRedundant]", - "locationName": "[parameters('multiwriteRegions')[copyIndex('databaseAccount_locations')].locationName]" - } - }, - { - "name": "capabilities", - "count": "[length(parameters('capabilitiesToAdd'))]", - "input": { - "name": "[parameters('capabilitiesToAdd')[copyIndex('capabilities')]]" - } - } - ], - "name": "[format('dba-{0}{1}', replace(parameters('resourceName'), '-', ''), uniqueString(resourceGroup().id, parameters('resourceName')))]", - "identityType": "[if(parameters('systemAssignedIdentity'), if(not(empty(parameters('userAssignedIdentities'))), 'SystemAssigned,UserAssigned', 'SystemAssigned'), if(not(empty(parameters('userAssignedIdentities'))), 'UserAssigned', 'None'))]", - "consistencyPolicy": { - "Eventual": { - "defaultConsistencyLevel": "Eventual" - }, - "ConsistentPrefix": { - "defaultConsistencyLevel": "ConsistentPrefix" - }, - "Session": { - "defaultConsistencyLevel": "Session" - }, - "BoundedStaleness": { - "defaultConsistencyLevel": "BoundedStaleness", - "maxStalenessPrefix": "[parameters('maxStalenessPrefix')]", - "maxIntervalInSeconds": "[parameters('maxIntervalInSeconds')]" - }, - "Strong": { - "defaultConsistencyLevel": "Strong" - } - }, - "kind": "[if(or(not(empty(parameters('sqlDatabases'))), not(empty(parameters('gremlinDatabases')))), 'GlobalDocumentDB', if(not(empty(parameters('mongodbDatabases'))), 'MongoDB', 'Parse'))]", - "enableReferencedModulesTelemetry": false, - "backupPolicy": "[if(equals(parameters('backupPolicyType'), 'Continuous'), createObject('type', parameters('backupPolicyType'), 'continuousModeProperties', createObject('tier', parameters('backupPolicyContinuousTier'))), createObject('type', parameters('backupPolicyType'), 'periodicModeProperties', createObject('backupIntervalInMinutes', parameters('backupIntervalInMinutes'), 'backupRetentionIntervalInHours', parameters('backupRetentionIntervalInHours'), 'backupStorageRedundancy', parameters('backupStorageRedundancy'))))]", - "databaseAccount_properties": "[union(createObject('databaseAccountOfferType', parameters('databaseAccountOfferType')), if(or(or(not(empty(parameters('sqlDatabases'))), not(empty(parameters('mongodbDatabases')))), not(empty(parameters('gremlinDatabases')))), createObject('consistencyPolicy', variables('consistencyPolicy')[parameters('defaultConsistencyLevel')], 'enableMultipleWriteLocations', if(empty(parameters('multiwriteRegions')), false(), true()), 'locations', if(empty(parameters('multiwriteRegions')), createArray(createObject('failoverPriority', 0, 'isZoneRedundant', false(), 'locationName', parameters('resourceLocation'))), variables('databaseAccount_locations')), 'capabilities', variables('capabilities'), 'backupPolicy', variables('backupPolicy')), createObject()), if(not(empty(parameters('sqlDatabases'))), createObject('enableAutomaticFailover', parameters('automaticFailover'), 'AnalyticalStorageConfiguration', createObject('schemaType', 'WellDefined'), 'defaultIdentity', if(not(empty(parameters('defaultIdentity'))), format('UserAssignedIdentity={0}', parameters('defaultIdentity')), 'FirstPartyIdentity'), 'enablePartitionKeyMonitor', true(), 'enablePartitionMerge', false(), 'keyVaultKeyUri', if(not(empty(parameters('kvKeyUri'))), parameters('kvKeyUri'), json('null'))), createObject()), if(not(empty(parameters('mongodbDatabases'))), createObject('apiProperties', createObject('serverVersion', parameters('serverVersion'))), createObject('EnabledApiTypes', createArray('Sql'))))]", - "enablePrivateLink": "[and(not(equals(parameters('privateLinkSettings').vnetId, '1')), not(equals(parameters('privateLinkSettings').subnetId, '1')))]", - "privateEndpointName": "[format('{0}-PrivateEndpoint', variables('name'))]", - "privateDNSZoneName": "privatelink.documents.azure.com" - }, - "resources": [ - { - "type": "Microsoft.DocumentDB/databaseAccounts", - "apiVersion": "2022-08-15", - "name": "[if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))]", - "location": "[parameters('resourceLocation')]", - "tags": "[parameters('tags')]", - "identity": { - "type": "[variables('identityType')]", - "userAssignedIdentities": "[if(not(empty(parameters('userAssignedIdentities'))), parameters('userAssignedIdentities'), createObject())]" - }, - "kind": "[variables('kind')]", - "properties": "[variables('databaseAccount_properties')]" - }, - { - "condition": "[parameters('enableDeleteLock')]", - "type": "Microsoft.Authorization/locks", - "apiVersion": "2017-04-01", - "scope": "[format('Microsoft.DocumentDB/databaseAccounts/{0}', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]", - "name": "[format('{0}-lock', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]", - "properties": { - "level": "CanNotDelete" - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "condition": "[or(or(or(not(empty(parameters('diagnosticStorageAccountId'))), not(empty(parameters('diagnosticWorkspaceId')))), not(empty(parameters('diagnosticEventHubAuthorizationRuleId')))), not(empty(parameters('diagnosticEventHubName'))))]", - "type": "Microsoft.Insights/diagnosticSettings", - "apiVersion": "2021-05-01-preview", - "scope": "[format('Microsoft.DocumentDB/databaseAccounts/{0}', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]", - "name": "storage-diagnostics", - "properties": { - "storageAccountId": "[if(not(empty(parameters('diagnosticStorageAccountId'))), parameters('diagnosticStorageAccountId'), null())]", - "workspaceId": "[if(not(empty(parameters('diagnosticWorkspaceId'))), parameters('diagnosticWorkspaceId'), null())]", - "eventHubAuthorizationRuleId": "[if(not(empty(parameters('diagnosticEventHubAuthorizationRuleId'))), parameters('diagnosticEventHubAuthorizationRuleId'), null())]", - "eventHubName": "[if(not(empty(parameters('diagnosticEventHubName'))), parameters('diagnosticEventHubName'), null())]", - "metrics": "[variables('diagnosticsMetrics')]", - "logs": "[variables('diagnosticsLogs')]", - "logAnalyticsDestinationType": "AzureDiagnostics" - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "condition": "[variables('enablePrivateLink')]", - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2021-02-01", - "name": "[variables('privateEndpointName')]", - "location": "[parameters('resourceLocation')]", - "properties": { - "subnet": { - "id": "[parameters('privateLinkSettings').subnetId]" - }, - "privateLinkServiceConnections": [ - { - "name": "[variables('privateEndpointName')]", - "properties": { - "privateLinkServiceId": "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]", - "groupIds": [ - "Sql" - ] - } - } - ], - "customDnsConfigs": [ - { - "fqdn": "[variables('privateDNSZoneName')]" - } - ] - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "condition": "[variables('enablePrivateLink')]", - "type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks", - "apiVersion": "2020-06-01", - "name": "[format('{0}/{1}', variables('privateDNSZoneName'), format('{0}-link', variables('privateDNSZoneName')))]", - "location": "global", - "properties": { - "registrationEnabled": false, - "virtualNetwork": { - "id": "[parameters('privateLinkSettings').vnetId]" - } - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDNSZoneName'))]" - ] - }, - { - "condition": "[variables('enablePrivateLink')]", - "type": "Microsoft.Network/privateDnsZones", - "apiVersion": "2020-06-01", - "name": "[variables('privateDNSZoneName')]", - "location": "global" - }, - { - "condition": "[variables('enablePrivateLink')]", - "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", - "apiVersion": "2022-01-01", - "name": "[format('{0}/{1}', variables('privateEndpointName'), 'dnsgroupname')]", - "properties": { - "privateDnsZoneConfigs": [ - { - "name": "config1", - "properties": { - "privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDNSZoneName'))]" - } - } - ] - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDNSZoneName'))]", - "[resourceId('Microsoft.Network/privateEndpoints', variables('privateEndpointName'))]" - ] - }, - { - "copy": { - "name": "databaseAccount_sqlDatabases", - "count": "[length(parameters('sqlDatabases'))]" - }, - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-{1}', deployment().name, parameters('sqlDatabases')[copyIndex()].name)]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "databaseAccountName": { - "value": "[if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))]" - }, - "name": { - "value": "[parameters('sqlDatabases')[copyIndex()].name]" - }, - "throughput": { - "value": "[parameters('throughput')]" - }, - "maxThroughput": { - "value": "[parameters('maxThroughput')]" - }, - "containers": "[if(contains(parameters('sqlDatabases')[copyIndex()], 'containers'), createObject('value', parameters('sqlDatabases')[copyIndex()].containers), createObject('value', createArray()))]", - "enableDefaultTelemetry": { - "value": "[variables('enableReferencedModulesTelemetry')]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "11026842919452399369" - } - }, - "parameters": { - "databaseAccountName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent Database Account. Required if the template is used in a standalone deployment." - } - }, - "name": { - "type": "string", - "metadata": { - "description": "Required. Name of the SQL database ." - } - }, - "containers": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Array of containers to deploy in the SQL database." - } - }, - "maxThroughput": { - "type": "int", - "defaultValue": 400, - "metadata": { - "description": "Optional. Represents maximum throughput, the resource can scale up to. Cannot be set together with `throughput`. If `throughput` is set to something else than -1, this autoscale setting is ignored." - } - }, - "throughput": { - "type": "int", - "defaultValue": -1, - "metadata": { - "description": "Optional. Request Units per second (for example 10000). Cannot be set together with `maxThroughput`." - } - }, - "tags": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Optional. Tags of the SQL database resource." - } - }, - "enableDefaultTelemetry": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Optional. Enable telemetry via the Customer Usage Attribution ID (GUID)." - } - } - }, - "variables": { - "enableReferencedModulesTelemetry": false - }, - "resources": [ - { - "condition": "[parameters('enableDefaultTelemetry')]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2021-04-01", - "name": "[format('pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-{0}', uniqueString(deployment().name))]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [] - } - } - }, - { - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases", - "apiVersion": "2022-08-15", - "name": "[format('{0}/{1}', parameters('databaseAccountName'), parameters('name'))]", - "tags": "[parameters('tags')]", - "properties": { - "options": "[if(contains(reference(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), '2022-08-15').capabilities, createObject('name', 'EnableServerless')), createObject(), createObject('autoscaleSettings', if(equals(parameters('throughput'), -1), createObject('maxThroughput', parameters('maxThroughput')), null()), 'throughput', if(not(equals(parameters('throughput'), -1)), parameters('throughput'), null())))]", - "resource": { - "id": "[parameters('name')]" - } - } - }, - { - "copy": { - "name": "container", - "count": "[length(parameters('containers'))]" - }, - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-{1}-sql-{2}', deployment().name, parameters('name'), copyIndex())]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "databaseAccountName": { - "value": "[parameters('databaseAccountName')]" - }, - "sqlDatabaseName": { - "value": "[parameters('name')]" - }, - "name": { - "value": "[parameters('containers')[copyIndex()].name]" - }, - "paths": { - "value": "[parameters('containers')[copyIndex()].paths]" - }, - "kind": { - "value": "[parameters('containers')[copyIndex()].kind]" - }, - "enableDefaultTelemetry": { - "value": "[variables('enableReferencedModulesTelemetry')]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "3987175024047996725" - } - }, - "parameters": { - "databaseAccountName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent Database Account. Required if the template is used in a standalone deployment." - } - }, - "sqlDatabaseName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent SQL Database. Required if the template is used in a standalone deployment." - } - }, - "name": { - "type": "string", - "metadata": { - "description": "Required. Name of the container." - } - }, - "throughput": { - "type": "int", - "defaultValue": -1, - "metadata": { - "description": "Optional. Request Units per second (for example 10000). Cannot be set together with `maxThroughput`." - } - }, - "tags": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Optional. Tags of the SQL Database resource." - } - }, - "paths": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. List of paths using which data within the container can be partitioned." - } - }, - "uniqueKeyPaths": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. List of unique key paths using which data within the container can be partitioned." - } - }, - "kind": { - "type": "string", - "defaultValue": "Hash", - "allowedValues": [ - "Hash", - "MultiHash", - "Range" - ], - "metadata": { - "description": "Optional. Indicates the kind of algorithm used for partitioning." - } - }, - "enableDefaultTelemetry": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Optional. Enable telemetry via the Customer Usage Attribution ID (GUID)." - } - } - }, - "resources": [ - { - "condition": "[parameters('enableDefaultTelemetry')]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2021-04-01", - "name": "[format('pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-{0}', uniqueString(deployment().name))]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [] - } - } - }, - { - "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers", - "apiVersion": "2022-08-15", - "name": "[format('{0}/{1}/{2}', parameters('databaseAccountName'), parameters('sqlDatabaseName'), parameters('name'))]", - "tags": "[parameters('tags')]", - "properties": { - "resource": { - "id": "[parameters('name')]", - "partitionKey": { - "paths": "[parameters('paths')]", - "kind": "[parameters('kind')]" - }, - "uniqueKeyPolicy": "[if(empty(parameters('uniqueKeyPaths')), null(), createObject('uniqueKeys', createArray(createObject('paths', parameters('uniqueKeyPaths')))))]" - }, - "options": "[if(or(contains(reference(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), '2022-08-15').capabilities, createObject('name', 'EnableServerless')), equals(parameters('throughput'), -1)), null(), createObject('throughput', parameters('throughput')))]" - } - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the container." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the container." - }, - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers', parameters('databaseAccountName'), parameters('sqlDatabaseName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the container was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('name'))]" - ] - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the SQL database." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the SQL database." - }, - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the SQL database was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "copy": { - "name": "databaseAccount_gremlinDatabases", - "count": "[length(parameters('gremlinDatabases'))]" - }, - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-{1}', deployment().name, parameters('gremlinDatabases')[copyIndex()].name)]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "databaseAccountName": { - "value": "[if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))]" - }, - "name": { - "value": "[parameters('gremlinDatabases')[copyIndex()].name]" - }, - "throughput": { - "value": "[parameters('throughput')]" - }, - "maxThroughput": { - "value": "[parameters('maxThroughput')]" - }, - "graphs": "[if(contains(parameters('gremlinDatabases')[copyIndex()], 'graphs'), createObject('value', parameters('gremlinDatabases')[copyIndex()].graphs), createObject('value', createArray()))]", - "enableDefaultTelemetry": { - "value": "[variables('enableReferencedModulesTelemetry')]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "17388289659429914951" - } - }, - "parameters": { - "name": { - "type": "string", - "metadata": { - "description": "Required. Name of the Gremlin database." - } - }, - "tags": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Optional. Tags of the Gremlin database resource." - } - }, - "databaseAccountName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent Gremlin database. Required if the template is used in a standalone deployment." - } - }, - "graphs": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Array of graphs to deploy in the Gremlin database." - } - }, - "maxThroughput": { - "type": "int", - "defaultValue": 400, - "metadata": { - "description": "Optional. Represents maximum throughput, the resource can scale up to. Cannot be set together with `throughput`. If `throughput` is set to something else than -1, this autoscale setting is ignored." - } - }, - "throughput": { - "type": "int", - "defaultValue": -1, - "metadata": { - "description": "Optional. Request Units per second (for example 10000). Cannot be set together with `maxThroughput`." - } - }, - "enableDefaultTelemetry": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Optional. Enable telemetry via the Customer Usage Attribution ID (GUID)." - } - } - }, - "variables": { - "enableReferencedModulesTelemetry": false - }, - "resources": [ - { - "condition": "[parameters('enableDefaultTelemetry')]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2021-04-01", - "name": "[format('pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-{0}', uniqueString(deployment().name))]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [] - } - } - }, - { - "type": "Microsoft.DocumentDB/databaseAccounts/gremlinDatabases", - "apiVersion": "2022-08-15", - "name": "[format('{0}/{1}', parameters('databaseAccountName'), parameters('name'))]", - "tags": "[parameters('tags')]", - "properties": { - "options": "[if(contains(reference(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), '2022-08-15').capabilities, createObject('name', 'EnableServerless')), createObject(), createObject('autoscaleSettings', if(equals(parameters('throughput'), -1), createObject('maxThroughput', parameters('maxThroughput')), null()), 'throughput', if(not(equals(parameters('throughput'), -1)), parameters('throughput'), null())))]", - "resource": { - "id": "[parameters('name')]" - } - } - }, - { - "copy": { - "name": "gremlinDatabase_gremlinGraphs", - "count": "[length(parameters('graphs'))]" - }, - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-{1}-graph-{2}', deployment().name, parameters('name'), parameters('graphs')[copyIndex()].name)]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "name": { - "value": "[parameters('graphs')[copyIndex()].name]" - }, - "gremlinDatabaseName": { - "value": "[parameters('name')]" - }, - "databaseAccountName": { - "value": "[parameters('databaseAccountName')]" - }, - "enableDefaultTelemetry": { - "value": "[variables('enableReferencedModulesTelemetry')]" - }, - "automaticIndexing": "[if(contains(parameters('graphs')[copyIndex()], 'automaticIndexing'), createObject('value', parameters('graphs')[copyIndex()].automaticIndexing), createObject('value', true()))]", - "partitionKeyPaths": "[if(not(empty(parameters('graphs')[copyIndex()].partitionKeyPaths)), createObject('value', parameters('graphs')[copyIndex()].partitionKeyPaths), createObject('value', createArray()))]" - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "933017375529243120" - } - }, - "parameters": { - "name": { - "type": "string", - "metadata": { - "description": "Required. Name of the graph." - } - }, - "tags": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Optional. Tags of the Gremlin graph resource." - } - }, - "enableDefaultTelemetry": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Optional. Enable telemetry via the Customer Usage Attribution ID (GUID)." - } - }, - "databaseAccountName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent Database Account. Required if the template is used in a standalone deployment." - } - }, - "gremlinDatabaseName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent Gremlin Database. Required if the template is used in a standalone deployment." - } - }, - "automaticIndexing": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Optional. Indicates if the indexing policy is automatic." - } - }, - "partitionKeyPaths": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. List of paths using which data within the container can be partitioned." - } - } - }, - "resources": [ - { - "condition": "[parameters('enableDefaultTelemetry')]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2021-04-01", - "name": "[format('pid-47ed15a6-730a-4827-bcb4-0fd963ffbd82-{0}', uniqueString(deployment().name))]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "resources": [] - } - } - }, - { - "type": "Microsoft.DocumentDB/databaseAccounts/gremlinDatabases/graphs", - "apiVersion": "2022-08-15", - "name": "[format('{0}/{1}/{2}', parameters('databaseAccountName'), parameters('gremlinDatabaseName'), parameters('name'))]", - "tags": "[parameters('tags')]", - "properties": { - "resource": { - "id": "[parameters('name')]", - "indexingPolicy": { - "automatic": "[parameters('automaticIndexing')]" - }, - "partitionKey": { - "paths": "[if(not(empty(parameters('partitionKeyPaths'))), parameters('partitionKeyPaths'), null())]" - } - } - } - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the graph." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the graph." - }, - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts/gremlinDatabases/graphs', parameters('databaseAccountName'), parameters('gremlinDatabaseName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the graph was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts/gremlinDatabases', parameters('databaseAccountName'), parameters('name'))]" - ] - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the Gremlin database." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the Gremlin database." - }, - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts/gremlinDatabases', parameters('databaseAccountName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the Gremlin database was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "copy": { - "name": "databaseaccount_rbac", - "count": "[length(parameters('roleAssignments'))]" - }, - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-rbac-{1}', deployment().name, copyIndex())]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "description": "[if(contains(parameters('roleAssignments')[copyIndex()], 'description'), createObject('value', parameters('roleAssignments')[copyIndex()].description), createObject('value', ''))]", - "principals": { - "value": "[parameters('roleAssignments')[copyIndex()].principals]" - }, - "roleDefinitionIdOrName": { - "value": "[parameters('roleAssignments')[copyIndex()].roleDefinitionIdOrName]" - }, - "principalType": "[if(contains(parameters('roleAssignments')[copyIndex()], 'principalType'), createObject('value', parameters('roleAssignments')[copyIndex()].principalType), createObject('value', ''))]", - "resourceId": { - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - }, - "crossTenant": { - "value": "[parameters('crossTenant')]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "1838371839130646629" - } - }, - "parameters": { - "description": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The description of the role assignment." - } - }, - "principals": { - "type": "array", - "metadata": { - "description": "Required. The IDs of the principals to assign the role to. A resourceId is required when used in a cross tenant scenario (i.e. crossTenant is true)" - } - }, - "principalType": { - "type": "string", - "defaultValue": "", - "allowedValues": [ - "ServicePrincipal", - "Group", - "User", - "ForeignGroup", - "Device", - "" - ], - "metadata": { - "description": "Optional. The principal type of the assigned principal ID." - } - }, - "roleDefinitionIdOrName": { - "type": "string", - "metadata": { - "description": "Required. The name of the role to assign. If it cannot be found you can specify the role definition ID instead." - } - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "Required. The resource ID of the resource to apply the role assignment to." - } - }, - "condition": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase \"foo_storage_container\"" - } - }, - "conditionVersion": { - "type": "string", - "defaultValue": "2.0", - "allowedValues": [ - "2.0" - ], - "metadata": { - "description": "Optional. Version of the condition." - } - }, - "crossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment's principal object." - } - } - }, - "variables": { - "builtInRoleNames": { - "Owner": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", - "Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", - "Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]", - "Cosmos DB Account Reader Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'fbdf93bf-df7d-467e-a4d2-9458aa1360c8')]", - "Cosmos DB Operator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '230815da-be43-4aae-9cb4-875f7bd000aa')]", - "CosmosBackupOperator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'db7b14f2-5adf-42da-9f96-f2ee17bab5cb')]", - "DocumentDB Account Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5bd9cd88-fe45-4216-938b-f97437e15450')]", - "Log Analytics Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '92aaf0da-9dab-42b6-94a3-d43ce8d16293')]", - "Log Analytics Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '73c42c96-874c-492b-b04d-ab87d138a893')]", - "Managed Application Contributor Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '641177b8-a67a-45b9-a033-47bc880bb21e')]", - "Managed Application Operator Role": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c7393b34-138c-406f-901b-d8cf2b17e6ae')]", - "Managed Applications Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b9331d33-8a36-4f8c-b097-4f54124fdb44')]", - "Monitoring Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '749f88d5-cbae-40b8-bcfc-e573ddc772fa')]", - "Monitoring Metrics Publisher": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb')]", - "Monitoring Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '43d0d8ad-25c7-4714-9337-8ba259a9fe05')]", - "Resource Policy Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '36243c78-bf99-498c-9df9-86d9f8d28608')]", - "User Access Administrator": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9')]" - } - }, - "resources": [ - { - "copy": { - "name": "roleAssignment", - "count": "[length(parameters('principals'))]" - }, - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "scope": "[format('Microsoft.DocumentDB/databaseAccounts/{0}', last(split(parameters('resourceId'), '/')))]", - "name": "[guid(resourceId('Microsoft.DocumentDB/databaseAccounts', last(split(parameters('resourceId'), '/'))), parameters('principals')[copyIndex()].id, parameters('roleDefinitionIdOrName'))]", - "properties": { - "description": "[parameters('description')]", - "roleDefinitionId": "[if(contains(variables('builtInRoleNames'), parameters('roleDefinitionIdOrName')), variables('builtInRoleNames')[parameters('roleDefinitionIdOrName')], parameters('roleDefinitionIdOrName'))]", - "principalId": "[parameters('principals')[copyIndex()].id]", - "principalType": "[if(not(empty(parameters('principalType'))), parameters('principalType'), null())]", - "condition": "[if(not(empty(parameters('condition'))), parameters('condition'), null())]", - "conditionVersion": "[if(and(not(empty(parameters('conditionVersion'))), not(empty(parameters('condition')))), parameters('conditionVersion'), null())]", - "delegatedManagedIdentityResourceId": "[if(parameters('crossTenant'), parameters('principals')[copyIndex()].resourceId, null())]" - } - } - ] - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "condition": "[and(not(empty(parameters('keyVaultName'))), not(empty(parameters('databaseEndpointSecretName'))))]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-secret-name', deployment().name)]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "keyVaultName": { - "value": "[parameters('keyVaultName')]" - }, - "name": { - "value": "[parameters('databaseEndpointSecretName')]" - }, - "value": { - "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))), '2022-08-15').documentEndpoint]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "13234384176658043502" - } - }, - "parameters": { - "keyVaultName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent key vault. Required if the template is used in a standalone deployment." - } - }, - "name": { - "type": "string", - "metadata": { - "description": "Required. The name of the secret." - } - }, - "value": { - "type": "securestring", - "metadata": { - "description": "Required. The value of the secret. NOTE: \"value\" will never be returned from the service, as APIs using this model are is intended for internal use in ARM deployments. Users should use the data-plane REST service for interaction with vault secrets." - } - } - }, - "resources": [ - { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2022-07-01", - "name": "[format('{0}/{1}', parameters('keyVaultName'), parameters('name'))]", - "properties": { - "value": "[parameters('value')]" - } - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the secret." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the secret." - }, - "value": "[resourceId('Microsoft.KeyVault/vaults/secrets', parameters('keyVaultName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the secret was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "condition": "[and(not(empty(parameters('keyVaultName'))), not(empty(parameters('databasePrimaryKeySecretName'))))]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-secret-key', deployment().name)]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "keyVaultName": { - "value": "[parameters('keyVaultName')]" - }, - "name": { - "value": "[parameters('databasePrimaryKeySecretName')]" - }, - "value": { - "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))), '2022-08-15').primaryMasterKey]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "13234384176658043502" - } - }, - "parameters": { - "keyVaultName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent key vault. Required if the template is used in a standalone deployment." - } - }, - "name": { - "type": "string", - "metadata": { - "description": "Required. The name of the secret." - } - }, - "value": { - "type": "securestring", - "metadata": { - "description": "Required. The value of the secret. NOTE: \"value\" will never be returned from the service, as APIs using this model are is intended for internal use in ARM deployments. Users should use the data-plane REST service for interaction with vault secrets." - } - } - }, - "resources": [ - { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2022-07-01", - "name": "[format('{0}/{1}', parameters('keyVaultName'), parameters('name'))]", - "properties": { - "value": "[parameters('value')]" - } - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the secret." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the secret." - }, - "value": "[resourceId('Microsoft.KeyVault/vaults/secrets', parameters('keyVaultName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the secret was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - }, - { - "condition": "[and(not(empty(parameters('keyVaultName'))), not(empty(parameters('databaseConnectionStringSecretName'))))]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2022-09-01", - "name": "[format('{0}-secret-accountName', deployment().name)]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "keyVaultName": { - "value": "[parameters('keyVaultName')]" - }, - "name": { - "value": "[parameters('databaseConnectionStringSecretName')]" - }, - "value": { - "value": "[listConnectionStrings(resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))), '2022-08-15').connectionStrings[0].connectionString]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.16.2.56959", - "templateHash": "13234384176658043502" - } - }, - "parameters": { - "keyVaultName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent key vault. Required if the template is used in a standalone deployment." - } - }, - "name": { - "type": "string", - "metadata": { - "description": "Required. The name of the secret." - } - }, - "value": { - "type": "securestring", - "metadata": { - "description": "Required. The value of the secret. NOTE: \"value\" will never be returned from the service, as APIs using this model are is intended for internal use in ARM deployments. Users should use the data-plane REST service for interaction with vault secrets." - } - } - }, - "resources": [ - { - "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2022-07-01", - "name": "[format('{0}/{1}', parameters('keyVaultName'), parameters('name'))]", - "properties": { - "value": "[parameters('value')]" - } - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the secret." - }, - "value": "[parameters('name')]" - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "The resource ID of the secret." - }, - "value": "[resourceId('Microsoft.KeyVault/vaults/secrets', parameters('keyVaultName'), parameters('name'))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the secret was created in." - }, - "value": "[resourceGroup().name]" - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - ] - } - ], - "outputs": { - "name": { - "type": "string", - "metadata": { - "description": "The name of the database account." - }, - "value": "[if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))]" - }, - "id": { - "type": "string", - "metadata": { - "description": "The resource ID of the database account." - }, - "value": "[resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name')))]" - }, - "resourceGroupName": { - "type": "string", - "metadata": { - "description": "The name of the resource group the database account was created in." - }, - "value": "[resourceGroup().name]" - }, - "systemAssignedPrincipalId": { - "type": "string", - "metadata": { - "description": "The principal ID of the system assigned identity." - }, - "value": "[if(and(parameters('systemAssignedIdentity'), contains(reference(resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))), '2022-08-15', 'full').identity, 'principalId')), reference(resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))), '2022-08-15', 'full').identity.principalId, '')]" - }, - "location": { - "type": "string", - "metadata": { - "description": "The location the resource was deployed into." - }, - "value": "[reference(resourceId('Microsoft.DocumentDB/databaseAccounts', if(greater(length(variables('name')), 44), substring(variables('name'), 0, 44), variables('name'))), '2022-08-15', 'full').location]" - } - } -} \ No newline at end of file diff --git a/bicep/modules/private-endpoint/main.json b/bicep/modules/private-endpoint/main.json deleted file mode 100644 index 141b396a..00000000 --- a/bicep/modules/private-endpoint/main.json +++ /dev/null @@ -1,438 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.13.1.58284", - "templateHash": "13918084530796890990" - } - }, - "parameters": { - "resourceName": { - "type": "string", - "metadata": { - "description": "Required. Name of the private endpoint resource to create." - } - }, - "subnetResourceId": { - "type": "string", - "metadata": { - "description": "Required. Resource ID of the subnet where the endpoint needs to be created." - } - }, - "serviceResourceId": { - "type": "string", - "metadata": { - "description": "Required. Resource ID of the resource that needs to be connected to the network." - } - }, - "groupIds": { - "type": "array", - "metadata": { - "description": "Required. Subtype(s) of the connection to be created. The allowed values depend on the type serviceResourceId refers to." - } - }, - "applicationSecurityGroups": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Application security groups in which the private endpoint IP configuration is included." - } - }, - "customNetworkInterfaceName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The custom name of the network interface attached to the private endpoint." - } - }, - "ipConfigurations": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. A list of IP configurations of the private endpoint. This will be used to map to the First Party Service endpoints." - } - }, - "privateDnsZoneGroup": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Optional. The private DNS zone group configuration used to associate the private endpoint with one or multiple private DNS zones. A DNS zone group can support up to 5 DNS zones." - } - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "Optional. Location for all Resources." - } - }, - "crossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment's principal object." - } - }, - "roleAssignments": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Array of objects that describe RBAC permissions, format { roleDefinitionResourceId (string), principalId (string), principalType (enum), enabled (bool) }. Ref: https://docs.microsoft.com/en-us/azure/templates/microsoft.authorization/roleassignments?tabs=bicep" - } - }, - "tags": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Tags." - } - }, - "lock": { - "type": "string", - "defaultValue": "NotSpecified", - "metadata": { - "description": "Optional. Specify the type of lock." - }, - "allowedValues": [ - "CanNotDelete", - "NotSpecified", - "ReadOnly" - ] - }, - "customDnsConfigs": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Custom DNS configurations." - } - }, - "manualPrivateLinkServiceConnections": { - "type": "array", - "defaultValue": [], - "metadata": { - "description": "Optional. Manual PrivateLink Service Connections." - } - } - }, - "variables": { - "name": "[format('endpoint-{0}{1}', replace(parameters('resourceName'), '-', ''), uniqueString(resourceGroup().id, parameters('resourceName')))]" - }, - "resources": [ - { - "type": "Microsoft.Network/privateEndpoints", - "apiVersion": "2022-05-01", - "name": "[if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name'))]", - "location": "[parameters('location')]", - "tags": "[parameters('tags')]", - "properties": { - "applicationSecurityGroups": "[parameters('applicationSecurityGroups')]", - "customNetworkInterfaceName": "[parameters('customNetworkInterfaceName')]", - "ipConfigurations": "[parameters('ipConfigurations')]", - "manualPrivateLinkServiceConnections": "[parameters('manualPrivateLinkServiceConnections')]", - "customDnsConfigs": "[parameters('customDnsConfigs')]", - "privateLinkServiceConnections": [ - { - "name": "[parameters('resourceName')]", - "properties": { - "privateLinkServiceId": "[parameters('serviceResourceId')]", - "groupIds": "[parameters('groupIds')]" - } - } - ], - "subnet": { - "id": "[parameters('subnetResourceId')]" - } - } - }, - { - "condition": "[not(equals(parameters('lock'), 'NotSpecified'))]", - "type": "Microsoft.Authorization/locks", - "apiVersion": "2017-04-01", - "scope": "[format('Microsoft.Network/privateEndpoints/{0}', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]", - "name": "[format('{0}-{1}-lock', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')), parameters('lock'))]", - "properties": { - "level": "[parameters('lock')]", - "notes": "[if(equals(parameters('lock'), 'CanNotDelete'), 'Cannot delete resource or child resources.', 'Cannot modify the resource or child resources.')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]" - ] - }, - { - "condition": "[not(empty(parameters('privateDnsZoneGroup')))]", - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-10-01", - "name": "[format('{0}-{1}', deployment().name, if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "privateDNSResourceIds": { - "value": "[parameters('privateDnsZoneGroup').privateDNSResourceIds]" - }, - "privateEndpointName": { - "value": "[if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name'))]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.13.1.58284", - "templateHash": "10948839627963430016" - } - }, - "parameters": { - "privateEndpointName": { - "type": "string", - "metadata": { - "description": "Conditional. The name of the parent private endpoint. Required if the template is used in a standalone deployment." - } - }, - "privateDNSResourceIds": { - "type": "array", - "maxLength": 5, - "minLength": 1, - "metadata": { - "description": "Required. Array of private DNS zone resource IDs. A DNS zone group can support up to 5 DNS zones." - } - }, - "resourceName": { - "type": "string", - "defaultValue": "default", - "metadata": { - "description": "Optional. The name of the private DNS zone group." - } - } - }, - "variables": { - "copy": [ - { - "name": "privateDnsZoneConfigs", - "count": "[length(parameters('privateDNSResourceIds'))]", - "input": { - "name": "[last(split(parameters('privateDNSResourceIds')[copyIndex('privateDnsZoneConfigs')], '/'))]", - "properties": { - "privateDnsZoneId": "[parameters('privateDNSResourceIds')[copyIndex('privateDnsZoneConfigs')]]" - } - } - } - ] - }, - "resources": [ - { - "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", - "apiVersion": "2022-05-01", - "name": "[format('{0}/{1}', parameters('privateEndpointName'), parameters('resourceName'))]", - "properties": { - "privateDnsZoneConfigs": "[variables('privateDnsZoneConfigs')]" - } - } - ], - "outputs": { - "resourceGroupName": { - "type": "string", - "value": "[resourceGroup().name]", - "metadata": { - "description": "The resource group the private endpoint DNS zone group was deployed into." - } - }, - "name": { - "type": "string", - "value": "[parameters('resourceName')]", - "metadata": { - "description": "The name of the private endpoint DNS zone group." - } - }, - "id": { - "type": "string", - "value": "[resourceId('Microsoft.Network/privateEndpoints/privateDnsZoneGroups', parameters('privateEndpointName'), parameters('resourceName'))]", - "metadata": { - "description": "The resource ID of the private endpoint DNS zone group." - } - } - } - } - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]" - ] - }, - { - "copy": { - "name": "privateEndpoint_roleAssignments", - "count": "[length(parameters('roleAssignments'))]" - }, - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-10-01", - "name": "[format('{0}-rbac-{1}', deployment().name, copyIndex())]", - "properties": { - "expressionEvaluationOptions": { - "scope": "inner" - }, - "mode": "Incremental", - "parameters": { - "description": "[if(contains(parameters('roleAssignments')[copyIndex()], 'description'), createObject('value', parameters('roleAssignments')[copyIndex()].description), createObject('value', ''))]", - "principals": { - "value": "[parameters('roleAssignments')[copyIndex()].principals]" - }, - "principalType": "[if(contains(parameters('roleAssignments')[copyIndex()], 'principalType'), createObject('value', parameters('roleAssignments')[copyIndex()].principalType), createObject('value', ''))]", - "roleDefinitionIdOrName": { - "value": "[parameters('roleAssignments')[copyIndex()].roleDefinitionIdOrName]" - }, - "condition": "[if(contains(parameters('roleAssignments')[copyIndex()], 'condition'), createObject('value', parameters('roleAssignments')[copyIndex()].condition), createObject('value', ''))]", - "resourceId": { - "value": "[resourceId('Microsoft.Network/privateEndpoints', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]" - }, - "crossTenant": { - "value": "[parameters('crossTenant')]" - } - }, - "template": { - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.13.1.58284", - "templateHash": "11434261217921661174" - } - }, - "parameters": { - "roleDefinitionIdOrName": { - "type": "string", - "metadata": { - "description": "Required. The name of the role to assign. If it cannot be found you can specify the role definition ID instead." - } - }, - "resourceId": { - "type": "string", - "metadata": { - "description": "Required. The resource ID of the resource to apply the role assignment to." - } - }, - "principalType": { - "type": "string", - "defaultValue": "", - "allowedValues": [ - "ServicePrincipal", - "Group", - "User", - "ForeignGroup", - "Device", - "" - ], - "metadata": { - "description": "Optional. The principal type of the assigned principal ID." - } - }, - "description": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The description of the role assignment." - } - }, - "condition": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase \"foo_storage_container\"" - } - }, - "conditionVersion": { - "type": "string", - "defaultValue": "2.0", - "allowedValues": [ - "2.0" - ], - "metadata": { - "description": "Optional. Version of the condition." - } - }, - "crossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Indicates if the module is used in a cross tenant scenario. If true, a resourceId must be provided in the role assignment's principal object." - } - }, - "principals": { - "type": "array", - "metadata": { - "description": "Required. The IDs of the principals to assign the role to. A resourceId is required when used in a cross tenant scenario (i.e. crossTenant is true)" - } - } - }, - "variables": { - "builtInRoleNames": { - "Owner": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635')]", - "Contributor": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", - "Reader": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]" - } - }, - "resources": [ - { - "copy": { - "name": "roleAssignment", - "count": "[length(parameters('principals'))]" - }, - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "scope": "[format('Microsoft.Network/privateEndpoints/{0}', last(split(parameters('resourceId'), '/')))]", - "name": "[guid(resourceId('Microsoft.Network/privateEndpoints', last(split(parameters('resourceId'), '/'))), parameters('principals')[copyIndex()].id, parameters('roleDefinitionIdOrName'))]", - "properties": { - "description": "[parameters('description')]", - "roleDefinitionId": "[if(contains(variables('builtInRoleNames'), parameters('roleDefinitionIdOrName')), variables('builtInRoleNames')[parameters('roleDefinitionIdOrName')], parameters('roleDefinitionIdOrName'))]", - "principalId": "[parameters('principals')[copyIndex()].id]", - "principalType": "[if(not(empty(parameters('principalType'))), parameters('principalType'), null())]", - "condition": "[if(not(empty(parameters('condition'))), parameters('condition'), null())]", - "conditionVersion": "[if(and(not(empty(parameters('conditionVersion'))), not(empty(parameters('condition')))), parameters('conditionVersion'), null())]", - "delegatedManagedIdentityResourceId": "[if(parameters('crossTenant'), parameters('principals')[copyIndex()].resourceId, null())]" - } - } - ] - } - }, - "dependsOn": [ - "[resourceId('Microsoft.Network/privateEndpoints', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]" - ] - } - ], - "outputs": { - "resourceGroupName": { - "type": "string", - "value": "[resourceGroup().name]", - "metadata": { - "description": "The resource group the private endpoint was deployed into." - } - }, - "name": { - "type": "string", - "value": "[if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name'))]", - "metadata": { - "description": "The name of the private endpoint." - } - }, - "id": { - "type": "string", - "value": "[resourceId('Microsoft.Network/privateEndpoints', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name')))]", - "metadata": { - "description": "The resource ID of the private endpoint." - } - }, - "location": { - "type": "string", - "value": "[reference(resourceId('Microsoft.Network/privateEndpoints', if(greater(length(variables('name')), 64), substring(variables('name'), 0, 64), variables('name'))), '2022-05-01', 'full').location]", - "metadata": { - "description": "The location the resource was deployed into." - } - } - } -} \ No newline at end of file diff --git a/bicep/modules/script-blob-upload/main.bicep b/bicep/modules/script-blob-upload/main.bicep index 7d0bc377..ea7d2507 100644 --- a/bicep/modules/script-blob-upload/main.bicep +++ b/bicep/modules/script-blob-upload/main.bicep @@ -64,7 +64,7 @@ resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty( } } -resource uploadFile 'Microsoft.Resources/deploymentScripts@2020-10-01' = { +resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = { name: 'script-${storageAccount.name}-${replace(replace(filename, ':', ''), '/', '-')}' location: location identity: { @@ -75,8 +75,8 @@ resource uploadFile 'Microsoft.Resources/deploymentScripts@2020-10-01' = { dependsOn: [ rbac ] properties: { forceUpdateTag: forceUpdateTag - azCliVersion: '2.61.0' - timeout: 'PT15M' + azCliVersion: '2.63.0' + timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } diff --git a/bicep/modules/script-kv-certificate/main.bicep b/bicep/modules/script-kv-certificate/main.bicep index 03a7bed6..c9b97a89 100644 --- a/bicep/modules/script-kv-certificate/main.bicep +++ b/bicep/modules/script-kv-certificate/main.bicep @@ -105,7 +105,7 @@ resource rbacKv 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (perfo } } -resource createImportCerts 'Microsoft.Resources/deploymentScripts@2020-10-01' = [for (certificateName, index) in certificateNames: { +resource createImportCerts 'Microsoft.Resources/deploymentScripts@2023-08-01' = [for (certificateName, index) in certificateNames: { name: 'script-${akv.name}-${replace(replace(certificateName, ':', ''), '/', '-')}' location: location identity: { @@ -120,8 +120,8 @@ resource createImportCerts 'Microsoft.Resources/deploymentScripts@2020-10-01' = ] properties: { forceUpdateTag: forceUpdateTag - azCliVersion: '2.61.0' - timeout: 'PT15M' + azCliVersion: '2.63.0' + timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ { name: 'akvName', value: kvName } diff --git a/bicep/modules/script-kv-certificate/main.json b/bicep/modules/script-kv-certificate/main.json deleted file mode 100644 index 9e89dc20..00000000 --- a/bicep/modules/script-kv-certificate/main.json +++ /dev/null @@ -1,345 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.23.1.45101", - "templateHash": "11944081156064634174" - }, - "name": "Key Vault Certificate", - "description": "This module creates a Key Vault Certificate and stores it in an Azure Key Vault", - "owner": "azure-global-energy" - }, - "parameters": { - "kvName": { - "type": "string", - "metadata": { - "description": "The name of the Azure Key Vault" - } - }, - "location": { - "type": "string", - "metadata": { - "description": "The location to deploy the resources to" - } - }, - "forceUpdateTag": { - "type": "string", - "defaultValue": "[utcNow()]", - "metadata": { - "description": "How the deployment script should be forced to execute" - } - }, - "rbacRolesNeededOnKV": { - "type": "string", - "defaultValue": "a4417e6f-fecd-4de8-b567-7b0420556985", - "metadata": { - "description": "The RoleDefinitionId required for the DeploymentScript resource to interact with KeyVault" - } - }, - "useExistingManagedIdentity": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Does the Managed Identity already exists, or should be created" - } - }, - "managedIdentityName": { - "type": "string", - "defaultValue": "[format('id-KeyVaultCertificateCreator-{0}', parameters('location'))]", - "metadata": { - "description": "Name of the Managed Identity resource" - } - }, - "existingManagedIdentitySubId": { - "type": "string", - "defaultValue": "[subscription().subscriptionId]", - "metadata": { - "description": "For an existing Managed Identity, the Subscription Id it is located in" - } - }, - "existingManagedIdentityResourceGroupName": { - "type": "string", - "defaultValue": "[resourceGroup().name]", - "metadata": { - "description": "For an existing Managed Identity, the Resource Group it is located in" - } - }, - "certificateNames": { - "type": "array", - "metadata": { - "description": "The names of the certificate to create. Use when creating many certificates." - } - }, - "certificateCommonNames": { - "type": "array", - "defaultValue": "[parameters('certificateNames')]", - "metadata": { - "description": "The common names of the certificate to create. Use when creating many certificates." - } - }, - "initialScriptDelay": { - "type": "string", - "defaultValue": "0", - "metadata": { - "description": "A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate" - } - }, - "cleanupPreference": { - "type": "string", - "defaultValue": "OnSuccess", - "allowedValues": [ - "OnSuccess", - "OnExpiration", - "Always" - ], - "metadata": { - "description": "When the script resource is cleaned up" - } - }, - "issuerName": { - "type": "string", - "defaultValue": "Self", - "metadata": { - "description": "Self, or user defined {IssuerName} for certificate signing" - } - }, - "issuerProvider": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Certificate Issuer Provider, DigiCert, GlobalSign, or internal options may be used." - } - }, - "disabled": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Create certificate in disabled state. Default: false" - } - }, - "accountId": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Account ID of Certificate Issuer Account" - } - }, - "issuerPassword": { - "type": "securestring", - "defaultValue": "", - "metadata": { - "description": "Password of Certificate Issuer Account" - } - }, - "organizationId": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Organization ID of Certificate Issuer Account" - } - }, - "isCrossTenant": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Override this parameter if using this in cross tenant scenarios" - } - }, - "reuseKey": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "The default policy might cause errors about CSR being used before, so set this to false if that happens" - } - }, - "validity": { - "type": "int", - "defaultValue": 12, - "minValue": 1, - "maxValue": 1200, - "metadata": { - "description": "Optional. Override default validityInMonths 12 value" - } - }, - "performRoleAssignment": { - "type": "bool", - "defaultValue": true, - "metadata": { - "description": "Set to false to disable role assignments within this module. Default: true" - } - } - }, - "variables": { - "$fxv#0": "#!/bin/bash\nset -e\ninitialDelay=\"${initialDelay:-5}\"\nretryMax=\"${retryMax:-5}\"\ncertName=\"${certName:-default-cert}\"\ncertCommonName=\"${certCommonName:-default}\"\nvalidity=\"${validity:-12}\"\nakvName=\"${akvName:-keyvault}\"\nissuerName=\"${issuerName:-}\"\nreuseKey=\"${reuseKey:-true}\"\nretrySleep=\"${retrySleep:-5}\"\n\necho \"Waiting on Identity RBAC replication (\\\"$initialDelay\\\")\"\nsleep \"$initialDelay\"\n\n#Retry loop to catch errors (usually RBAC delays)\nretryLoopCount=0\nuntil [ \"$retryLoopCount\" -ge \"$retryMax\" ]\ndo\n echo \"Creating AKV Cert $certName with CN $certCommonName (attempt $retryLoopCount)...\"\n\n if [ -z \"$issuerName\" ] || [ -z \"$issuerProvider\" ]; then\n policy=$(az keyvault certificate get-default-policy \\\n | sed -e s/\\\"validityInMonths\\\":\\ 12/\\\"validityInMonths\\\":\\ \"${validity}\"/g \\\n | sed -e s/CN=CLIGetDefaultPolicy/CN=\"${certCommonName}\"/g )\n else\n if [ \"$issuerProvider\" == \"DigiCert\" ] || [ \"$issuerProvider\" == \"GlobalCert\" ]; then\n az keyvault certificate issuer create \\\n --vault-name \"$akvName\" \\\n --issuer-name \"$issuerName\" \\\n --provider-name \"$issuerProvider\" \\\n --account-id \"$accountId\" \\\n --password \"$issuerPassword\" \\\n --organizatiion-id \"$organizationId\"\n else\n az keyvault certificate issuer create \\\n --vault-name \"$akvName\" \\\n --issuer-name \"$issuerName\" \\\n --provider-name \"$issuerProvider\"\n fi\n policy=$(az keyvault certificate get-default-policy \\\n | sed -e s/\\\"validityInMonths\\\":\\ 12/\\\"validityInMonths\\\":\\ \"${validity}\"/g \\\n | sed -e s/CN=CLIGetDefaultPolicy/CN=\"${certCommonName}\"/g \\\n | sed -e s/\\\"name\\\":\\ \\\"Self\\\"/\\\"name\\\":\\ \\\"\"${issuerName}\"\\\"/g \\\n | sed -e s/\\\"reuseKey\\\":\\ true/\\\"reuseKey\\\":\\ \"${reuseKey}\"/g )\n fi\n az keyvault certificate create \\\n --vault-name \"$akvName\" \\\n -n \"$certName\" \\\n -p \"$policy\" \\\n --disabled \"$disabled\" \\\n && break\n\n sleep \"$retrySleep\"\n retryLoopCount=$((retryLoopCount+1))\ndone\n\necho \"Getting Certificate $certName\";\nretryLoopCount=0\ncreatedCert=$(az keyvault certificate show -n \"$certName\" --vault-name \"$akvName\" -o json)\nwhile [ -z \"$(echo \"$createdCert\" | jq -r '.x509ThumbprintHex')\" ] && [ $retryLoopCount -lt \"$retryMax\" ]\ndo\n echo \"Waiting for cert creation (attempt $retryLoopCount)...\"\n sleep $retrySleep\n createdCert=$(az keyvault certificate show -n $certName --vault-name $akvName -o json)\n retryLoopCount=$((retryLoopCount+1))\ndone\n\nunversionedSecretId=$(echo $createdCert | jq -r \".sid\" | cut -d'/' -f-5) # remove the version from the url;\njsonOutputString=$(echo $createdCert | jq --arg usid $unversionedSecretId '{name: .name ,certSecretId: {versioned: .sid, unversioned: $usid }, thumbprint: .x509Thumbprint, thumbprintHex: .x509ThumbprintHex}')\necho $jsonOutputString > $AZ_SCRIPTS_OUTPUT_PATH", - "delegatedManagedIdentityResourceId": "[if(parameters('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')))]" - }, - "resources": [ - { - "condition": "[not(parameters('useExistingManagedIdentity'))]", - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2018-11-30", - "name": "[parameters('managedIdentityName')]", - "location": "[parameters('location')]", - "metadata": { - "description": "A new managed identity that will be created in this Resource Group, this is the default option" - } - }, - { - "condition": "[parameters('performRoleAssignment')]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "scope": "[format('Microsoft.KeyVault/vaults/{0}', parameters('kvName'))]", - "name": "[guid(resourceId('Microsoft.KeyVault/vaults', parameters('kvName')), parameters('rbacRolesNeededOnKV'), parameters('managedIdentityName'), string(parameters('useExistingManagedIdentity')))]", - "properties": { - "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', parameters('rbacRolesNeededOnKV'))]", - "principalId": "[if(parameters('useExistingManagedIdentity'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2018-11-30').principalId, reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2018-11-30').principalId)]", - "principalType": "ServicePrincipal", - "delegatedManagedIdentityResourceId": "[if(parameters('isCrossTenant'), variables('delegatedManagedIdentityResourceId'), null())]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]" - ] - }, - { - "copy": { - "name": "createImportCerts", - "count": "[length(parameters('certificateNames'))]" - }, - "type": "Microsoft.Resources/deploymentScripts", - "apiVersion": "2020-10-01", - "name": "[format('create-cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('certificateNames')[copyIndex()], ':', ''), '/', '-'))]", - "location": "[parameters('location')]", - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[format('{0}', if(parameters('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]": {} - } - }, - "kind": "AzureCLI", - "properties": { - "forceUpdateTag": "[parameters('forceUpdateTag')]", - "azCliVersion": "2.35.0", - "timeout": "PT10M", - "retentionInterval": "P1D", - "environmentVariables": [ - { - "name": "akvName", - "value": "[parameters('kvName')]" - }, - { - "name": "certName", - "value": "[parameters('certificateNames')[copyIndex()]]" - }, - { - "name": "certCommonName", - "value": "[parameters('certificateCommonNames')[copyIndex()]]" - }, - { - "name": "initialDelay", - "value": "[parameters('initialScriptDelay')]" - }, - { - "name": "issuerName", - "value": "[parameters('issuerName')]" - }, - { - "name": "issuerProvider", - "value": "[parameters('issuerProvider')]" - }, - { - "name": "disabled", - "value": "[toLower(string(parameters('disabled')))]" - }, - { - "name": "retryMax", - "value": "10" - }, - { - "name": "retrySleep", - "value": "5s" - }, - { - "name": "accountId", - "value": "[parameters('accountId')]" - }, - { - "name": "issuerPassword", - "secureValue": "[parameters('issuerPassword')]" - }, - { - "name": "organizationId", - "value": "[parameters('organizationId')]" - }, - { - "name": "reuseKey", - "value": "[toLower(string(parameters('reuseKey')))]" - }, - { - "name": "validity", - "value": "[string(parameters('validity'))]" - } - ], - "scriptContent": "[variables('$fxv#0')]", - "cleanupPreference": "[parameters('cleanupPreference')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]", - "[extensionResourceId(resourceId('Microsoft.KeyVault/vaults', parameters('kvName')), 'Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.KeyVault/vaults', parameters('kvName')), parameters('rbacRolesNeededOnKV'), parameters('managedIdentityName'), string(parameters('useExistingManagedIdentity'))))]" - ] - } - ], - "outputs": { - "certificateNames": { - "type": "array", - "metadata": { - "description": "Certificate names" - }, - "copy": { - "count": "[length(parameters('certificateNames'))]", - "input": "[createArray(reference(resourceId('Microsoft.Resources/deploymentScripts', format('create-cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('certificateNames')[copyIndex()], ':', ''), '/', '-'))), '2020-10-01').outputs.name)]" - } - }, - "certificateSecretIds": { - "type": "array", - "metadata": { - "description": "KeyVault secret ids to the created version" - }, - "copy": { - "count": "[length(parameters('certificateNames'))]", - "input": "[createArray(reference(resourceId('Microsoft.Resources/deploymentScripts', format('create-cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('certificateNames')[copyIndex()], ':', ''), '/', '-'))), '2020-10-01').outputs.certSecretId.versioned)]" - } - }, - "certificateSecretIdUnversioneds": { - "type": "array", - "metadata": { - "description": "KeyVault secret ids which uses the unversioned uri" - }, - "copy": { - "count": "[length(parameters('certificateNames'))]", - "input": "[createArray(reference(resourceId('Microsoft.Resources/deploymentScripts', format('create-cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('certificateNames')[copyIndex()], ':', ''), '/', '-'))), '2020-10-01').outputs.certSecretId.unversioned)]" - } - }, - "certificateThumbpints": { - "type": "array", - "metadata": { - "description": "Certificate Thumbprints" - }, - "copy": { - "count": "[length(parameters('certificateNames'))]", - "input": "[createArray(reference(resourceId('Microsoft.Resources/deploymentScripts', format('create-cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('certificateNames')[copyIndex()], ':', ''), '/', '-'))), '2020-10-01').outputs.thumbprint)]" - } - }, - "certificateThumbprintHexs": { - "type": "array", - "metadata": { - "description": "Certificate Thumbprints (in hex)" - }, - "copy": { - "count": "[length(parameters('certificateNames'))]", - "input": "[createArray(reference(resourceId('Microsoft.Resources/deploymentScripts', format('create-cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('certificateNames')[copyIndex()], ':', ''), '/', '-'))), '2020-10-01').outputs.thumbprintHex)]" - } - } - } -} \ No newline at end of file diff --git a/bicep/modules/script-share-upload/main.bicep b/bicep/modules/script-share-upload/main.bicep index cbcf462c..49346e3f 100644 --- a/bicep/modules/script-share-upload/main.bicep +++ b/bicep/modules/script-share-upload/main.bicep @@ -67,7 +67,7 @@ resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty( } } -resource uploadFile 'Microsoft.Resources/deploymentScripts@2020-10-01' = { +resource uploadFile 'Microsoft.Resources/deploymentScripts@2023-08-01' = { name: 'script-${storageAccount.name}-${replace(replace(filename, ':', ''), '/', '-')}' location: location identity: { @@ -78,8 +78,8 @@ resource uploadFile 'Microsoft.Resources/deploymentScripts@2020-10-01' = { dependsOn: [ rbac ] properties: { forceUpdateTag: forceUpdateTag - azCliVersion: '2.61.0' - timeout: 'PT15M' + azCliVersion: '2.63.0' + timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storageAccount.name } diff --git a/bicep/modules/script-sshkeypair/main.bicep b/bicep/modules/script-sshkeypair/main.bicep index 1714181f..78f4a041 100644 --- a/bicep/modules/script-sshkeypair/main.bicep +++ b/bicep/modules/script-sshkeypair/main.bicep @@ -66,7 +66,7 @@ resource rbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty( } } -resource createSshKeyPair 'Microsoft.Resources/deploymentScripts@2020-10-01' = { +resource createSshKeyPair 'Microsoft.Resources/deploymentScripts@2023-08-01' = { name: 'script-${akv.name}-${replace(replace(sshKeyName, ':', ''), '/', '-')}' location: location identity: { @@ -77,8 +77,8 @@ resource createSshKeyPair 'Microsoft.Resources/deploymentScripts@2020-10-01' = { dependsOn: [ rbac ] properties: { forceUpdateTag: forceUpdateTag - azCliVersion: '2.61.0' - timeout: 'PT15M' + azCliVersion: '2.63.0' + timeout: 'PT30M' retentionInterval: 'PT1H' environmentVariables: [ { name: 'keyVaultName', value: akv.name } diff --git a/bicep/modules/script-sshkeypair/main.json b/bicep/modules/script-sshkeypair/main.json deleted file mode 100644 index 7ef0c44a..00000000 --- a/bicep/modules/script-sshkeypair/main.json +++ /dev/null @@ -1,198 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "metadata": { - "_generator": { - "name": "bicep", - "version": "0.24.24.22086", - "templateHash": "14010778880958868816" - }, - "name": "SSH Key Pair", - "description": "This module creates a SSH Key Pair and stores it in an Azure Key Vault", - "owner": "azure-global-energy" - }, - "parameters": { - "kvName": { - "type": "string", - "metadata": { - "description": "The name of the Azure Key Vault" - } - }, - "location": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "The location of the Key Vault and where to deploy the module resources to" - } - }, - "forceUpdateTag": { - "type": "string", - "defaultValue": "[utcNow()]", - "metadata": { - "description": "How the deployment script should be forced to execute" - } - }, - "rbacRoleNeeded": { - "type": "string", - "defaultValue": "b86a8fe4-44ce-4948-aee5-eccb2c155cd7", - "metadata": { - "description": "Azure RoleId that are required for the DeploymentScript resource to import images" - } - }, - "useExistingManagedIdentity": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Does the Managed Identity already exists, or should be created" - } - }, - "managedIdentityName": { - "type": "string", - "defaultValue": "[format('id-keyvault-ssh-{0}', parameters('location'))]", - "metadata": { - "description": "Name of the Managed Identity resource" - } - }, - "existingManagedIdentitySubId": { - "type": "string", - "defaultValue": "[subscription().subscriptionId]", - "metadata": { - "description": "For an existing Managed Identity, the Subscription Id it is located in" - } - }, - "existingManagedIdentityResourceGroupName": { - "type": "string", - "defaultValue": "[resourceGroup().name]", - "metadata": { - "description": "For an existing Managed Identity, the Resource Group it is located in" - } - }, - "sshKeyName": { - "type": "string", - "metadata": { - "description": "The name of the SSH Key to be created.\nif name is my-virtual-machine-ssh then the private key will be named my-virtual-machine-sshprivate and the public key will be named my-virtual-machine-sshpublic.\n" - } - }, - "initialScriptDelay": { - "type": "string", - "defaultValue": "30s", - "metadata": { - "description": "A delay before the script import operation starts. Primarily to allow Azure AAD Role Assignments to propagate" - } - }, - "cleanupPreference": { - "type": "string", - "defaultValue": "OnSuccess", - "allowedValues": [ - "OnSuccess", - "OnExpiration", - "Always" - ], - "metadata": { - "description": "When the script resource is cleaned up" - } - } - }, - "variables": { - "$fxv#0": "#!/bin/bash\nset -e\n\necho \"Waiting on Identity RBAC replication ($initialDelay)\"\nsleep $initialDelay\n\n# Generate the SSH key pair\necho \"Generating SSH key pair...\"\n#ssh-keygen -t rsa -b 4096 -C \"azure@example.com\" -f id_rsa -N \"\"\nssh-keygen -m PEM -t rsa -b 4096 -f id_rsa -q\n\n# Import the private key and public key as strings\nprivateKey=$(cat id_rsa)\npublicKey=$(cat id_rsa.pub)\n\n# Re-Login to Azure CLI using the managed identity\necho \"Logging in to Azure CLI using managed identity...\"\naz login --identity\n\necho \"Storing secret ${sshKeyName}private in Key Vault $keyVaultName...\"\nprivSecret=$(az keyvault secret set --vault-name \"$keyVaultName\" --name \"${sshKeyNamePrivate}\" --value \"$privateKey\")\n\necho \"Storing secret ${sshKeyName}public in Key Vault $keyVaultName...\"\npubSecret=$(az keyvault secret set --vault-name \"$keyVaultName\" --name \"${sshKeyNamePublic}\" --value \"$publicKey\")\n\nprivateSecretId=$(echo $privSecret | jq -r \".id\" | cut -d'/' -f-5) # remove the version from the url;\npublicSecretId=$(echo $pubSecret | jq -r \".id\" | cut -d'/' -f-5) # remove the version from the url;\n\njsonOutputString=$(jq -cn --arg public $publicSecretId --arg private $privateSecretId '{secretUris: $ARGS.named}')\necho $jsonOutputString\necho $jsonOutputString > $AZ_SCRIPTS_OUTPUT_PATH\n\n# Cleanup\nrm -f id_rsa id_rsa.pub", - "privateKeySecretName": "[format('{0}private', parameters('sshKeyName'))]", - "publicKeySecretName": "[format('{0}public', parameters('sshKeyName'))]" - }, - "resources": [ - { - "condition": "[not(parameters('useExistingManagedIdentity'))]", - "type": "Microsoft.ManagedIdentity/userAssignedIdentities", - "apiVersion": "2023-01-31", - "name": "[parameters('managedIdentityName')]", - "location": "[parameters('location')]" - }, - { - "condition": "[not(empty(parameters('rbacRoleNeeded')))]", - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "2022-04-01", - "scope": "[format('Microsoft.KeyVault/vaults/{0}', parameters('kvName'))]", - "name": "[guid(resourceId('Microsoft.KeyVault/vaults', parameters('kvName')), parameters('rbacRoleNeeded'), if(parameters('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]", - "properties": { - "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', parameters('rbacRoleNeeded'))]", - "principalId": "[if(parameters('useExistingManagedIdentity'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2023-01-31').principalId, reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), '2023-01-31').principalId)]", - "principalType": "ServicePrincipal" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]" - ] - }, - { - "type": "Microsoft.Resources/deploymentScripts", - "apiVersion": "2020-10-01", - "name": "[format('AKV-Cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('sshKeyName'), ':', ''), '/', '-'))]", - "location": "[parameters('location')]", - "identity": { - "type": "UserAssigned", - "userAssignedIdentities": { - "[format('{0}', if(parameters('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))))]": {} - } - }, - "kind": "AzureCLI", - "properties": { - "forceUpdateTag": "[parameters('forceUpdateTag')]", - "azCliVersion": "2.45.0", - "timeout": "PT15M", - "retentionInterval": "PT1H", - "environmentVariables": [ - { - "name": "keyVaultName", - "value": "[parameters('kvName')]" - }, - { - "name": "sshKeyNamePrivate", - "value": "[variables('privateKeySecretName')]" - }, - { - "name": "sshKeyNamePublic", - "value": "[variables('publicKeySecretName')]" - }, - { - "name": "initialDelay", - "value": "[parameters('initialScriptDelay')]" - } - ], - "scriptContent": "[variables('$fxv#0')]", - "cleanupPreference": "[parameters('cleanupPreference')]" - }, - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]", - "[extensionResourceId(resourceId('Microsoft.KeyVault/vaults', parameters('kvName')), 'Microsoft.Authorization/roleAssignments', guid(resourceId('Microsoft.KeyVault/vaults', parameters('kvName')), parameters('rbacRoleNeeded'), if(parameters('useExistingManagedIdentity'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', parameters('existingManagedIdentitySubId'), parameters('existingManagedIdentityResourceGroupName')), 'Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName')))))]" - ] - } - ], - "outputs": { - "publicKeyUri": { - "type": "string", - "metadata": { - "description": "The URI of the public key secret in the Key Vault" - }, - "value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', format('AKV-Cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('sshKeyName'), ':', ''), '/', '-'))), '2020-10-01').outputs.secretUris.public]" - }, - "privateKeyUri": { - "type": "string", - "metadata": { - "description": "The URI of the private key secret in the Key Vault" - }, - "value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', format('AKV-Cert-{0}-{1}', parameters('kvName'), replace(replace(parameters('sshKeyName'), ':', ''), '/', '-'))), '2020-10-01').outputs.secretUris.private]" - }, - "publicKeySecretName": { - "type": "string", - "metadata": { - "description": "The name of the public key secret in the Key Vault" - }, - "value": "[variables('publicKeySecretName')]" - }, - "privateKeySecretName": { - "type": "string", - "metadata": { - "description": "The name of the private key secret in the Key Vault" - }, - "value": "[variables('privateKeySecretName')]" - } - } -} \ No newline at end of file diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 15885a0f..6a32ca84 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -77,19 +77,8 @@ spec: 'airflow-airflow-variables': useHelmHooks: false data: | - AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - AIRFLOW_VAR_ENV_VARS_ENABLED: "true" - AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME: "entitlements_client" - AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2" - AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" - AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + AIRFLOW_VAR_HELLO: "world" + # Airflow create user job settings createUserJob: @@ -103,6 +92,32 @@ spec: env: - name: ENVIRONMENT value: dev + - name: AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH + value: "/opt/airflow/dags/configs/dataload.ini" + - name: AIRFLOW_VAR_ENV_VARS_ENABLED + value: "true" + - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME + value: "entitlements_client" + - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL + value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST + value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL + value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL + value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL + value: "http://search.osdu-core.svc.cluster.local/api/search/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL + value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL + value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST + value: "http://file.osdu-core.svc.cluster.local/api/file" + - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST + value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST + value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" # Airflow database & redis config data: From e6758ce9ec19ba0332ce55188be71e94ad37f614 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 08:52:20 -0500 Subject: [PATCH 02/55] airflow adjustments. --- software/components/airflow/release.yaml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 6a32ca84..d3a04639 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -72,13 +72,6 @@ spec: enableBuiltInSecretEnvVars: AIRFLOW__CORE__FERNET_KEY: false AIRFLOW__WEBSERVER__SECRET_KEY: false - - extraConfigMaps: - 'airflow-airflow-variables': - useHelmHooks: false - data: | - AIRFLOW_VAR_HELLO: "world" - # Airflow create user job settings createUserJob: @@ -119,6 +112,19 @@ spec: - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + extraEnv: + - name: CLOUD_PROVIDER + value: "azure" + + extraEnvFrom: | + - configMapRef: + name: 'airflow-variables' + + extraConfigMaps: + 'airflow-variables': + data: | + AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" + # Airflow database & redis config data: metadataSecretName: airflow-secrets From 639138e9af0402fef4c1da5f96ab4332a041b567 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:01:59 -0500 Subject: [PATCH 03/55] airflow adjustments. --- software/components/airflow/release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index d3a04639..11a410a0 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -116,12 +116,12 @@ spec: - name: CLOUD_PROVIDER value: "azure" - extraEnvFrom: | - - configMapRef: - name: 'airflow-variables' + # extraEnvFrom: | + # - configMapRef: + # name: 'airflow-airflow-variables' extraConfigMaps: - 'airflow-variables': + airflow-variables: data: | AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" From 48d9bb1f2f30a4d510cc3899da5115176435a0f1 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:09:56 -0500 Subject: [PATCH 04/55] airflow adjustments. --- software/components/airflow/release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 11a410a0..9832d703 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -112,9 +112,9 @@ spec: - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - extraEnv: - - name: CLOUD_PROVIDER - value: "azure" + # extraEnv: + # - name: CLOUD_PROVIDER + # value: "azure" # extraEnvFrom: | # - configMapRef: From 895c56bc64f1e2002d0a4d5a34b7264598a8e650 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:22:12 -0500 Subject: [PATCH 05/55] airflow adjustments. --- software/components/airflow/release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 9832d703..13d8b511 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -116,9 +116,9 @@ spec: # - name: CLOUD_PROVIDER # value: "azure" - # extraEnvFrom: | - # - configMapRef: - # name: 'airflow-airflow-variables' + extraEnvFrom: | + - configMapRef: + name: "airflow-variables" extraConfigMaps: airflow-variables: From 46286531f9bf6a12ccc90045094e044ef13eccb5 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:26:20 -0500 Subject: [PATCH 06/55] airflow adjustments. --- software/components/airflow/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 13d8b511..b069e12b 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -118,7 +118,7 @@ spec: extraEnvFrom: | - configMapRef: - name: "airflow-variables" + name: 'airflow-variables' extraConfigMaps: airflow-variables: From 6ad6bb32c19cfeed722fd6a7beabf811600412be Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:34:28 -0500 Subject: [PATCH 07/55] airflow adjustments. --- software/components/airflow/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index b069e12b..f906993c 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -118,10 +118,10 @@ spec: extraEnvFrom: | - configMapRef: - name: 'airflow-variables' + name: 'airflow-airflow-variables' extraConfigMaps: - airflow-variables: + airflow-airflow-variables: data: | AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" From 4a4875047f3b6de5ee6d2a5704ef75eebdceb3d3 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:42:58 -0500 Subject: [PATCH 08/55] airflow adjustments. --- software/components/airflow/release.yaml | 43 +++++++++--------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index f906993c..e0a2328a 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -91,39 +91,28 @@ spec: value: "true" - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME value: "entitlements_client" - - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL - value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST - value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL - value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL - value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL - value: "http://search.osdu-core.svc.cluster.local/api/search/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL - value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL - value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST - value: "http://file.osdu-core.svc.cluster.local/api/file" - - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST - value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST - value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - - # extraEnv: - # - name: CLOUD_PROVIDER - # value: "azure" + + extraEnv: + - name: CLOUD_PROVIDER + value: "azure" extraEnvFrom: | - configMapRef: - name: 'airflow-airflow-variables' + name: 'airflow-endpoint-variables' extraConfigMaps: - airflow-airflow-variables: + airflow-endpoint-variables: data: | - AIRFLOW_VAR_HELLO_MESSAGE: "Hi!" + AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2" + AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" + AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" # Airflow database & redis config data: From 3c19c337ce0bb230aa5b3f4231825a618ea621d3 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:48:48 -0500 Subject: [PATCH 09/55] airflow adjustments. --- software/components/airflow/release.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index e0a2328a..7763a163 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -83,18 +83,20 @@ spec: # Environment variables for all airflow containers env: - - name: ENVIRONMENT - value: dev - name: AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH value: "/opt/airflow/dags/configs/dataload.ini" - name: AIRFLOW_VAR_ENV_VARS_ENABLED value: "true" - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME value: "entitlements_client" - + - name: CLOUD_PROVIDER + value: "azure" + - name: ENVIRONMENT + value: dev extraEnv: - name: CLOUD_PROVIDER value: "azure" + extraEnvFrom: | - configMapRef: From 891179a56940f688eb21855e603b0802ed45c0a7 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 09:55:23 -0500 Subject: [PATCH 10/55] airflow adjustments. --- software/components/airflow/release.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 7763a163..702ff787 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -92,15 +92,11 @@ spec: - name: CLOUD_PROVIDER value: "azure" - name: ENVIRONMENT - value: dev - extraEnv: - - name: CLOUD_PROVIDER - value: "azure" - + value: "dev" extraEnvFrom: | - configMapRef: - name: 'airflow-endpoint-variables' + name: "airflow-endpoint-variables" extraConfigMaps: airflow-endpoint-variables: From f7f5f2f5ad10e4763ade7ecb34ee8716be98251c Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 10:10:41 -0500 Subject: [PATCH 11/55] airflow adjustments. --- software/components/airflow/release.yaml | 27 ++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 702ff787..f296f90f 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -83,10 +83,12 @@ spec: # Environment variables for all airflow containers env: - - name: AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH - value: "/opt/airflow/dags/configs/dataload.ini" - - name: AIRFLOW_VAR_ENV_VARS_ENABLED - value: "true" + # Needed for installing python osdu python sdk. In future this will be changed + - name: CI_COMMIT_TAG + value: "v0.12.0" + - name: BUILD_TAG + value: "v0.12.0" + # - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME value: "entitlements_client" - name: CLOUD_PROVIDER @@ -97,6 +99,8 @@ spec: extraEnvFrom: | - configMapRef: name: "airflow-endpoint-variables" + - configMapRef: + name: "airflow-settings-variables" extraConfigMaps: airflow-endpoint-variables: @@ -111,6 +115,21 @@ spec: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + airflow-settings-variables: + data: | + AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 + AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" + AIRFLOW__CORE__STORE_DAG_CODE: "True" + AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + AIRFLOW_VAR_NAMESPACE: "airflow" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + # Airflow database & redis config data: From 689ac62e24a2a76b6901e9ffee061cf25ddfb81a Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 10:22:24 -0500 Subject: [PATCH 12/55] airflow adjustments. --- software/components/airflow/release.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index f296f90f..36660841 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -25,7 +25,7 @@ spec: chart: spec: chart: airflow - # version: 1.11.0 + version: 1.15.0 sourceRef: kind: HelmRepository name: airflow @@ -95,6 +95,10 @@ spec: value: "azure" - name: ENVIRONMENT value: "dev" + + extraEnv: | + - name: AIRFLOW_VAR_ANOTHER_KEY + value: 'value_1' extraEnvFrom: | - configMapRef: From 8d7086238ef66d82c4c0357ba8006acc3a694da2 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 10:28:04 -0500 Subject: [PATCH 13/55] airflow adjustments. --- software/components/airflow/release.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 36660841..8deaa1c4 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -103,8 +103,6 @@ spec: extraEnvFrom: | - configMapRef: name: "airflow-endpoint-variables" - - configMapRef: - name: "airflow-settings-variables" extraConfigMaps: airflow-endpoint-variables: From c5793aa109aabbf638511a892f6ceab1167200f9 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 10:31:59 -0500 Subject: [PATCH 14/55] airflow adjustments. --- software/components/airflow/release.yaml | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 8deaa1c4..b71f134a 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -117,20 +117,20 @@ spec: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - airflow-settings-variables: - data: | - AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 - AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" - AIRFLOW__CORE__STORE_DAG_CODE: "True" - AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" - AIRFLOW_VAR_NAMESPACE: "airflow" - AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + # airflow-settings-variables: + # data: | + # AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + # AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + # AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + # AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 + # AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + # AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" + # AIRFLOW__CORE__STORE_DAG_CODE: "True" + # AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + # AIRFLOW_VAR_NAMESPACE: "airflow" + # AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" # Airflow database & redis config From 543364a5b85bdb8191d2df8cf10f5a1eef2032e4 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 10:39:14 -0500 Subject: [PATCH 15/55] airflow adjustments. --- software/components/airflow/release.yaml | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index b71f134a..8deaa1c4 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -117,20 +117,20 @@ spec: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - # airflow-settings-variables: - # data: | - # AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - # AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - # AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - # AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 - # AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - # AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" - # AIRFLOW__CORE__STORE_DAG_CODE: "True" - # AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" - # AIRFLOW_VAR_NAMESPACE: "airflow" - # AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + airflow-settings-variables: + data: | + AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 + AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" + AIRFLOW__CORE__STORE_DAG_CODE: "True" + AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + AIRFLOW_VAR_NAMESPACE: "airflow" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" # Airflow database & redis config From 01fa650bc5644acbfa76a4cc306ebce1b94a1211 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 10:58:40 -0500 Subject: [PATCH 16/55] airflow adjustments. --- software/components/airflow/release.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 8deaa1c4..677e965e 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -105,7 +105,7 @@ spec: name: "airflow-endpoint-variables" extraConfigMaps: - airflow-endpoint-variables: + 'airflow-endpoint-variables': data: | AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" @@ -117,7 +117,8 @@ spec: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - airflow-settings-variables: + + 'airflow-settings-variables': data: | AIRFLOW_VAR_ENV_VARS_ENABLED: "True" AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" @@ -148,13 +149,18 @@ spec: logGroomerSidecar: enabled: false - # Airflow scheduler settings + + ################################### + ## COMPONENT | Airflow Scheduler + ################################### scheduler: enabled: true logGroomerSidecar: enabled: false - # Airflow webserver settings + ################################### + ## COMPONENT | Airflow Webserver + ################################### webserver: enabled: true From 79e5dae090ddd6ce718ae6f78d236b2b28d62980 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 11:09:11 -0500 Subject: [PATCH 17/55] airflow adjustments. --- software/components/airflow/release.yaml | 40 ++++++++++++++---------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 677e965e..322551c6 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -102,23 +102,10 @@ spec: extraEnvFrom: | - configMapRef: - name: "airflow-endpoint-variables" + name: "airflow-settings-variables" - extraConfigMaps: - 'airflow-endpoint-variables': - data: | - AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2" - AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" - AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - - 'airflow-settings-variables': + extraConfigMaps: + airflow-settings-variables: data: | AIRFLOW_VAR_ENV_VARS_ENABLED: "True" AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" @@ -148,6 +135,27 @@ spec: workers: logGroomerSidecar: enabled: false + env: + - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL + value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST + value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL + value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL + value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL + value: "http://search.osdu-core.svc.cluster.local/api/search/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL + value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL + value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST + value: "http://file.osdu-core.svc.cluster.local/api/file" + - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST + value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST + value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" ################################### From ffd96473fb9d4fb09db7a191f900f5aed4c10b1a Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 11:30:04 -0500 Subject: [PATCH 18/55] airflow adjustments. --- software/components/airflow/release.yaml | 83 ++++++++++++++---------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 322551c6..2f4db050 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -104,21 +104,34 @@ spec: - configMapRef: name: "airflow-settings-variables" - extraConfigMaps: - airflow-settings-variables: + extraConfigMaps: + 'airflow-settings-variables': data: | - AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 - AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" - AIRFLOW__CORE__STORE_DAG_CODE: "True" - AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" - AIRFLOW_VAR_NAMESPACE: "airflow" - AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2" + AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" + AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + + # airflow-settings-variables': + # data: | + # AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + # AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + # AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + # AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 + # AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + # AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" + # AIRFLOW__CORE__STORE_DAG_CODE: "True" + # AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + # AIRFLOW_VAR_NAMESPACE: "airflow" + # AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" # Airflow database & redis config @@ -135,27 +148,27 @@ spec: workers: logGroomerSidecar: enabled: false - env: - - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL - value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST - value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL - value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL - value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL - value: "http://search.osdu-core.svc.cluster.local/api/search/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL - value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL - value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST - value: "http://file.osdu-core.svc.cluster.local/api/file" - - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST - value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST - value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + # env: + # - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL + # value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST + # value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL + # value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + # - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL + # value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL + # value: "http://search.osdu-core.svc.cluster.local/api/search/v2" + # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL + # value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + # - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL + # value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + # - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST + # value: "http://file.osdu-core.svc.cluster.local/api/file" + # - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST + # value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST + # value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" ################################### From b338410ce2c17d2af77496ee81ff0c72cff9cd91 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 11:37:38 -0500 Subject: [PATCH 19/55] airflow adjustments. --- software/components/airflow/release.yaml | 54 ++++++++++-------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 2f4db050..562be7e7 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -107,16 +107,8 @@ spec: extraConfigMaps: 'airflow-settings-variables': data: | - AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2" - AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" - AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + # airflow-settings-variables': # data: | @@ -148,27 +140,27 @@ spec: workers: logGroomerSidecar: enabled: false - # env: - # - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL - # value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST - # value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL - # value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - # - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL - # value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL - # value: "http://search.osdu-core.svc.cluster.local/api/search/v2" - # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL - # value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - # - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL - # value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - # - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST - # value: "http://file.osdu-core.svc.cluster.local/api/file" - # - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST - # value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST - # value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + env: + - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL + value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST + value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL + value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL + value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL + value: "http://search.osdu-core.svc.cluster.local/api/search/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL + value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL + value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST + value: "http://file.osdu-core.svc.cluster.local/api/file" + - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST + value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST + value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" ################################### From 2e126b621a160a77c6c01ca772e95427ed148b62 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 11:46:18 -0500 Subject: [PATCH 20/55] airflow adjustments. --- software/components/airflow/release.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 562be7e7..e8e4ce66 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -108,6 +108,16 @@ spec: 'airflow-settings-variables': data: | AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" + AIRFLOW__CORE__STORE_DAG_CODE: "True" + AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + AIRFLOW_VAR_NAMESPACE: "airflow" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" # airflow-settings-variables': From 5ba2159b82f4c25703eb0c4cbd6217f466b30245 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 11:52:07 -0500 Subject: [PATCH 21/55] airflow adjustments. --- software/components/airflow/release.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index e8e4ce66..bd36017e 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -118,22 +118,7 @@ spec: AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" AIRFLOW_VAR_NAMESPACE: "airflow" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - - - # airflow-settings-variables': - # data: | - # AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - # AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - # AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - # AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 - # AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - # AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" - # AIRFLOW__CORE__STORE_DAG_CODE: "True" - # AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" - # AIRFLOW_VAR_NAMESPACE: "airflow" - # AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 # Airflow database & redis config From 792200f265263e2415d778798a2da49225fae25e Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 11:56:24 -0500 Subject: [PATCH 22/55] airflow adjustments. --- software/components/airflow/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index bd36017e..6312b746 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -118,7 +118,7 @@ spec: AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" AIRFLOW_VAR_NAMESPACE: "airflow" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 60 + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "60" # Airflow database & redis config From f96feee013c18b5197627c5cb421931c5bd95002 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:11:04 -0500 Subject: [PATCH 23/55] airflow adjustments. --- software/components/airflow/release.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 6312b746..3341a869 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -102,10 +102,10 @@ spec: extraEnvFrom: | - configMapRef: - name: "airflow-settings-variables" + name: "airflow-settings" extraConfigMaps: - 'airflow-settings-variables': + 'airflow-settings': data: | AIRFLOW_VAR_ENV_VARS_ENABLED: "True" AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" @@ -126,10 +126,10 @@ spec: metadataSecretName: airflow-secrets # Fernet key settings - fernetKeySecretName: keyvault-secrets + fernetKeySecretName: airflow-secrets # Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg - webserverSecretKeySecretName: keyvault-secrets + webserverSecretKeySecretName: airflow-secrets # Airflow Worker Config workers: @@ -227,7 +227,6 @@ spec: operators: default_owner: OSDU - # Pull DAGS from file share dags: persistence: @@ -237,11 +236,9 @@ spec: gitSync: enabled: false - + # Global Logs go to pvc file share. logs: persistence: enabled: true existingClaim: airflow-logs-pvc storageClassName: azurefile-csi-premium - - \ No newline at end of file From 6484a7e9d739401fb8f0660dfd5f9414e3b1c887 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:21:18 -0500 Subject: [PATCH 24/55] airflow adjustments. --- software/components/airflow/release.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 3341a869..a4ff2b3f 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -32,9 +32,11 @@ spec: namespace: flux-system values: - # Isolate version + # Version Isolation airflowVersion: 2.10.1 defaultAirflowTag: 2.10.1 + # Used for mount paths + airflowHome: /opt/airflow # Ensure this goes on the correct node pool. affinity: @@ -70,7 +72,7 @@ spec: # might want to provide some of those variables with _CMD or _SECRET variable, and you should # in this case disable setting of those variables by setting the relevant configuration to false. enableBuiltInSecretEnvVars: - AIRFLOW__CORE__FERNET_KEY: false + AIRFLOW__CORE__FERNET_KEY: true AIRFLOW__WEBSERVER__SECRET_KEY: false # Airflow create user job settings From 6114d62d8c1697f9ec158feeb2ab9959f17568f3 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:32:56 -0500 Subject: [PATCH 25/55] airflow adjustments. --- software/components/airflow/release.yaml | 4 +- .../components/airflow/vault-secrets.yaml | 45 ++++++++++++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index a4ff2b3f..4e5bb0ca 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -104,10 +104,10 @@ spec: extraEnvFrom: | - configMapRef: - name: "airflow-settings" + name: "airflow-variables" extraConfigMaps: - 'airflow-settings': + 'airflow-variables': data: | AIRFLOW_VAR_ENV_VARS_ENABLED: "True" AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index 25f13701..620bb62f 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -43,4 +43,47 @@ spec: - key: connection vaultSecret: airflow-db-connection - key: db-password - vaultSecret: cpng-user-password \ No newline at end of file + vaultSecret: cpng-user-password + - key: tenant-id + vaultSecret: app-dev-sp-tenant-id + - key: client-id + vaultSecret: app-dev-sp-id + - key: client-key + vaultSecret: app-dev-sp-password +--- +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: azure-keyvault-airflow-variables + namespace: default + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + targetNamespace: airflow + releaseName: keyvault-airflow-variables + chart: + spec: + chart: ./charts/keyvault-secrets + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: config-map-values + valuesKey: values.yaml + values: + secrets: + - secretName: airflow-variables + data: + - key: tenant-id + vaultSecret: app-dev-sp-tenant-id + - key: client-id + vaultSecret: app-dev-sp-id + - key: client-key + vaultSecret: app-dev-sp-password \ No newline at end of file From 7039af4cafb44a9c6f6d864497979130bddc8f46 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:36:53 -0500 Subject: [PATCH 26/55] airflow adjustments. --- software/components/airflow/vault-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index 620bb62f..2982d2d2 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -51,7 +51,6 @@ spec: - key: client-key vaultSecret: app-dev-sp-password --- ---- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: @@ -78,6 +77,7 @@ spec: name: config-map-values valuesKey: values.yaml values: + identity: false secrets: - secretName: airflow-variables data: From bafd36c1391aca48e2cdfc0fe9af882f2aeb9dbd Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:48:29 -0500 Subject: [PATCH 27/55] airflow adjustments. --- software/components/airflow/vault-secrets.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index 2982d2d2..d123f82a 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -81,9 +81,9 @@ spec: secrets: - secretName: airflow-variables data: - - key: tenant-id + - key: AIRFLOW_VAR_AZURE_TENANT_ID vaultSecret: app-dev-sp-tenant-id - - key: client-id + - key: AIRFLOW_VAR_AZURE_CLIENT_ID vaultSecret: app-dev-sp-id - - key: client-key + - key: AIRFLOW_VAR_AZURE_CLIENT_SECRET vaultSecret: app-dev-sp-password \ No newline at end of file From b148a66dc7c06dc7e30a681b686af0507f62627c Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:54:29 -0500 Subject: [PATCH 28/55] airflow adjustments. --- software/components/airflow/release.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 4e5bb0ca..49b6382a 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -101,10 +101,6 @@ spec: extraEnv: | - name: AIRFLOW_VAR_ANOTHER_KEY value: 'value_1' - - extraEnvFrom: | - - configMapRef: - name: "airflow-variables" extraConfigMaps: 'airflow-variables': @@ -122,6 +118,11 @@ spec: AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "60" + extraEnvFrom: | + - configMapRef: + name: "airflow-variables" + - secretRef: + name: 'airflow-variables' # Airflow database & redis config data: From 0dbe59c1f93df344a4e4ec09f9131ea6497326cd Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 12:58:03 -0500 Subject: [PATCH 29/55] airflow adjustments. --- software/components/airflow/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 49b6382a..ac5de2af 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -120,9 +120,9 @@ spec: extraEnvFrom: | - configMapRef: - name: "airflow-variables" + name: "airflow-variables" - secretRef: - name: 'airflow-variables' + name: 'airflow-variables' # Airflow database & redis config data: From bc709ba1c2e0777648d8057e467fff61bb491583 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 14:30:57 -0500 Subject: [PATCH 30/55] airflow adjustments. --- software/components/airflow/release.yaml | 1 - software/components/airflow/vault-secrets.yaml | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index ac5de2af..7b8f70d4 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -160,7 +160,6 @@ spec: - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - ################################### ## COMPONENT | Airflow Scheduler ################################### diff --git a/software/components/airflow/vault-secrets.yaml b/software/components/airflow/vault-secrets.yaml index d123f82a..30620d39 100644 --- a/software/components/airflow/vault-secrets.yaml +++ b/software/components/airflow/vault-secrets.yaml @@ -86,4 +86,8 @@ spec: - key: AIRFLOW_VAR_AZURE_CLIENT_ID vaultSecret: app-dev-sp-id - key: AIRFLOW_VAR_AZURE_CLIENT_SECRET - vaultSecret: app-dev-sp-password \ No newline at end of file + vaultSecret: app-dev-sp-password + - key: AIRFLOW_VAR_AAD_CLIENT_ID + vaultSecret: app-dev-sp-id + - key: AIRFLOW_VAR_APPINSIGHTS_KEY + vaultSecret: insights-key \ No newline at end of file From 389c52dd56e679395c49fb986364543106291dcf Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 15:22:17 -0500 Subject: [PATCH 31/55] airflow adjustments. --- software/components/airflow/release.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 7b8f70d4..ad57d799 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -111,12 +111,9 @@ spec: AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "True" - AIRFLOW__CORE__STORE_DAG_CODE: "True" AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" AIRFLOW_VAR_NAMESPACE: "airflow" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "60" extraEnvFrom: | - configMapRef: @@ -136,6 +133,7 @@ spec: # Airflow Worker Config workers: + enabled: true logGroomerSidecar: enabled: false env: From 58c13f2ac4d6e1ff7e1fbc912238bc8c77e6ae37 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 15:41:43 -0500 Subject: [PATCH 32/55] airflow adjustments. --- software/components/airflow/release.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index ad57d799..c2c7cbdb 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -133,7 +133,6 @@ spec: # Airflow Worker Config workers: - enabled: true logGroomerSidecar: enabled: false env: @@ -179,8 +178,8 @@ spec: enabled: false # Airflow Dag Processor Config - dagProcessor: - enabled: false + # dagProcessor: + # enabled: false # Flower settings flower: From da03e0689921625e4078a81c6a904d4556d9cb19 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 15:51:07 -0500 Subject: [PATCH 33/55] airflow adjustments. --- software/components/airflow/release.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index c2c7cbdb..739a2f87 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -229,11 +229,15 @@ spec: # Pull DAGS from file share dags: persistence: - enabled: false + enabled: true existingClaim: airflow-dags-pvc subPath: dags gitSync: enabled: false + repo: 'https://github.com/azure/osdu-developer' + branch: main + wait: 40 + subPath: dags # Global Logs go to pvc file share. logs: From 823837b40932d8e83501b23eafcba61831313f40 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 15:59:46 -0500 Subject: [PATCH 34/55] airflow adjustments. --- software/components/airflow/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 739a2f87..6247eb2d 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -229,11 +229,11 @@ spec: # Pull DAGS from file share dags: persistence: - enabled: true + enabled: false existingClaim: airflow-dags-pvc subPath: dags gitSync: - enabled: false + enabled: true repo: 'https://github.com/azure/osdu-developer' branch: main wait: 40 From a5b2ce630bec6a818794e130801aef078a9effd9 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 16:12:24 -0500 Subject: [PATCH 35/55] airflow adjustments. --- software/components/airflow/release.yaml | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 6247eb2d..003d2f25 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -160,22 +160,22 @@ spec: ################################### ## COMPONENT | Airflow Scheduler ################################### - scheduler: - enabled: true - logGroomerSidecar: - enabled: false + # scheduler: + # enabled: true + # logGroomerSidecar: + # enabled: false ################################### ## COMPONENT | Airflow Webserver ################################### - webserver: - enabled: true + # webserver: + # enabled: true # Airflow Triggerer Config - triggerer: - enabled: true - logGroomerSidecar: - enabled: false + # triggerer: + # enabled: true + # logGroomerSidecar: + # enabled: false # Airflow Dag Processor Config # dagProcessor: @@ -198,9 +198,9 @@ spec: enabled: false # Configuration for the redis provisioned by the chart - redis: - enabled: true - terminationGracePeriodSeconds: 600 + # redis: + # enabled: true + # terminationGracePeriodSeconds: 600 # Configuration for postgresql subchart # Not recommended for production From 09674d5076fc45eeb90537cd304bebd9d31252b5 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 16:23:48 -0500 Subject: [PATCH 36/55] airflow adjustments. --- software/components/airflow/release.yaml | 396 ++++++++++++++++++----- 1 file changed, 317 insertions(+), 79 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 003d2f25..4d1a1858 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -32,12 +32,6 @@ spec: namespace: flux-system values: - # Version Isolation - airflowVersion: 2.10.1 - defaultAirflowTag: 2.10.1 - # Used for mount paths - airflowHome: /opt/airflow - # Ensure this goes on the correct node pool. affinity: nodeAffinity: @@ -65,14 +59,14 @@ spec: # Airflow executor # One of: LocalExecutor, LocalKubernetesExecutor, CeleryExecutor, KubernetesExecutor, CeleryKubernetesExecutor - executor: KubernetesExecutor + executor: LocalKubernetesExecutor # Enables selected built-in secrets that are set via environment variables by default. # Those secrets are provided by the Helm Chart secrets by default but in some cases you # might want to provide some of those variables with _CMD or _SECRET variable, and you should # in this case disable setting of those variables by setting the relevant configuration to false. enableBuiltInSecretEnvVars: - AIRFLOW__CORE__FERNET_KEY: true + AIRFLOW__CORE__FERNET_KEY: false AIRFLOW__WEBSERVER__SECRET_KEY: false # Airflow create user job settings @@ -84,42 +78,42 @@ spec: useHelmHooks: false # Environment variables for all airflow containers - env: - # Needed for installing python osdu python sdk. In future this will be changed - - name: CI_COMMIT_TAG - value: "v0.12.0" - - name: BUILD_TAG - value: "v0.12.0" - # - - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME - value: "entitlements_client" - - name: CLOUD_PROVIDER - value: "azure" - - name: ENVIRONMENT - value: "dev" - - extraEnv: | - - name: AIRFLOW_VAR_ANOTHER_KEY - value: 'value_1' - - extraConfigMaps: - 'airflow-variables': - data: | - AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" - AIRFLOW_VAR_NAMESPACE: "airflow" - AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - - extraEnvFrom: | - - configMapRef: - name: "airflow-variables" - - secretRef: - name: 'airflow-variables' + # env: + # # Needed for installing python osdu python sdk. In future this will be changed + # - name: CI_COMMIT_TAG + # value: "v0.12.0" + # - name: BUILD_TAG + # value: "v0.12.0" + # # + # - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME + # value: "entitlements_client" + # - name: CLOUD_PROVIDER + # value: "azure" + # - name: ENVIRONMENT + # value: "dev" + + # extraEnv: | + # - name: AIRFLOW_VAR_ANOTHER_KEY + # value: 'value_1' + + # extraConfigMaps: + # 'airflow-variables': + # data: | + # AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + # AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + # AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + # AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + # AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + # AIRFLOW_VAR_NAMESPACE: "airflow" + # AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + + # extraEnvFrom: | + # - configMapRef: + # name: "airflow-variables" + # - secretRef: + # name: 'airflow-variables' # Airflow database & redis config data: @@ -132,30 +126,30 @@ spec: webserverSecretKeySecretName: airflow-secrets # Airflow Worker Config - workers: - logGroomerSidecar: - enabled: false - env: - - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL - value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST - value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL - value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL - value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL - value: "http://search.osdu-core.svc.cluster.local/api/search/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL - value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL - value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST - value: "http://file.osdu-core.svc.cluster.local/api/file" - - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST - value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST - value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + # workers: + # logGroomerSidecar: + # enabled: false + # env: + # - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL + # value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST + # value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL + # value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + # - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL + # value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL + # value: "http://search.osdu-core.svc.cluster.local/api/search/v2" + # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL + # value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + # - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL + # value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + # - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST + # value: "http://file.osdu-core.svc.cluster.local/api/file" + # - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST + # value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + # - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST + # value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" ################################### ## COMPONENT | Airflow Scheduler @@ -168,8 +162,8 @@ spec: ################################### ## COMPONENT | Airflow Webserver ################################### - # webserver: - # enabled: true + webserver: + enabled: true # Airflow Triggerer Config # triggerer: @@ -181,11 +175,6 @@ spec: # dagProcessor: # enabled: false - # Flower settings - flower: - # Enable flower. - # If True, and using CeleryExecutor/CeleryKubernetesExecutor, will deploy flower app. - enabled: false # StatsD settings statsd: @@ -228,10 +217,6 @@ spec: # Pull DAGS from file share dags: - persistence: - enabled: false - existingClaim: airflow-dags-pvc - subPath: dags gitSync: enabled: true repo: 'https://github.com/azure/osdu-developer' @@ -245,3 +230,256 @@ spec: enabled: true existingClaim: airflow-logs-pvc storageClassName: azurefile-csi-premium + + + + + + +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: airflow +# namespace: flux-system +# labels: +# name: airflow +# spec: +# targetNamespace: airflow +# releaseName: airflow +# dependsOn: +# - name: azure-keyvault-airflow +# namespace: default +# install: +# remediation: +# retries: 3 +# upgrade: +# remediation: +# retries: 10 +# strategy: rollback +# test: +# enable: true +# interval: 10m0s +# chart: +# spec: +# chart: airflow +# version: 1.15.0 +# sourceRef: +# kind: HelmRepository +# name: airflow +# namespace: flux-system +# values: + +# # Version Isolation +# airflowVersion: 2.10.1 +# defaultAirflowTag: 2.10.1 +# # Used for mount paths +# airflowHome: /opt/airflow + +# # Ensure this goes on the correct node pool. +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: agentpool +# operator: In +# values: +# - poolz1 +# - poolz2 +# - poolz3 +# topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: topology.kubernetes.io/zone +# whenUnsatisfiable: DoNotSchedule +# labelSelector: +# matchLabels: +# release: airflow # Adjust to match your Helm release name +# tolerations: +# - effect: NoSchedule +# key: app +# value: "cluster" + + +# # Airflow executor +# # One of: LocalExecutor, LocalKubernetesExecutor, CeleryExecutor, KubernetesExecutor, CeleryKubernetesExecutor +# executor: KubernetesExecutor + +# # Enables selected built-in secrets that are set via environment variables by default. +# # Those secrets are provided by the Helm Chart secrets by default but in some cases you +# # might want to provide some of those variables with _CMD or _SECRET variable, and you should +# # in this case disable setting of those variables by setting the relevant configuration to false. +# enableBuiltInSecretEnvVars: +# AIRFLOW__CORE__FERNET_KEY: true +# AIRFLOW__WEBSERVER__SECRET_KEY: false + +# # Airflow create user job settings +# createUserJob: +# useHelmHooks: false + +# # Airflow database migration job settings +# migrateDatabaseJob: +# useHelmHooks: false + +# # Environment variables for all airflow containers +# env: +# # Needed for installing python osdu python sdk. In future this will be changed +# - name: CI_COMMIT_TAG +# value: "v0.12.0" +# - name: BUILD_TAG +# value: "v0.12.0" +# # +# - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME +# value: "entitlements_client" +# - name: CLOUD_PROVIDER +# value: "azure" +# - name: ENVIRONMENT +# value: "dev" + +# extraEnv: | +# - name: AIRFLOW_VAR_ANOTHER_KEY +# value: 'value_1' + +# extraConfigMaps: +# 'airflow-variables': +# data: | +# AIRFLOW_VAR_ENV_VARS_ENABLED: "True" +# AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" +# AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" +# AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" +# AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" +# AIRFLOW__CORE__LOGGING_LEVEL: "INFO" +# AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" +# AIRFLOW_VAR_NAMESPACE: "airflow" +# AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + +# extraEnvFrom: | +# - configMapRef: +# name: "airflow-variables" +# - secretRef: +# name: 'airflow-variables' + +# # Airflow database & redis config +# data: +# metadataSecretName: airflow-secrets + +# # Fernet key settings +# fernetKeySecretName: airflow-secrets + +# # Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg +# webserverSecretKeySecretName: airflow-secrets + +# # Airflow Worker Config +# workers: +# logGroomerSidecar: +# enabled: false +# env: +# - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL +# value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" +# - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST +# value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" +# - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL +# value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" +# - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL +# value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" +# - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL +# value: "http://search.osdu-core.svc.cluster.local/api/search/v2" +# - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL +# value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" +# - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL +# value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" +# - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST +# value: "http://file.osdu-core.svc.cluster.local/api/file" +# - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST +# value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" +# - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST +# value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + +# ################################### +# ## COMPONENT | Airflow Scheduler +# ################################### +# # scheduler: +# # enabled: true +# # logGroomerSidecar: +# # enabled: false + +# ################################### +# ## COMPONENT | Airflow Webserver +# ################################### +# # webserver: +# # enabled: true + +# # Airflow Triggerer Config +# # triggerer: +# # enabled: true +# # logGroomerSidecar: +# # enabled: false + +# # Airflow Dag Processor Config +# # dagProcessor: +# # enabled: false + +# # Flower settings +# flower: +# # Enable flower. +# # If True, and using CeleryExecutor/CeleryKubernetesExecutor, will deploy flower app. +# enabled: false + +# # StatsD settings +# statsd: +# enabled: true + +# # PgBouncer settings +# # CloudNativePG provides native support for connection pooling with PgBouncer +# pgbouncer: +# # Enable PgBouncer +# enabled: false + +# # Configuration for the redis provisioned by the chart +# # redis: +# # enabled: true +# # terminationGracePeriodSeconds: 600 + +# # Configuration for postgresql subchart +# # Not recommended for production +# postgresql: +# enabled: false + +# # Config settings to go into the mounted airflow.cfg +# # +# # Please note that these values are passed through the `tpl` function, so are +# # all subject to being rendered as go templates. If you need to include a +# # literal `{{` in a value, it must be expressed like this: +# # +# # a: '{{ "{{ not a template }}" }}' +# # +# # Do not set config containing secrets via plain text values, use Env Var or k8s secret object +# # yamllint disable rule:line-length +# config: +# webserver: +# expose_config: 'True' +# instance_name: OSDU +# enable_proxy_fix: 'True' +# # base_url: 'http://localhost/airflow' +# operators: +# default_owner: OSDU + +# # Pull DAGS from file share +# dags: +# persistence: +# enabled: false +# existingClaim: airflow-dags-pvc +# subPath: dags +# gitSync: +# enabled: true +# repo: 'https://github.com/azure/osdu-developer' +# branch: main +# wait: 40 +# subPath: dags + +# # Global Logs go to pvc file share. +# logs: +# persistence: +# enabled: true +# existingClaim: airflow-logs-pvc +# storageClassName: azurefile-csi-premium From 9e05fd7600e13d3f04a61d1abbf10cfa1c72f006 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 16:35:18 -0500 Subject: [PATCH 37/55] airflow adjustments. --- software/components/airflow/release.yaml | 158 +++-------------------- 1 file changed, 20 insertions(+), 138 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 4d1a1858..e873f71e 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -57,179 +57,61 @@ spec: value: "cluster" - # Airflow executor - # One of: LocalExecutor, LocalKubernetesExecutor, CeleryExecutor, KubernetesExecutor, CeleryKubernetesExecutor + # Airflow Executor executor: LocalKubernetesExecutor - # Enables selected built-in secrets that are set via environment variables by default. - # Those secrets are provided by the Helm Chart secrets by default but in some cases you - # might want to provide some of those variables with _CMD or _SECRET variable, and you should - # in this case disable setting of those variables by setting the relevant configuration to false. + # Enable secrets enableBuiltInSecretEnvVars: AIRFLOW__CORE__FERNET_KEY: false AIRFLOW__WEBSERVER__SECRET_KEY: false - - # Airflow create user job settings + + # Create user and migrate database jobs createUserJob: useHelmHooks: false - - # Airflow database migration job settings migrateDatabaseJob: useHelmHooks: false - # Environment variables for all airflow containers - # env: - # # Needed for installing python osdu python sdk. In future this will be changed - # - name: CI_COMMIT_TAG - # value: "v0.12.0" - # - name: BUILD_TAG - # value: "v0.12.0" - # # - # - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME - # value: "entitlements_client" - # - name: CLOUD_PROVIDER - # value: "azure" - # - name: ENVIRONMENT - # value: "dev" - - # extraEnv: | - # - name: AIRFLOW_VAR_ANOTHER_KEY - # value: 'value_1' - - # extraConfigMaps: - # 'airflow-variables': - # data: | - # AIRFLOW_VAR_ENV_VARS_ENABLED: "True" - # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - # AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - # AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - # AIRFLOW__CORE__LOGGING_LEVEL: "INFO" - # AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" - # AIRFLOW_VAR_NAMESPACE: "airflow" - # AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - - # extraEnvFrom: | - # - configMapRef: - # name: "airflow-variables" - # - secretRef: - # name: 'airflow-variables' - - # Airflow database & redis config + + # Database and secrets data: metadataSecretName: airflow-secrets - - # Fernet key settings fernetKeySecretName: airflow-secrets - - # Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg webserverSecretKeySecretName: airflow-secrets - # Airflow Worker Config - # workers: - # logGroomerSidecar: - # enabled: false - # env: - # - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL - # value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST - # value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL - # value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" - # - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL - # value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL - # value: "http://search.osdu-core.svc.cluster.local/api/search/v2" - # - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL - # value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" - # - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL - # value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" - # - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST - # value: "http://file.osdu-core.svc.cluster.local/api/file" - # - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST - # value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" - # - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST - # value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - - ################################### - ## COMPONENT | Airflow Scheduler - ################################### - # scheduler: - # enabled: true - # logGroomerSidecar: - # enabled: false - - ################################### - ## COMPONENT | Airflow Webserver - ################################### - webserver: - enabled: true - - # Airflow Triggerer Config - # triggerer: - # enabled: true - # logGroomerSidecar: - # enabled: false - - # Airflow Dag Processor Config - # dagProcessor: - # enabled: false - - - # StatsD settings - statsd: - enabled: true - - # PgBouncer settings - # CloudNativePG provides native support for connection pooling with PgBouncer - pgbouncer: - # Enable PgBouncer - enabled: false - - # Configuration for the redis provisioned by the chart - # redis: - # enabled: true - # terminationGracePeriodSeconds: 600 - - # Configuration for postgresql subchart - # Not recommended for production - postgresql: - enabled: false - - # Config settings to go into the mounted airflow.cfg - # - # Please note that these values are passed through the `tpl` function, so are - # all subject to being rendered as go templates. If you need to include a - # literal `{{` in a value, it must be expressed like this: - # - # a: '{{ "{{ not a template }}" }}' - # - # Do not set config containing secrets via plain text values, use Env Var or k8s secret object - # yamllint disable rule:line-length + # Airflow Webserver configuration config: webserver: expose_config: 'True' instance_name: OSDU enable_proxy_fix: 'True' - # base_url: 'http://localhost/airflow' operators: default_owner: OSDU - # Pull DAGS from file share + # GitSync for DAGS dags: gitSync: enabled: true - repo: 'https://github.com/azure/osdu-developer' + repo: 'https://github.com/polyseam/demo-dag-bag' + credentialsSecret: airflow-git-credentials branch: main wait: 40 subPath: dags - # Global Logs go to pvc file share. + # Logs configuration logs: persistence: enabled: true existingClaim: airflow-logs-pvc storageClassName: azurefile-csi-premium + + # StatsD settings + statsd: + enabled: true + + # PostgreSQL config + postgresql: + enabled: false + From 6c90556e59109281fe1e789b321362a1e7728389 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 16:39:34 -0500 Subject: [PATCH 38/55] airflow adjustments. --- software/components/airflow/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index e873f71e..eed38bd1 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -68,8 +68,10 @@ spec: # Create user and migrate database jobs createUserJob: useHelmHooks: false + applyCustomEnv: false migrateDatabaseJob: useHelmHooks: false + applyCustomEnv: false # Database and secrets From 20c4831ab57d24ec4084dcf93339aa587dd6b0dd Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 16:47:38 -0500 Subject: [PATCH 39/55] airflow adjustments. --- software/components/airflow/release.yaml | 145 +++++++++++++---------- 1 file changed, 85 insertions(+), 60 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index eed38bd1..eec3cb5b 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -25,12 +25,92 @@ spec: chart: spec: chart: airflow - version: 1.15.0 + # version: 1.11.0 sourceRef: kind: HelmRepository name: airflow namespace: flux-system values: + # Airflow executor + executor: KubernetesExecutor + + # Environment variables for all airflow containers + env: + - name: ENVIRONMENT + value: dev + + # Admin user configuration + admin: + existingSecret: airflow-secrets + usernameKey: username + passwordKey: password + + # Disable the internal PostgreSQL chart + postgresql: + enabled: false + + # External PostgreSQL configuration + data: + metadataSecretName: airflow-secrets + + # Disable pgbouncer. + # CloudNativePG provides native support for connection pooling with PgBouncer + pgbouncer: + enabled: false + + # Pull DAGS from file share and from a git repo + dags: + gitSync: + enabled: true + repo: 'https://github.com/azure/osdu-developer.git' + branch: main + wait: 40 + subPath: dags + # persistence: + # enabled: false + # existingClaim: airflow-dags-pvc + # subPath: dags + + logs: + persistence: + enabled: true + existingClaim: airflow-logs-pvc + storageClassName: azurefile-csi-premium + + config: + webserver: + expose_config: 'True' + instance_name: OSDU + enable_proxy_fix: 'True' + # base_url: 'http://localhost/airflow' + operators: + default_owner: OSDU + + # We disable the log groomer sidecars because we use Azure File Storage for logs + triggerer: + logGroomerSidecar: + enabled: false + scheduler: + logGroomerSidecar: + enabled: false + workers: + logGroomerSidecar: + enabled: false + + # Ensure we are using the secrets from Azure Key Vault + enableBuiltInSecretEnvVars: + AIRFLOW__CORE__FERNET_KEY: false + AIRFLOW__WEBSERVER__SECRET_KEY: false + fernetKeySecretName: keyvault-secrets + webserverSecretKeySecretName: keyvault-secrets + + # Airflow create user job settings + createUserJob: + useHelmHooks: false + + # Airflow database migration job settings + migrateDatabaseJob: + useHelmHooks: false # Ensure this goes on the correct node pool. affinity: @@ -43,7 +123,8 @@ spec: values: - poolz1 - poolz2 - - poolz3 + - poolz3 # Adjust these values to match your node pools + topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone @@ -51,70 +132,14 @@ spec: labelSelector: matchLabels: release: airflow # Adjust to match your Helm release name + + # Tolerations should be defined outside of affinity tolerations: - effect: NoSchedule key: app value: "cluster" - # Airflow Executor - executor: LocalKubernetesExecutor - - # Enable secrets - enableBuiltInSecretEnvVars: - AIRFLOW__CORE__FERNET_KEY: false - AIRFLOW__WEBSERVER__SECRET_KEY: false - - # Create user and migrate database jobs - createUserJob: - useHelmHooks: false - applyCustomEnv: false - migrateDatabaseJob: - useHelmHooks: false - applyCustomEnv: false - - - # Database and secrets - data: - metadataSecretName: airflow-secrets - fernetKeySecretName: airflow-secrets - webserverSecretKeySecretName: airflow-secrets - - # Airflow Webserver configuration - config: - webserver: - expose_config: 'True' - instance_name: OSDU - enable_proxy_fix: 'True' - operators: - default_owner: OSDU - - # GitSync for DAGS - dags: - gitSync: - enabled: true - repo: 'https://github.com/polyseam/demo-dag-bag' - credentialsSecret: airflow-git-credentials - branch: main - wait: 40 - subPath: dags - - # Logs configuration - logs: - persistence: - enabled: true - existingClaim: airflow-logs-pvc - storageClassName: azurefile-csi-premium - - # StatsD settings - statsd: - enabled: true - - # PostgreSQL config - postgresql: - enabled: false - - From 31b3cc0fec2b49e2ab02731b51508b3a65ee5b78 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 16:55:23 -0500 Subject: [PATCH 40/55] airflow adjustments. --- software/components/airflow/release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index eec3cb5b..fab96e21 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -47,11 +47,11 @@ spec: # Disable the internal PostgreSQL chart postgresql: - enabled: false + enabled: true # External PostgreSQL configuration - data: - metadataSecretName: airflow-secrets + # data: + # metadataSecretName: airflow-secrets # Disable pgbouncer. # CloudNativePG provides native support for connection pooling with PgBouncer From 28d43592fef1ddf677a052b62cadab6d0304626c Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 17:11:37 -0500 Subject: [PATCH 41/55] airflow adjustments. --- software/components/airflow/release.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index fab96e21..5a9e79ef 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -62,7 +62,8 @@ spec: dags: gitSync: enabled: true - repo: 'https://github.com/azure/osdu-developer.git' + repo: 'https://github.com/polyseam/demo-dag-bag' + credentialsSecret: airflow-git-credentials branch: main wait: 40 subPath: dags From 29cfb4f6f4d28b1fca7d0585e433790eb38ba962 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 17:38:11 -0500 Subject: [PATCH 42/55] airflow adjustments. --- software/components/airflow/release.yaml | 279 +++++++++++------------ 1 file changed, 138 insertions(+), 141 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 5a9e79ef..e4b73bd2 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -1,144 +1,141 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: airflow - namespace: flux-system - labels: - name: airflow -spec: - targetNamespace: airflow - releaseName: airflow - dependsOn: - - name: azure-keyvault-airflow - namespace: default - install: - remediation: - retries: 3 - upgrade: - remediation: - retries: 10 - strategy: rollback - test: - enable: true - interval: 10m0s - chart: - spec: - chart: airflow - # version: 1.11.0 - sourceRef: - kind: HelmRepository - name: airflow - namespace: flux-system - values: - # Airflow executor - executor: KubernetesExecutor - - # Environment variables for all airflow containers - env: - - name: ENVIRONMENT - value: dev - - # Admin user configuration - admin: - existingSecret: airflow-secrets - usernameKey: username - passwordKey: password - - # Disable the internal PostgreSQL chart - postgresql: - enabled: true - - # External PostgreSQL configuration - # data: - # metadataSecretName: airflow-secrets - - # Disable pgbouncer. - # CloudNativePG provides native support for connection pooling with PgBouncer - pgbouncer: - enabled: false - - # Pull DAGS from file share and from a git repo - dags: - gitSync: - enabled: true - repo: 'https://github.com/polyseam/demo-dag-bag' - credentialsSecret: airflow-git-credentials - branch: main - wait: 40 - subPath: dags - # persistence: - # enabled: false - # existingClaim: airflow-dags-pvc - # subPath: dags - - logs: - persistence: - enabled: true - existingClaim: airflow-logs-pvc - storageClassName: azurefile-csi-premium - - config: - webserver: - expose_config: 'True' - instance_name: OSDU - enable_proxy_fix: 'True' - # base_url: 'http://localhost/airflow' - operators: - default_owner: OSDU - - # We disable the log groomer sidecars because we use Azure File Storage for logs - triggerer: - logGroomerSidecar: - enabled: false - scheduler: - logGroomerSidecar: - enabled: false - workers: - logGroomerSidecar: - enabled: false - - # Ensure we are using the secrets from Azure Key Vault - enableBuiltInSecretEnvVars: - AIRFLOW__CORE__FERNET_KEY: false - AIRFLOW__WEBSERVER__SECRET_KEY: false - fernetKeySecretName: keyvault-secrets - webserverSecretKeySecretName: keyvault-secrets - - # Airflow create user job settings - createUserJob: - useHelmHooks: false - - # Airflow database migration job settings - migrateDatabaseJob: - useHelmHooks: false - - # Ensure this goes on the correct node pool. - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: agentpool - operator: In - values: - - poolz1 - - poolz2 - - poolz3 # Adjust these values to match your node pools - - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - release: airflow # Adjust to match your Helm release name - - # Tolerations should be defined outside of affinity - tolerations: - - effect: NoSchedule - key: app - value: "cluster" +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: airflow +# namespace: flux-system +# labels: +# name: airflow +# spec: +# targetNamespace: airflow +# releaseName: airflow +# dependsOn: +# - name: azure-keyvault-airflow +# namespace: default +# install: +# remediation: +# retries: 3 +# upgrade: +# remediation: +# retries: 10 +# strategy: rollback +# test: +# enable: true +# interval: 10m0s +# chart: +# spec: +# chart: airflow +# # version: 1.11.0 +# sourceRef: +# kind: HelmRepository +# name: airflow +# namespace: flux-system +# values: +# # Airflow executor +# executor: KubernetesExecutor + +# # Environment variables for all airflow containers +# env: +# - name: ENVIRONMENT +# value: dev + +# # Admin user configuration +# admin: +# existingSecret: airflow-secrets +# usernameKey: username +# passwordKey: password + +# # Disable the internal PostgreSQL chart +# postgresql: +# enabled: true + +# # External PostgreSQL configuration +# # data: +# # metadataSecretName: airflow-secrets + +# # Disable pgbouncer. +# # CloudNativePG provides native support for connection pooling with PgBouncer +# pgbouncer: +# enabled: false + +# # Pull DAGS from file share and from a git repo +# dags: +# persistence: +# enabled: false +# gitSync: +# enabled: true +# repo: 'https://github.com/azure/osdu-developer' +# branch: main +# subPath: dags +# depth: 1 + +# logs: +# persistence: +# enabled: true +# existingClaim: airflow-logs-pvc +# storageClassName: azurefile-csi-premium + +# config: +# webserver: +# expose_config: 'True' +# instance_name: OSDU +# enable_proxy_fix: 'True' +# # base_url: 'http://localhost/airflow' +# operators: +# default_owner: OSDU + +# # We disable the log groomer sidecars because we use Azure File Storage for logs +# triggerer: +# logGroomerSidecar: +# enabled: false +# scheduler: +# logGroomerSidecar: +# enabled: false +# workers: +# logGroomerSidecar: +# enabled: false + +# # Ensure we are using the secrets from Azure Key Vault +# enableBuiltInSecretEnvVars: +# AIRFLOW__CORE__FERNET_KEY: false +# AIRFLOW__WEBSERVER__SECRET_KEY: false +# fernetKeySecretName: keyvault-secrets +# webserverSecretKeySecretName: keyvault-secrets + +# # Airflow create user job settings +# createUserJob: +# useHelmHooks: false + +# # Airflow database migration job settings +# migrateDatabaseJob: +# useHelmHooks: false + +# # Ensure this goes on the correct node pool. +# affinity: +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: agentpool +# operator: In +# values: +# - poolz1 +# - poolz2 +# - poolz3 # Adjust these values to match your node pools + +# topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: topology.kubernetes.io/zone +# whenUnsatisfiable: DoNotSchedule +# labelSelector: +# matchLabels: +# release: airflow # Adjust to match your Helm release name + +# # Tolerations should be defined outside of affinity +# tolerations: +# - effect: NoSchedule +# key: app +# value: "cluster" From ea0daf9cab4692a3c1052ff02174c75d13df8fd5 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 19:47:12 -0500 Subject: [PATCH 43/55] airflow adjustments. --- software/components/airflow/pvc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index c3d9daea..5a091272 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -49,7 +49,7 @@ spec: accessModes: - ReadWriteMany azureFile: - secretName: keyvault-secrets + secretName: airflow-secrets shareName: airflow-dags volumeMode: Filesystem mountOptions: From f47a36d8d044f70947555667ba024e28c14848b1 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 19:50:45 -0500 Subject: [PATCH 44/55] airflow adjustments. --- software/components/airflow/pvc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index 5a091272..41c70ba7 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -12,7 +12,7 @@ spec: accessModes: - ReadWriteMany azureFile: - secretName: keyvault-secrets + secretName: airflow-secrets shareName: airflow-logs volumeMode: Filesystem mountOptions: From 80b3d3593164ac5dbb49f521c35dbad2d515a16c Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 20:29:36 -0500 Subject: [PATCH 45/55] airflow adjustments. --- software/components/airflow/pvc.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index 41c70ba7..33061886 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -28,13 +28,17 @@ kind: PersistentVolumeClaim metadata: name: airflow-logs-pvc namespace: airflow + annotations: + volume.beta.kubernetes.io/storage-class: "" spec: accessModes: - ReadWriteMany resources: requests: storage: 5Gi - storageClassName: azurefile-csi-premium # Use this for premium Azure Files + selector: + matchLabels: + usage: airflow-logs-pv --- apiVersion: v1 kind: PersistentVolume @@ -65,10 +69,14 @@ kind: PersistentVolumeClaim metadata: name: airflow-dags-pvc namespace: airflow + annotations: + volume.beta.kubernetes.io/storage-class: "" spec: accessModes: - ReadWriteMany resources: requests: storage: 5Gi - storageClassName: azurefile-csi-premium # Use this for premium Azure Files \ No newline at end of file + selector: + matchLabels: + usage: airflow-logs-pv \ No newline at end of file From ead697e2c9e97d274e69c5b9708bc614003cb1aa Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 20:50:47 -0500 Subject: [PATCH 46/55] airflow adjustments. --- software/components/airflow/pvc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/components/airflow/pvc.yaml b/software/components/airflow/pvc.yaml index 33061886..064e155f 100644 --- a/software/components/airflow/pvc.yaml +++ b/software/components/airflow/pvc.yaml @@ -79,4 +79,4 @@ spec: storage: 5Gi selector: matchLabels: - usage: airflow-logs-pv \ No newline at end of file + usage: airflow-dags-pv \ No newline at end of file From 9cbbb4a249f39b94e7640038888ca280c2a131b4 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 21:17:37 -0500 Subject: [PATCH 47/55] airflow adjustments. --- software/components/airflow/release.yaml | 272 +++++++++++------------ 1 file changed, 134 insertions(+), 138 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index e4b73bd2..2f309b82 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -1,141 +1,137 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: airflow -# namespace: flux-system -# labels: -# name: airflow -# spec: -# targetNamespace: airflow -# releaseName: airflow -# dependsOn: -# - name: azure-keyvault-airflow -# namespace: default -# install: -# remediation: -# retries: 3 -# upgrade: -# remediation: -# retries: 10 -# strategy: rollback -# test: -# enable: true -# interval: 10m0s -# chart: -# spec: -# chart: airflow -# # version: 1.11.0 -# sourceRef: -# kind: HelmRepository -# name: airflow -# namespace: flux-system -# values: -# # Airflow executor -# executor: KubernetesExecutor - -# # Environment variables for all airflow containers -# env: -# - name: ENVIRONMENT -# value: dev - -# # Admin user configuration -# admin: -# existingSecret: airflow-secrets -# usernameKey: username -# passwordKey: password - -# # Disable the internal PostgreSQL chart -# postgresql: -# enabled: true - -# # External PostgreSQL configuration -# # data: -# # metadataSecretName: airflow-secrets - -# # Disable pgbouncer. -# # CloudNativePG provides native support for connection pooling with PgBouncer -# pgbouncer: -# enabled: false - -# # Pull DAGS from file share and from a git repo -# dags: -# persistence: -# enabled: false -# gitSync: -# enabled: true -# repo: 'https://github.com/azure/osdu-developer' -# branch: main -# subPath: dags -# depth: 1 - -# logs: -# persistence: -# enabled: true -# existingClaim: airflow-logs-pvc -# storageClassName: azurefile-csi-premium - -# config: -# webserver: -# expose_config: 'True' -# instance_name: OSDU -# enable_proxy_fix: 'True' -# # base_url: 'http://localhost/airflow' -# operators: -# default_owner: OSDU - -# # We disable the log groomer sidecars because we use Azure File Storage for logs -# triggerer: -# logGroomerSidecar: -# enabled: false -# scheduler: -# logGroomerSidecar: -# enabled: false -# workers: -# logGroomerSidecar: -# enabled: false - -# # Ensure we are using the secrets from Azure Key Vault -# enableBuiltInSecretEnvVars: -# AIRFLOW__CORE__FERNET_KEY: false -# AIRFLOW__WEBSERVER__SECRET_KEY: false -# fernetKeySecretName: keyvault-secrets -# webserverSecretKeySecretName: keyvault-secrets - -# # Airflow create user job settings -# createUserJob: -# useHelmHooks: false - -# # Airflow database migration job settings -# migrateDatabaseJob: -# useHelmHooks: false - -# # Ensure this goes on the correct node pool. -# affinity: -# nodeAffinity: -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: agentpool -# operator: In -# values: -# - poolz1 -# - poolz2 -# - poolz3 # Adjust these values to match your node pools - -# topologySpreadConstraints: -# - maxSkew: 1 -# topologyKey: topology.kubernetes.io/zone -# whenUnsatisfiable: DoNotSchedule -# labelSelector: -# matchLabels: -# release: airflow # Adjust to match your Helm release name - -# # Tolerations should be defined outside of affinity -# tolerations: -# - effect: NoSchedule -# key: app -# value: "cluster" +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: airflow + namespace: flux-system + labels: + name: airflow +spec: + targetNamespace: airflow + releaseName: airflow + dependsOn: + - name: azure-keyvault-airflow + namespace: default + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 10 + strategy: rollback + test: + enable: true + interval: 10m0s + chart: + spec: + chart: airflow + # version: 1.11.0 + sourceRef: + kind: HelmRepository + name: airflow + namespace: flux-system + values: + # Airflow executor + executor: KubernetesExecutor + + # Environment variables for all airflow containers + env: + - name: ENVIRONMENT + value: dev + + # Admin user configuration + admin: + existingSecret: airflow-secrets + usernameKey: username + passwordKey: password + + # Disable the internal PostgreSQL chart + postgresql: + enabled: false + + # External PostgreSQL configuration + data: + metadataSecretName: airflow-secrets + + # Disable pgbouncer. + # CloudNativePG provides native support for connection pooling with PgBouncer + pgbouncer: + enabled: false + + # Pull DAGS from file share and from a git repo + dags: + persistence: + enabled: true + existingClaim: airflow-dags-pvc + gitSync: + enabled: false + + logs: + persistence: + enabled: true + existingClaim: airflow-logs-pvc + + config: + webserver: + expose_config: 'True' + instance_name: OSDU + enable_proxy_fix: 'True' + # base_url: 'http://localhost/airflow' + operators: + default_owner: OSDU + + # We disable the log groomer sidecars because we use Azure File Storage for logs + triggerer: + logGroomerSidecar: + enabled: false + scheduler: + logGroomerSidecar: + enabled: false + workers: + logGroomerSidecar: + enabled: false + + # Ensure we are using the secrets from Azure Key Vault + enableBuiltInSecretEnvVars: + AIRFLOW__CORE__FERNET_KEY: false + AIRFLOW__WEBSERVER__SECRET_KEY: false + fernetKeySecretName: keyvault-secrets + webserverSecretKeySecretName: keyvault-secrets + + # Airflow create user job settings + createUserJob: + useHelmHooks: false + + # Airflow database migration job settings + migrateDatabaseJob: + useHelmHooks: false + + # Ensure this goes on the correct node pool. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: agentpool + operator: In + values: + - poolz1 + - poolz2 + - poolz3 # Adjust these values to match your node pools + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + release: airflow # Adjust to match your Helm release name + + # Tolerations should be defined outside of affinity + tolerations: + - effect: NoSchedule + key: app + value: "cluster" From 2308dc2c66a3bba7975c20ec2081b3992f0fa871 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 21:40:25 -0500 Subject: [PATCH 48/55] airflow adjustments. --- software/components/airflow/release.yaml | 256 +---------------------- 1 file changed, 9 insertions(+), 247 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 2f309b82..20c084bb 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -31,9 +31,18 @@ spec: name: airflow namespace: flux-system values: + + # Version Isolation + airflowVersion: 2.10.1 + defaultAirflowTag: 2.10.1 + # Airflow executor executor: KubernetesExecutor + enableBuiltInSecretEnvVars: + AIRFLOW__CORE__FERNET_KEY: true + AIRFLOW__WEBSERVER__SECRET_KEY: false + # Environment variables for all airflow containers env: - name: ENVIRONMENT @@ -139,250 +148,3 @@ spec: -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: airflow -# namespace: flux-system -# labels: -# name: airflow -# spec: -# targetNamespace: airflow -# releaseName: airflow -# dependsOn: -# - name: azure-keyvault-airflow -# namespace: default -# install: -# remediation: -# retries: 3 -# upgrade: -# remediation: -# retries: 10 -# strategy: rollback -# test: -# enable: true -# interval: 10m0s -# chart: -# spec: -# chart: airflow -# version: 1.15.0 -# sourceRef: -# kind: HelmRepository -# name: airflow -# namespace: flux-system -# values: - -# # Version Isolation -# airflowVersion: 2.10.1 -# defaultAirflowTag: 2.10.1 -# # Used for mount paths -# airflowHome: /opt/airflow - -# # Ensure this goes on the correct node pool. -# affinity: -# nodeAffinity: -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: agentpool -# operator: In -# values: -# - poolz1 -# - poolz2 -# - poolz3 -# topologySpreadConstraints: -# - maxSkew: 1 -# topologyKey: topology.kubernetes.io/zone -# whenUnsatisfiable: DoNotSchedule -# labelSelector: -# matchLabels: -# release: airflow # Adjust to match your Helm release name -# tolerations: -# - effect: NoSchedule -# key: app -# value: "cluster" - - -# # Airflow executor -# # One of: LocalExecutor, LocalKubernetesExecutor, CeleryExecutor, KubernetesExecutor, CeleryKubernetesExecutor -# executor: KubernetesExecutor - -# # Enables selected built-in secrets that are set via environment variables by default. -# # Those secrets are provided by the Helm Chart secrets by default but in some cases you -# # might want to provide some of those variables with _CMD or _SECRET variable, and you should -# # in this case disable setting of those variables by setting the relevant configuration to false. -# enableBuiltInSecretEnvVars: -# AIRFLOW__CORE__FERNET_KEY: true -# AIRFLOW__WEBSERVER__SECRET_KEY: false - -# # Airflow create user job settings -# createUserJob: -# useHelmHooks: false - -# # Airflow database migration job settings -# migrateDatabaseJob: -# useHelmHooks: false - -# # Environment variables for all airflow containers -# env: -# # Needed for installing python osdu python sdk. In future this will be changed -# - name: CI_COMMIT_TAG -# value: "v0.12.0" -# - name: BUILD_TAG -# value: "v0.12.0" -# # -# - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME -# value: "entitlements_client" -# - name: CLOUD_PROVIDER -# value: "azure" -# - name: ENVIRONMENT -# value: "dev" - -# extraEnv: | -# - name: AIRFLOW_VAR_ANOTHER_KEY -# value: 'value_1' - -# extraConfigMaps: -# 'airflow-variables': -# data: | -# AIRFLOW_VAR_ENV_VARS_ENABLED: "True" -# AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" -# AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" -# AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" -# AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" -# AIRFLOW__CORE__LOGGING_LEVEL: "INFO" -# AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" -# AIRFLOW_VAR_NAMESPACE: "airflow" -# AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - -# extraEnvFrom: | -# - configMapRef: -# name: "airflow-variables" -# - secretRef: -# name: 'airflow-variables' - -# # Airflow database & redis config -# data: -# metadataSecretName: airflow-secrets - -# # Fernet key settings -# fernetKeySecretName: airflow-secrets - -# # Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg -# webserverSecretKeySecretName: airflow-secrets - -# # Airflow Worker Config -# workers: -# logGroomerSidecar: -# enabled: false -# env: -# - name: AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL -# value: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" -# - name: AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST -# value: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" -# - name: AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL -# value: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" -# - name: AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL -# value: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" -# - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL -# value: "http://search.osdu-core.svc.cluster.local/api/search/v2" -# - name: AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL -# value: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" -# - name: AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL -# value: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" -# - name: AIRFLOW_VAR_CORE__SERVICE__FILE__HOST -# value: "http://file.osdu-core.svc.cluster.local/api/file" -# - name: AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST -# value: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" -# - name: AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST -# value: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" - -# ################################### -# ## COMPONENT | Airflow Scheduler -# ################################### -# # scheduler: -# # enabled: true -# # logGroomerSidecar: -# # enabled: false - -# ################################### -# ## COMPONENT | Airflow Webserver -# ################################### -# # webserver: -# # enabled: true - -# # Airflow Triggerer Config -# # triggerer: -# # enabled: true -# # logGroomerSidecar: -# # enabled: false - -# # Airflow Dag Processor Config -# # dagProcessor: -# # enabled: false - -# # Flower settings -# flower: -# # Enable flower. -# # If True, and using CeleryExecutor/CeleryKubernetesExecutor, will deploy flower app. -# enabled: false - -# # StatsD settings -# statsd: -# enabled: true - -# # PgBouncer settings -# # CloudNativePG provides native support for connection pooling with PgBouncer -# pgbouncer: -# # Enable PgBouncer -# enabled: false - -# # Configuration for the redis provisioned by the chart -# # redis: -# # enabled: true -# # terminationGracePeriodSeconds: 600 - -# # Configuration for postgresql subchart -# # Not recommended for production -# postgresql: -# enabled: false - -# # Config settings to go into the mounted airflow.cfg -# # -# # Please note that these values are passed through the `tpl` function, so are -# # all subject to being rendered as go templates. If you need to include a -# # literal `{{` in a value, it must be expressed like this: -# # -# # a: '{{ "{{ not a template }}" }}' -# # -# # Do not set config containing secrets via plain text values, use Env Var or k8s secret object -# # yamllint disable rule:line-length -# config: -# webserver: -# expose_config: 'True' -# instance_name: OSDU -# enable_proxy_fix: 'True' -# # base_url: 'http://localhost/airflow' -# operators: -# default_owner: OSDU - -# # Pull DAGS from file share -# dags: -# persistence: -# enabled: false -# existingClaim: airflow-dags-pvc -# subPath: dags -# gitSync: -# enabled: true -# repo: 'https://github.com/azure/osdu-developer' -# branch: main -# wait: 40 -# subPath: dags - -# # Global Logs go to pvc file share. -# logs: -# persistence: -# enabled: true -# existingClaim: airflow-logs-pvc -# storageClassName: azurefile-csi-premium From a40960a0c936eb4ccedecb8513dacc1b712416f2 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 21:48:29 -0500 Subject: [PATCH 49/55] airflow adjustments. --- software/components/airflow/release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 20c084bb..895f9833 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -39,10 +39,6 @@ spec: # Airflow executor executor: KubernetesExecutor - enableBuiltInSecretEnvVars: - AIRFLOW__CORE__FERNET_KEY: true - AIRFLOW__WEBSERVER__SECRET_KEY: false - # Environment variables for all airflow containers env: - name: ENVIRONMENT @@ -67,6 +63,10 @@ spec: pgbouncer: enabled: false + # StatsD settings + statsd: + enabled: true + # Pull DAGS from file share and from a git repo dags: persistence: From c8eb5b8ae11c3e6b4c73ca588c219ee641cdf5dd Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 21:56:33 -0500 Subject: [PATCH 50/55] airflow adjustments. --- software/components/airflow/release.yaml | 33 +++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 895f9833..30b80a00 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -42,7 +42,38 @@ spec: # Environment variables for all airflow containers env: - name: ENVIRONMENT - value: dev + value: "dev" + - name: CLOUD_PROVIDER + value: "azure" + - name: CI_COMMIT_TAG + value: "v0.12.0" + - name: BUILD_TAG + value: "v0.12.0" + - name: AIRFLOW_VAR_ENTITLEMENTS_MODULE_NAME + value: "entitlements_client" + + extraEnv: | + - name: AIRFLOW_VAR_ANOTHER_KEY + value: 'value_1' + + extraConfigMaps: + 'airflow-variables': + data: | + AIRFLOW_VAR_ENV_VARS_ENABLED: "True" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" + AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" + AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" + AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" + AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" + AIRFLOW_VAR_NAMESPACE: "airflow" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + + extraEnvFrom: | + - configMapRef: + name: "airflow-variables" + - secretRef: + name: 'airflow-variables' # Admin user configuration admin: From 21f4c8bd6130adae1b5a35256ffc03e66d956055 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 22:07:39 -0500 Subject: [PATCH 51/55] airflow adjustments. --- software/components/airflow/release.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 30b80a00..d977088a 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -69,6 +69,20 @@ spec: AIRFLOW_VAR_NAMESPACE: "airflow" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + extraConfigMaps: + 'airflow-service-endpoints': + data: | + AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" + AIRFLOW_VAR_CORE__SERVICE__LEGAL__HOST: "http://legal.osdu-core.svc.cluster.local/api/legal/v1" + AIRFLOW_VAR_CORE__SERVICE__ENTITLEMENTS__URL: "http://entitlements.osdu-core.svc.cluster.local/api/entitlements/v2" + AIRFLOW_VAR_CORE__SERVICE__SCHEMA__URL: "http://schema.osdu-core.svc.cluster.local/api/schema-service/v1" + AIRFLOW_VAR_CORE__SERVICE__SEARCH__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2" + AIRFLOW_VAR_CORE__SERVICE__SEARCH_WITH_CURSOR__URL: "http://search.osdu-core.svc.cluster.local/api/search/v2/query_with_cursor" + AIRFLOW_VAR_CORE__SERVICE__STORAGE__URL: "http://storage.osdu-core.svc.cluster.local/api/storage/v2" + AIRFLOW_VAR_CORE__SERVICE__FILE__HOST: "http://file.osdu-core.svc.cluster.local/api/file" + AIRFLOW_VAR_CORE__SERVICE__WORKFLOW__HOST: "http://workflow.osdu-core.svc.cluster.local/api/workflow/v1" + AIRFLOW_VAR_CORE__SERVICE__DATASET__HOST: "http://dataset.osdu-core.svc.cluster.local/api/dataset/v1" + extraEnvFrom: | - configMapRef: name: "airflow-variables" From 6c5eecee5b4484398788773dd90a05472af12a17 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 22:11:16 -0500 Subject: [PATCH 52/55] airflow adjustments. --- software/components/airflow/release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index d977088a..856b0d59 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -69,7 +69,6 @@ spec: AIRFLOW_VAR_NAMESPACE: "airflow" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - extraConfigMaps: 'airflow-service-endpoints': data: | AIRFLOW_VAR_CORE__SERVICE__PARTITION__URL: "http://partition.osdu-core.svc.cluster.local/api/partition/v1" From 5ce59db468ad97f3e9bf0e0c5f55468285167ce6 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 22:16:32 -0500 Subject: [PATCH 53/55] airflow adjustments. --- software/components/airflow/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 856b0d59..c311bbd5 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -85,6 +85,8 @@ spec: extraEnvFrom: | - configMapRef: name: "airflow-variables" + - configMapRef: + name: "airflow-service-endpoints" - secretRef: name: 'airflow-variables' From 3704a4ca1bfa7ae42c647490e2ddb2a8d1c923c1 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 22:35:11 -0500 Subject: [PATCH 54/55] airflow adjustments. --- software/components/airflow/release.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index c311bbd5..f88e6cda 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -33,8 +33,8 @@ spec: values: # Version Isolation - airflowVersion: 2.10.1 - defaultAirflowTag: 2.10.1 + # airflowVersion: 2.10.1 + # defaultAirflowTag: 2.10.1 # Airflow executor executor: KubernetesExecutor @@ -61,13 +61,11 @@ spec: data: | AIRFLOW_VAR_ENV_VARS_ENABLED: "True" AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False" - AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" - AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True" - AIRFLOW__CORE__PLUGINS_FOLDER: "/opt/airflow/plugins" - AIRFLOW__CORE__LOGGING_LEVEL: "INFO" + AIRFLOW__LOGGING__LOGGING_LEVEL: "INFO" AIRFLOW__LOGGING__ENABLE_TASK_CONTEXT_LOGGER: "False" AIRFLOW_VAR_NAMESPACE: "airflow" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" + AIRFLOW__METRICS__USE_PATTERN_MATCH: "True" 'airflow-service-endpoints': data: | From 7930d32999e40ec8795fafa9d603dae36601d777 Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Tue, 10 Sep 2024 22:54:31 -0500 Subject: [PATCH 55/55] airflow adjustments. --- software/components/airflow/release.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index f88e6cda..78d1720f 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -185,10 +185,3 @@ spec: - effect: NoSchedule key: app value: "cluster" - - - - - - -