From ed060bfc3d4ab1f86dfda88fe168d04286e0dcb0 Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Mon, 16 Dec 2024 14:08:29 -0800 Subject: [PATCH 1/7] Allow dynamic worker queue to be set --- internal/provider/resources/resource_deployment.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/provider/resources/resource_deployment.go b/internal/provider/resources/resource_deployment.go index 6b15e272..0c66925e 100644 --- a/internal/provider/resources/resource_deployment.go +++ b/internal/provider/resources/resource_deployment.go @@ -645,20 +645,6 @@ func (r *DeploymentResource) ValidateConfig( return } - // Need to do dynamic validation based on the executor and worker queues - if data.Executor.ValueString() == string(platform.DeploymentExecutorKUBERNETES) && len(data.WorkerQueues.Elements()) > 0 { - resp.Diagnostics.AddError( - "worker_queues are not supported for 'KUBERNETES' executor", - "Either change the executor to 'CELERY' or remove worker_queues", - ) - } - if data.Executor.ValueString() == string(platform.DeploymentExecutorCELERY) && (data.WorkerQueues.IsNull() || len(data.WorkerQueues.Elements()) == 0) { - resp.Diagnostics.AddError( - "worker_queues must be included for 'CELERY' executor", - "Either change the executor to 'KUBERNETES' or include worker_queues", - ) - } - // Type specific validation switch platform.DeploymentType(data.Type.ValueString()) { case platform.DeploymentTypeSTANDARD: From 53cca2a9d8f224957c6aca30a8b9a8b59b08c537 Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Mon, 16 Dec 2024 15:38:00 -0800 Subject: [PATCH 2/7] fix --- internal/provider/resources/resource_cluster.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/provider/resources/resource_cluster.go b/internal/provider/resources/resource_cluster.go index bed7884c..4912e868 100644 --- a/internal/provider/resources/resource_cluster.go +++ b/internal/provider/resources/resource_cluster.go @@ -212,8 +212,8 @@ func (r *ClusterResource) Create( // Wait for the cluster to be created (or fail) stateConf := &retry.StateChangeConf{ - Pending: []string{string(platform.ClusterStatusCREATING), string(platform.ClusterStatusUPDATING)}, - Target: []string{string(platform.ClusterStatusCREATED), string(platform.ClusterStatusUPDATEFAILED), string(platform.ClusterStatusCREATEFAILED)}, + Pending: []string{string(platform.ClusterStatusCREATING), string(platform.ClusterStatusUPDATING), string(platform.ClusterStatusUPGRADEPENDING)}, + Target: []string{string(platform.ClusterStatusCREATED), string(platform.ClusterStatusUPDATEFAILED), string(platform.ClusterStatusCREATEFAILED), string(platform.ClusterStatusACCESSDENIED)}, Refresh: ClusterResourceRefreshFunc(ctx, r.platformClient, r.organizationId, cluster.JSON200.Id), Timeout: 3 * time.Hour, MinTimeout: 1 * time.Minute, @@ -364,8 +364,8 @@ func (r *ClusterResource) Update( // Wait for the cluster to be updated (or fail) stateConf := &retry.StateChangeConf{ - Pending: []string{string(platform.ClusterStatusCREATING), string(platform.ClusterStatusUPDATING)}, - Target: []string{string(platform.ClusterStatusCREATED), string(platform.ClusterStatusUPDATEFAILED), string(platform.ClusterStatusCREATEFAILED)}, + Pending: []string{string(platform.ClusterStatusCREATING), string(platform.ClusterStatusUPDATING), string(platform.ClusterStatusUPGRADEPENDING)}, + Target: []string{string(platform.ClusterStatusCREATED), string(platform.ClusterStatusUPDATEFAILED), string(platform.ClusterStatusCREATEFAILED), string(platform.ClusterStatusACCESSDENIED)}, Refresh: ClusterResourceRefreshFunc(ctx, r.platformClient, r.organizationId, cluster.JSON200.Id), Timeout: 3 * time.Hour, MinTimeout: 1 * time.Minute, @@ -435,7 +435,7 @@ func (r *ClusterResource) Delete( // Wait for the cluster to be deleted stateConf := &retry.StateChangeConf{ - Pending: []string{string(platform.ClusterStatusCREATING), string(platform.ClusterStatusUPDATING), string(platform.ClusterStatusCREATED), string(platform.ClusterStatusUPDATEFAILED), string(platform.ClusterStatusCREATEFAILED)}, + Pending: []string{string(platform.ClusterStatusCREATING), string(platform.ClusterStatusUPDATING), string(platform.ClusterStatusCREATED), string(platform.ClusterStatusUPDATEFAILED), string(platform.ClusterStatusCREATEFAILED), string(platform.ClusterStatusUPGRADEPENDING)}, Target: []string{"DELETED"}, Refresh: ClusterResourceRefreshFunc(ctx, r.platformClient, r.organizationId, data.Id.ValueString()), Timeout: 1 * time.Hour, From e01415f80bf349939cb11a6a668b599ee192071e Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Mon, 16 Dec 2024 20:27:29 -0800 Subject: [PATCH 3/7] add test --- .../resources/resource_deployment_test.go | 68 ++++++++++++++++--- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/internal/provider/resources/resource_deployment_test.go b/internal/provider/resources/resource_deployment_test.go index f824c172..5d8a64ce 100644 --- a/internal/provider/resources/resource_deployment_test.go +++ b/internal/provider/resources/resource_deployment_test.go @@ -155,7 +155,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "CELERY", SchedulerSize: string(platform.SchedulerMachineNameEXTRALARGE), IncludeEnvironmentVariables: false, - DuplicateWorkerQueues: true, + WorkerQueuesStr: workerQueuesDuplicateStr(""), }), ExpectError: regexp.MustCompile(`worker_queue names must be unique`), }, @@ -168,6 +168,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "KUBERNETES", SchedulerSize: string(platform.SchedulerMachineNameSMALL), IncludeEnvironmentVariables: true, + WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(awsResourceVar, "name", awsDeploymentName), @@ -193,6 +194,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "CELERY", SchedulerSize: string(platform.SchedulerMachineNameEXTRALARGE), IncludeEnvironmentVariables: false, + WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(awsResourceVar, "description", utils.TestResourceDescription), @@ -204,6 +206,56 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { testAccCheckDeploymentExistence(t, awsDeploymentName, true, true), ), }, + // Change worker queues to depend on a variable + { + Config: ` + variable "env" { + type = string + default = "dev" + } + + locals { + worker_queue_config = { + dev = [ + { + name = "default" + is_default = true + astro_machine = "A5" + max_worker_count = 10 + min_worker_count = 0 + worker_concurrency = 5 + } + ] + default = [ + { + name = "default" + is_default = false + astro_machine = "A10" + max_worker_count = 3 + min_worker_count = 1 + worker_concurrency = 10 + } + ] + } + } + ` + + astronomerprovider.ProviderConfig(t, astronomerprovider.HOSTED) + standardDeployment(standardDeploymentInput{ + Name: awsDeploymentName, + Description: utils.TestResourceDescription, + Region: "us-east-1", + CloudProvider: "AWS", + Executor: "CELERY", + SchedulerSize: string(platform.SchedulerMachineNameMEDIUM), + IncludeEnvironmentVariables: false, + WorkerQueuesStr: `worker_queues = lookup(local.worker_queue_config, var.env, local.worker_queue_config["default"])` + }), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(awsResourceVar, "executor", "CELERY"), + resource.TestCheckResourceAttr(awsResourceVar, "worker_queues.0.name", "default"), + // Check via API that deployment exists + testAccCheckDeploymentExistence(t, awsDeploymentName, true, true), + ), + }, // Change executor back to KUBERNETES and check it is correctly updated in terraform state { Config: astronomerprovider.ProviderConfig(t, astronomerprovider.HOSTED) + standardDeployment(standardDeploymentInput{ @@ -290,6 +342,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "CELERY", SchedulerSize: string(platform.SchedulerMachineNameSMALL), IncludeEnvironmentVariables: true, + WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(azureCeleryResourceVar, "name", azureCeleryDeploymentName), @@ -724,6 +777,7 @@ func developmentDeployment(scalingSpecDeploymentName, scalingSpec string) string SchedulerSize: string(platform.SchedulerMachineNameSMALL), IsDevelopmentMode: true, ScalingSpec: scalingSpec, + WorkerQueuesStr: workerQueuesStr(""), }) } @@ -737,18 +791,10 @@ type standardDeploymentInput struct { SchedulerSize string IsDevelopmentMode bool ScalingSpec string - DuplicateWorkerQueues bool + WorkerQueuesStr string } func standardDeployment(input standardDeploymentInput) string { - wqStr := "" - if input.Executor == string(platform.DeploymentExecutorCELERY) { - if input.DuplicateWorkerQueues { - wqStr = workerQueuesDuplicateStr("") - } else { - wqStr = workerQueuesStr("") - } - } var scalingSpecStr string if input.IsDevelopmentMode { @@ -802,7 +848,7 @@ resource "astro_deployment" "%v" { } `, input.Name, input.Name, utils.TestResourceDescription, input.Name, input.Name, input.Description, input.Region, input.CloudProvider, input.Executor, input.IsDevelopmentMode, input.SchedulerSize, input.Name, - envVarsStr(input.IncludeEnvironmentVariables), wqStr, scalingSpecStr) + envVarsStr(input.IncludeEnvironmentVariables), input.WorkerQueuesStr, scalingSpecStr) } func standardDeploymentWithVariableName(input standardDeploymentInput) string { From 12d2079bdf12640a62d3cd91a16a9661696785d0 Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Mon, 16 Dec 2024 20:37:27 -0800 Subject: [PATCH 4/7] fix test --- .../provider/resources/resource_deployment_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/provider/resources/resource_deployment_test.go b/internal/provider/resources/resource_deployment_test.go index 5d8a64ce..2b2b8ce9 100644 --- a/internal/provider/resources/resource_deployment_test.go +++ b/internal/provider/resources/resource_deployment_test.go @@ -168,7 +168,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "KUBERNETES", SchedulerSize: string(platform.SchedulerMachineNameSMALL), IncludeEnvironmentVariables: true, - WorkerQueuesStr: workerQueuesStr(""), + WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(awsResourceVar, "name", awsDeploymentName), @@ -194,7 +194,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "CELERY", SchedulerSize: string(platform.SchedulerMachineNameEXTRALARGE), IncludeEnvironmentVariables: false, - WorkerQueuesStr: workerQueuesStr(""), + WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(awsResourceVar, "description", utils.TestResourceDescription), @@ -247,7 +247,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "CELERY", SchedulerSize: string(platform.SchedulerMachineNameMEDIUM), IncludeEnvironmentVariables: false, - WorkerQueuesStr: `worker_queues = lookup(local.worker_queue_config, var.env, local.worker_queue_config["default"])` + WorkerQueuesStr: `worker_queues = lookup(local.worker_queue_config, var.env, local.worker_queue_config["default"])`, }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(awsResourceVar, "executor", "CELERY"), @@ -342,7 +342,7 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "CELERY", SchedulerSize: string(platform.SchedulerMachineNameSMALL), IncludeEnvironmentVariables: true, - WorkerQueuesStr: workerQueuesStr(""), + WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(azureCeleryResourceVar, "name", azureCeleryDeploymentName), @@ -777,7 +777,7 @@ func developmentDeployment(scalingSpecDeploymentName, scalingSpec string) string SchedulerSize: string(platform.SchedulerMachineNameSMALL), IsDevelopmentMode: true, ScalingSpec: scalingSpec, - WorkerQueuesStr: workerQueuesStr(""), + WorkerQueuesStr: workerQueuesStr(""), }) } From 980f405e0ab5fffd31825c5a8fd4832828243104 Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Mon, 16 Dec 2024 23:22:59 -0800 Subject: [PATCH 5/7] fix test --- internal/provider/resources/resource_deployment_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/provider/resources/resource_deployment_test.go b/internal/provider/resources/resource_deployment_test.go index 2b2b8ce9..da71c606 100644 --- a/internal/provider/resources/resource_deployment_test.go +++ b/internal/provider/resources/resource_deployment_test.go @@ -168,7 +168,6 @@ func TestAcc_ResourceDeploymentStandard(t *testing.T) { Executor: "KUBERNETES", SchedulerSize: string(platform.SchedulerMachineNameSMALL), IncludeEnvironmentVariables: true, - WorkerQueuesStr: workerQueuesStr(""), }), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(awsResourceVar, "name", awsDeploymentName), From 2d937f356ff6cb1004ff74f0cc714c34e0376348 Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Tue, 17 Dec 2024 09:28:28 -0800 Subject: [PATCH 6/7] fix test --- internal/provider/resources/resource_team_roles_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/provider/resources/resource_team_roles_test.go b/internal/provider/resources/resource_team_roles_test.go index f6642163..3a265b8e 100644 --- a/internal/provider/resources/resource_team_roles_test.go +++ b/internal/provider/resources/resource_team_roles_test.go @@ -67,6 +67,7 @@ func TestAcc_ResourceTeamRoles(t *testing.T) { IncludeEnvironmentVariables: false, SchedulerSize: string(platform.DeploymentSchedulerSizeSMALL), IsDevelopmentMode: false, + WorkerQueuesStr: workerQueuesStr(""), }) + teamRoles(string(iam.ORGANIZATIONMEMBER), fmt.Sprintf(`[{workspace_id = %s From c9041b218ef34983d33b5c7e07b481064dd2024a Mon Sep 17 00:00:00 2001 From: Vandy Liu Date: Tue, 17 Dec 2024 20:27:40 -0800 Subject: [PATCH 7/7] fix test --- internal/provider/resources/resource_cluster_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/provider/resources/resource_cluster_test.go b/internal/provider/resources/resource_cluster_test.go index 52fb0a7b..bad18892 100644 --- a/internal/provider/resources/resource_cluster_test.go +++ b/internal/provider/resources/resource_cluster_test.go @@ -190,9 +190,10 @@ func TestAcc_ResourceClusterAwsWithDedicatedDeployments(t *testing.T) { }, // Import existing cluster and check it is correctly imported - https://stackoverflow.com/questions/68824711/how-can-i-test-terraform-import-in-acceptance-tests { - ResourceName: awsResourceVar, - ImportState: true, - ImportStateVerify: true, + ResourceName: awsResourceVar, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"health_status", "health_status.value"}, }, }, })