From cd77a0dccebc608b45cfcfa481b5fea22af5f478 Mon Sep 17 00:00:00 2001 From: David Koenitzer Date: Tue, 13 Feb 2024 13:10:40 -0500 Subject: [PATCH] fix deployment inspect and list commands (#1544) * fix deployment inspect and list commands * fix lint * fix test * fix test * update test * fix lint --- cloud/deployment/deployment.go | 16 +++- cloud/deployment/fromfile/fromfile.go | 18 ++-- cloud/deployment/fromfile/fromfile_test.go | 102 +++++++++++++++++++++ cloud/deployment/inspect/inspect.go | 16 ++-- cloud/deployment/inspect/inspect_test.go | 7 +- 5 files changed, 133 insertions(+), 26 deletions(-) diff --git a/cloud/deployment/deployment.go b/cloud/deployment/deployment.go index 788f57572..07cd8659b 100644 --- a/cloud/deployment/deployment.go +++ b/cloud/deployment/deployment.go @@ -514,11 +514,14 @@ func deploymentToTableRow(table *printutil.Table, d *astroplatformcore.Deploymen cloudProvider := notApplicable clusterName := notApplicable region := notApplicable - releaseName := d.Namespace - if IsDeploymentStandard(*d.Type) { + if d.CloudProvider != nil { cloudProvider = *d.CloudProvider + } + if d.Region != nil { region = *d.Region - } else { + } + releaseName := d.Namespace + if !IsDeploymentStandard(*d.Type) { clusterName = *d.ClusterName } cols := []string{ @@ -1113,10 +1116,13 @@ func Update(deploymentID, name, ws, description, deploymentName, dagDeploy, exec clusterName := notApplicable cloudProvider := notApplicable region := notApplicable - if IsDeploymentStandard(*d.Type) { + if d.CloudProvider != nil { cloudProvider = *d.CloudProvider + } + if d.Region != nil { region = *d.Region - } else { + } + if !IsDeploymentStandard(*d.Type) { clusterName = *d.ClusterName } tabDeployment.AddRow([]string{d.Name, releaseName, clusterName, cloudProvider, region, d.Id, runtimeVersionText, strconv.FormatBool(d.IsDagDeployEnabled), strconv.FormatBool(d.IsCicdEnforced), string(*d.Type)}, false) diff --git a/cloud/deployment/fromfile/fromfile.go b/cloud/deployment/fromfile/fromfile.go index 39b485d1f..8c65adefa 100644 --- a/cloud/deployment/fromfile/fromfile.go +++ b/cloud/deployment/fromfile/fromfile.go @@ -456,10 +456,10 @@ func createOrUpdateDeployment(deploymentFromFile *inspect.FormattedDeployment, c } if deployment.IsDeploymentStandard(deploymentType) { var requestedExecutor astroplatformcore.UpdateStandardDeploymentRequestExecutor - switch deploymentFromFile.Deployment.Configuration.Executor { - case deployment.CeleryExecutor, deployment.CELERY: + switch strings.ToUpper(deploymentFromFile.Deployment.Configuration.Executor) { + case strings.ToUpper(deployment.CeleryExecutor), deployment.CELERY: requestedExecutor = astroplatformcore.UpdateStandardDeploymentRequestExecutorCELERY - case deployment.KubeExecutor, deployment.KUBERNETES: + case strings.ToUpper(deployment.KubeExecutor), deployment.KUBERNETES: requestedExecutor = astroplatformcore.UpdateStandardDeploymentRequestExecutorKUBERNETES } @@ -511,10 +511,10 @@ func createOrUpdateDeployment(deploymentFromFile *inspect.FormattedDeployment, c } if deployment.IsDeploymentDedicated(deploymentType) { var requestedExecutor astroplatformcore.UpdateDedicatedDeploymentRequestExecutor - switch deploymentFromFile.Deployment.Configuration.Executor { - case deployment.CeleryExecutor, deployment.CELERY: + switch strings.ToUpper(deploymentFromFile.Deployment.Configuration.Executor) { + case strings.ToUpper(deployment.CeleryExecutor), deployment.CELERY: requestedExecutor = astroplatformcore.UpdateDedicatedDeploymentRequestExecutorCELERY - case deployment.KubeExecutor, deployment.KUBERNETES: + case strings.ToUpper(deployment.KubeExecutor), deployment.KUBERNETES: requestedExecutor = astroplatformcore.UpdateDedicatedDeploymentRequestExecutorKUBERNETES } @@ -565,10 +565,10 @@ func createOrUpdateDeployment(deploymentFromFile *inspect.FormattedDeployment, c } if !deployment.IsDeploymentStandard(deploymentType) && !deployment.IsDeploymentDedicated(deploymentType) { var requestedExecutor astroplatformcore.UpdateHybridDeploymentRequestExecutor - switch deploymentFromFile.Deployment.Configuration.Executor { - case deployment.CeleryExecutor, deployment.CELERY: + switch strings.ToUpper(deploymentFromFile.Deployment.Configuration.Executor) { + case strings.ToUpper(deployment.CeleryExecutor), deployment.CELERY: requestedExecutor = astroplatformcore.UpdateHybridDeploymentRequestExecutorCELERY - case deployment.KubeExecutor, deployment.KUBERNETES: + case strings.ToUpper(deployment.KubeExecutor), deployment.KUBERNETES: requestedExecutor = astroplatformcore.UpdateHybridDeploymentRequestExecutorKUBERNETES } diff --git a/cloud/deployment/fromfile/fromfile_test.go b/cloud/deployment/fromfile/fromfile_test.go index a07913166..f293ebeef 100644 --- a/cloud/deployment/fromfile/fromfile_test.go +++ b/cloud/deployment/fromfile/fromfile_test.go @@ -2811,6 +2811,108 @@ func TestGetCreateOrUpdateInput(t *testing.T) { mockPlatformCoreClient.AssertExpectations(t) mockCoreClient.AssertExpectations(t) }) + t.Run("transforms formattedDeployment to UpdateDeploymentInput if Kubernetes executor was requested with a queue on standard", func(t *testing.T) { + deploymentID = "test-deployment-id" + deploymentFromFile = inspect.FormattedDeployment{} + deploymentFromFile.Deployment.Configuration.ClusterName = "test-cluster" + deploymentFromFile.Deployment.Configuration.Name = "test-deployment-modified" + deploymentFromFile.Deployment.Configuration.Description = "test-description" + deploymentFromFile.Deployment.Configuration.RunTimeVersion = "test-runtime-v" + deploymentFromFile.Deployment.Configuration.SchedulerAU = 4 + deploymentFromFile.Deployment.Configuration.SchedulerCount = 2 + deploymentFromFile.Deployment.Configuration.Executor = deployment.KubeExecutor + deploymentFromFile.Deployment.Configuration.DeploymentType = "STANDARD" + dagDeploy := true + deploymentFromFile.Deployment.Configuration.DagDeployEnabled = &dagDeploy + deploymentFromFile.Deployment.Configuration.DefaultWorkerType = "test-worker-1" + existingPools = []astroplatformcore.NodePool{ + { + Id: "test-pool-id", + IsDefault: false, + NodeInstanceType: "test-worker-1", + }, + { + Id: "test-pool-id-2", + IsDefault: false, + NodeInstanceType: "test-worker-2", + }, + } + clusterID := "test-cluster-id" + clusterName := "test-cluster" + defaultTaskPodMemory = "10" + defaultTaskPodCPU := "10" + resourceQuotaCPU := "10" + resourceQuotaMemory := "10" + existingDeployment := astroplatformcore.Deployment{ + Id: deploymentID, + Name: "test-deployment", + ClusterId: &clusterID, + ClusterName: &clusterName, + Executor: &executorCelery, + WorkerQueues: &expectedQList, + DefaultTaskPodMemory: &defaultTaskPodMemory, + DefaultTaskPodCpu: &defaultTaskPodCPU, + ResourceQuotaCpu: &resourceQuotaCPU, + ResourceQuotaMemory: &resourceQuotaMemory, + } + + mockPlatformCoreClient.On("UpdateDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&mockUpdateDeploymentResponse, nil).Times(1) + err = createOrUpdateDeployment(&deploymentFromFile, clusterID, workspaceID, "update", &existingDeployment, existingPools, dagDeploy, []astroplatformcore.DeploymentEnvironmentVariableRequest{}, mockCoreClient, mockPlatformCoreClient) + assert.NoError(t, err) + mockPlatformCoreClient.AssertExpectations(t) + mockCoreClient.AssertExpectations(t) + }) + t.Run("transforms formattedDeployment to UpdateDeploymentInput if Kubernetes executor was requested with a queue on dedicated", func(t *testing.T) { + deploymentID = "test-deployment-id" + deploymentFromFile = inspect.FormattedDeployment{} + deploymentFromFile.Deployment.Configuration.ClusterName = "test-cluster" + deploymentFromFile.Deployment.Configuration.Name = "test-deployment-modified" + deploymentFromFile.Deployment.Configuration.Description = "test-description" + deploymentFromFile.Deployment.Configuration.RunTimeVersion = "test-runtime-v" + deploymentFromFile.Deployment.Configuration.SchedulerAU = 4 + deploymentFromFile.Deployment.Configuration.SchedulerCount = 2 + deploymentFromFile.Deployment.Configuration.Executor = deployment.KubeExecutor + deploymentFromFile.Deployment.Configuration.DeploymentType = "DEDICATED" + dagDeploy := true + deploymentFromFile.Deployment.Configuration.DagDeployEnabled = &dagDeploy + deploymentFromFile.Deployment.Configuration.DefaultWorkerType = "test-worker-1" + existingPools = []astroplatformcore.NodePool{ + { + Id: "test-pool-id", + IsDefault: false, + NodeInstanceType: "test-worker-1", + }, + { + Id: "test-pool-id-2", + IsDefault: false, + NodeInstanceType: "test-worker-2", + }, + } + clusterID := "test-cluster-id" + clusterName := "test-cluster" + defaultTaskPodMemory = "10" + defaultTaskPodCPU := "10" + resourceQuotaCPU := "10" + resourceQuotaMemory := "10" + existingDeployment := astroplatformcore.Deployment{ + Id: deploymentID, + Name: "test-deployment", + ClusterId: &clusterID, + ClusterName: &clusterName, + Executor: &executorCelery, + WorkerQueues: &expectedQList, + DefaultTaskPodMemory: &defaultTaskPodMemory, + DefaultTaskPodCpu: &defaultTaskPodCPU, + ResourceQuotaCpu: &resourceQuotaCPU, + ResourceQuotaMemory: &resourceQuotaMemory, + } + + mockPlatformCoreClient.On("UpdateDeploymentWithResponse", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&mockUpdateDeploymentResponse, nil).Times(1) + err = createOrUpdateDeployment(&deploymentFromFile, clusterID, workspaceID, "update", &existingDeployment, existingPools, dagDeploy, []astroplatformcore.DeploymentEnvironmentVariableRequest{}, mockCoreClient, mockPlatformCoreClient) + assert.NoError(t, err) + mockPlatformCoreClient.AssertExpectations(t) + mockCoreClient.AssertExpectations(t) + }) t.Run("returns correct update deployment input when multiple queues are requested", func(t *testing.T) { deploymentID = "test-deployment-id" deploymentFromFile = inspect.FormattedDeployment{} diff --git a/cloud/deployment/inspect/inspect.go b/cloud/deployment/inspect/inspect.go index ef64be3df..6bfeabebc 100644 --- a/cloud/deployment/inspect/inspect.go +++ b/cloud/deployment/inspect/inspect.go @@ -38,12 +38,12 @@ type deploymentConfig struct { RunTimeVersion string `mapstructure:"runtime_version" yaml:"runtime_version" json:"runtime_version"` DagDeployEnabled *bool `mapstructure:"dag_deploy_enabled,omitempty" yaml:"dag_deploy_enabled,omitempty" json:"dag_deploy_enabled,omitempty"` APIKeyOnlyDeployments bool `mapstructure:"ci_cd_enforcement" yaml:"ci_cd_enforcement" json:"ci_cd_enforcement"` - SchedulerSize string `mapstructure:"scheduler_size" yaml:"scheduler_size" json:"scheduler_size"` + SchedulerSize string `mapstructure:"scheduler_size,omitempty" yaml:"scheduler_size,omitempty" json:"scheduler_size,omitempty"` IsHighAvailability bool `mapstructure:"is_high_availability" yaml:"is_high_availability" json:"is_high_availability"` Executor string `mapstructure:"executor" yaml:"executor" json:"executor"` - SchedulerAU int `mapstructure:"scheduler_au" yaml:"scheduler_au" json:"scheduler_au"` + SchedulerAU int `mapstructure:"scheduler_au,omitempty" yaml:"scheduler_au,omitempty" json:"scheduler_au,omitempty"` SchedulerCount int `mapstructure:"scheduler_count" yaml:"scheduler_count" json:"scheduler_count"` - ClusterName string `mapstructure:"cluster_name" yaml:"cluster_name" json:"cluster_name"` + ClusterName string `mapstructure:"cluster_name,omitempty" yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` WorkspaceName string `mapstructure:"workspace_name" yaml:"workspace_name" json:"workspace_name"` DeploymentType string `mapstructure:"deployment_type" yaml:"deployment_type" json:"deployment_type"` CloudProvider string `mapstructure:"cloud_provider" yaml:"cloud_provider" json:"cloud_provider"` @@ -224,15 +224,11 @@ func getDeploymentConfig(coreDeploymentPointer *astroplatformcore.Deployment, pl } if deployment.IsDeploymentStandard(*coreDeployment.Type) || deployment.IsDeploymentDedicated(*coreDeployment.Type) { deploymentMap["scheduler_size"] = *coreDeployment.SchedulerSize - deploymentMap["cloud_provider"] = *coreDeployment.CloudProvider deploymentMap["default_task_pod_cpu"] = *coreDeployment.DefaultTaskPodCpu deploymentMap["default_task_pod_memory"] = *coreDeployment.DefaultTaskPodMemory deploymentMap["resource_quota_cpu"] = *coreDeployment.ResourceQuotaCpu deploymentMap["resource_quota_memory"] = *coreDeployment.ResourceQuotaMemory } - if deployment.IsDeploymentStandard(*coreDeployment.Type) { - deploymentMap["region"] = *coreDeployment.Region - } if !deployment.IsDeploymentStandard(*coreDeployment.Type) { deploymentMap["default_worker_type"] = defaultWorkerType } @@ -246,6 +242,12 @@ func getDeploymentConfig(coreDeploymentPointer *astroplatformcore.Deployment, pl if coreDeployment.SchedulerAu != nil { deploymentMap["scheduler_au"] = *coreDeployment.SchedulerAu } + if coreDeployment.CloudProvider != nil { + deploymentMap["cloud_provider"] = *coreDeployment.CloudProvider + } + if coreDeployment.Region != nil { + deploymentMap["region"] = *coreDeployment.Region + } return deploymentMap, nil } diff --git a/cloud/deployment/inspect/inspect_test.go b/cloud/deployment/inspect/inspect_test.go index 684424ec1..b0243ff11 100644 --- a/cloud/deployment/inspect/inspect_test.go +++ b/cloud/deployment/inspect/inspect_test.go @@ -447,9 +447,9 @@ func TestGetDeploymentConfig(t *testing.T) { SchedulerCount: sourceDeployment.SchedulerReplicas, DagDeployEnabled: &sourceDeployment.IsDagDeployEnabled, Executor: string(*sourceDeployment.Executor), - Region: "", + Region: *sourceDeployment.Region, DeploymentType: string(*sourceDeployment.Type), - CloudProvider: "", + CloudProvider: *sourceDeployment.CloudProvider, } rawDeploymentConfig, err := getDeploymentConfig(&sourceDeployment, mockPlatformCoreClient) assert.NoError(t, err) @@ -707,7 +707,6 @@ func TestFormatPrintableDeployment(t *testing.T) { runtime_version: 6.0.0 dag_deploy_enabled: true ci_cd_enforcement: false - scheduler_size: "" is_high_availability: false executor: CELERY scheduler_au: 5 @@ -796,7 +795,6 @@ func TestFormatPrintableDeployment(t *testing.T) { executor: CELERY scheduler_au: 5 scheduler_count: 3 - cluster_name: "" workspace_name: test-ws deployment_type: STANDARD cloud_provider: azure @@ -981,7 +979,6 @@ func TestFormatPrintableDeployment(t *testing.T) { "runtime_version": "6.0.0", "dag_deploy_enabled": true, "ci_cd_enforcement": false, - "scheduler_size": "", "is_high_availability": false, "executor": "KUBERNETES", "scheduler_au": 5,