Skip to content

Commit

Permalink
Add New Deployment Selection Process for Cloud (#584)
Browse files Browse the repository at this point in the history 
* deployment api key work with deployment cmds

* implement new deployment selection process

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove workspace check

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix tests

* make API keys work with deploy

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix tests and setup

* Apply suggestions from code review

Co-authored-by: Neel Dalsania <[email protected]>

* add tests

* add test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix copy

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Neel Dalsania <[email protected]>
  • Loading branch information
@sunkickr @pre-commit-ci @neel-astro
3 people committed Jun 10, 2022
1 parent 115c7cc commit 50b270b
Show file tree
Hide file tree
Showing 10 changed files with 220 additions and 277 deletions.
1 change: 1 addition & 0 deletions astro-client/queries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ var (
}
}
workspace {
id
organizationId
}
}
Expand Down
122 changes: 12 additions & 110 deletions cloud/deploy/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,19 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"

"github.com/astronomer/astro-cli/airflow"
"github.com/astronomer/astro-cli/airflow/types"
airflowversions "github.com/astronomer/astro-cli/airflow_versions"
astro "github.com/astronomer/astro-cli/astro-client"
"github.com/astronomer/astro-cli/cloud/deployment"
"github.com/astronomer/astro-cli/config"
"github.com/astronomer/astro-cli/docker"
"github.com/astronomer/astro-cli/pkg/ansi"
"github.com/astronomer/astro-cli/pkg/httputil"
"github.com/astronomer/astro-cli/pkg/input"
"github.com/astronomer/astro-cli/pkg/printutil"
"github.com/astronomer/astro-cli/pkg/util"
"github.com/docker/docker/api/types/versions"
"github.com/pkg/errors"
Expand All @@ -33,12 +31,8 @@ const (
defaultRuntimeVersion = "4.2.5"
dagParseAllowedVersion = "4.1.0"

noWorkspaceMsg = "no workspaces with id (%s) found"

composeImageBuildingPromptMsg = "Building image..."
deployingPromptMsg = "Deploying: %s\n"
deploymentHeaderMsg = "Authenticated to %s \n\n"
selectDeploymentPromptMsg = "Select which Deployment you want to deploy to:"

warningInvaildImageNameMsg = "WARNING! The image in your Dockerfile '%s' is not based on Astro Runtime and is not supported. Change your Dockerfile with an image that pulls from 'quay.io/astronomer/astro-runtime' to proceed.\n"
warningInvalidImageTagMsg = "WARNING! You are about to push an image using the '%s' runtime tag. This is not supported.\nPlease use one of the following supported tags: %s"
Expand All @@ -47,24 +41,14 @@ const (
var (
splitNum = 2
pytestFile string
tab = printutil.Table{
Padding: []int{5, 30, 30, 50},
DynamicPadding: true,
Header: []string{"#", "DEPLOYMENT NAME", "RELEASE NAME", "DEPLOYMENT ID"},
}

dockerfile = "Dockerfile"

// Monkey patched to write unit tests
airflowImageHandler = airflow.ImageHandlerInit
containerHandlerInit = airflow.ContainerHandlerInit
)

var (
errInvalidDeploymentKey = errors.New("invalid deployment selection")
errDagsParseFailed = errors.New("your local DAGs did not parse. Please fix the listed errors or use `astro deploy [deployment-id] -f` to force deploy") //nolint:revive
errNoDeploymentsMsg = errors.New("no Deployments found in this Workspace")
)
var errDagsParseFailed = errors.New("your local DAGs did not parse. Fix the listed errors or use `astro deploy [deployment-id] -f` to force deploy") //nolint:revive

type deploymentInfo struct {
deploymentID string
Expand Down Expand Up @@ -165,28 +149,19 @@ func getDeploymentInfo(deploymentID, wsID string, prompt bool, cloudDomain strin

// check if deploymentID or if force prompt was requested was given by user
if deploymentID == "" || prompt {
// Validate workspace
currentWorkspace, err := validateWorkspace(wsID, client)
if err != nil {
return deploymentInfo{}, err
}

deploymentsInput := astro.DeploymentsInput{
WorkspaceID: currentWorkspace.ID,
}

deployments, err := client.ListDeployments(deploymentsInput)
currentDeployment, err := deployment.GetDeployment(wsID, deploymentID, client)
if err != nil {
return deploymentInfo{}, err
}

// Prompt user for deployment if no deployment passed in
deployInfo, err := promptUserForDeployment(cloudDomain, &currentWorkspace, deployments)
if err != nil {
return deploymentInfo{}, err
}
deployInfo.organizationID = currentWorkspace.OrganizationID
return deployInfo, nil
return deploymentInfo{
currentDeployment.ID,
currentDeployment.ReleaseName,
airflow.ImageName(currentDeployment.ReleaseName, "latest"),
currentDeployment.RuntimeRelease.Version,
currentDeployment.Workspace.OrganizationID,
currentDeployment.DeploymentSpec.Webserver.URL,
}, nil
}
deployInfo, err := getImageName(cloudDomain, deploymentID, client)
if err != nil {
Expand Down Expand Up @@ -236,7 +211,7 @@ func checkPytest(pytest, deployImage string, containerHandler airflow.ContainerH
exitCode, err := containerHandler.Pytest(pytestFile, deployImage)
if err != nil {
if strings.Contains(exitCode, "1") { // exit code is 1 meaning tests failed
return errors.New("pytests failed, please fix failures or rerun the command without the '--pytest' flag to deploy")
return errors.New("at least 1 pytest in your tests directory failed. Fix the issues listed or rerun the command without the '--pytest' flag to deploy")
}
return errors.Wrap(err, "Something went wrong while Pytesting your local DAGs,\nif the issue persists rerun the command without the '--pytest' flag to deploy")
}
Expand All @@ -245,79 +220,6 @@ func checkPytest(pytest, deployImage string, containerHandler airflow.ContainerH
return err
}

// Validate workspace
func validateWorkspace(wsID string, client astro.Client) (astro.Workspace, error) {
if wsID == "" {
return astro.Workspace{}, errors.New("no workspace id provided")
}

wsResp, err := client.ListWorkspaces()
if err != nil {
return astro.Workspace{}, errors.Wrap(err, astro.AstronomerConnectionErrMsg)
}

var currentWorkspace astro.Workspace
for i := range wsResp {
if wsResp[i].ID == wsID {
currentWorkspace = wsResp[i]
break
}
}

if currentWorkspace.ID == "" {
err = fmt.Errorf(noWorkspaceMsg, wsID) // nolint:goerr113
return astro.Workspace{}, err
}

return currentWorkspace, nil
}

// Prompt user for deployment if no deployment passed in
func promptUserForDeployment(cloudDomain string, currentWorkspace *astro.Workspace, deployments []astro.Deployment) (deploymentInfo, error) {
if len(deployments) == 0 {
return deploymentInfo{}, errNoDeploymentsMsg
}

if cloudDomain == astroDomain {
fmt.Printf(deploymentHeaderMsg, "Astro")
} else {
fmt.Printf(deploymentHeaderMsg, cloudDomain)
}

fmt.Printf("Current Workspace: %s\n\n", currentWorkspace.Label)
fmt.Println(selectDeploymentPromptMsg)

sort.Slice(deployments, func(i, j int) bool {
return deployments[i].CreatedAt.Before(deployments[j].CreatedAt)
})

deployMap := map[string]astro.Deployment{}
for i := range deployments {
index := i + 1
tab.AddRow([]string{strconv.Itoa(index), deployments[i].Label, deployments[i].ReleaseName, deployments[i].ID}, false)

deployMap[strconv.Itoa(index)] = deployments[i]
}

tab.Print(os.Stdout)
choice := input.Text("\n> ")
selected, ok := deployMap[choice]
if !ok {
return deploymentInfo{}, errInvalidDeploymentKey
}
deploymentID := selected.ID
currentVersion := selected.RuntimeRelease.Version
namespace := selected.ReleaseName
webserverURL := selected.DeploymentSpec.Webserver.URL

fmt.Printf(deployingPromptMsg, namespace)

// We use latest and keep this tag around after deployments to keep subsequent deploys quick
deployImage := airflow.ImageName(namespace, "latest")

return deploymentInfo{deploymentID: deploymentID, namespace: namespace, deployImage: deployImage, currentVersion: currentVersion, webserverURL: webserverURL}, nil
}

func getImageName(cloudDomain, deploymentID string, client astro.Client) (deploymentInfo, error) {
if cloudDomain == astroDomain {
fmt.Printf(deploymentHeaderMsg, "Astro")
Expand Down
32 changes: 1 addition & 31 deletions cloud/deploy/deploy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ func TestDeploySuccess(t *testing.T) {
testUtil.InitTestConfig(testUtil.CloudPlatform)
config.CFG.ShowWarnings.SetHomeString("false")
mockClient := new(astro_mocks.Client)
mockClient.On("ListWorkspaces").Return([]astro.Workspace{{ID: "test-ws-id"}}, nil).Once()
mockClient.On("ListDeployments", mock.Anything).Return(mockDeplyResp, nil).Twice()
mockClient.On("ListPublicRuntimeReleases").Return([]astro.RuntimeRelease{{Version: "4.2.5", AirflowVersion: "2.2.5"}}, nil).Twice()
mockClient.On("CreateImage", mock.Anything).Return(&astro.Image{}, nil).Twice()
Expand Down Expand Up @@ -107,11 +106,6 @@ func TestDeployFailure(t *testing.T) {
err = Deploy("./testfiles/", "test-id", "test-ws-id", "pytest", "", true, nil)
assert.EqualError(t, err, "no context set, have you authenticated to Astro or Astronomer Software? Run astro login and try again")

// no workspace id provided
testUtil.InitTestConfig(testUtil.CloudPlatform)
err = Deploy("./testfiles/", "", "", "parse", "", true, nil)
assert.EqualError(t, err, "no workspace id provided")

// airflow parse failure
mockDeplyResp := []astro.Deployment{
{
Expand All @@ -125,7 +119,6 @@ func TestDeployFailure(t *testing.T) {
}
testUtil.InitTestConfig(testUtil.CloudPlatform)
mockClient := new(astro_mocks.Client)
mockClient.On("ListWorkspaces").Return([]astro.Workspace{{ID: "test-ws-id"}}, nil).Once()
mockClient.On("ListDeployments", mock.Anything).Return(mockDeplyResp, nil).Once()
mockClient.On("ListPublicRuntimeReleases").Return([]astro.RuntimeRelease{{Version: "4.2.5", AirflowVersion: "2.2.5"}}, nil).Once()

Expand Down Expand Up @@ -236,14 +229,6 @@ func TestCheckVersionBeta(t *testing.T) {
assert.Contains(t, buf.String(), "")
}

func TestPromptUserForDeployment(t *testing.T) {
_, err := promptUserForDeployment("", &astro.Workspace{}, []astro.Deployment{}) //nolint:dogsled
assert.ErrorIs(t, err, errNoDeploymentsMsg)

_, err = promptUserForDeployment("astrodev.io", &astro.Workspace{}, []astro.Deployment{{ID: "test-id-1"}, {ID: "test-id-2"}}) //nolint:dogsled
assert.ErrorIs(t, err, errInvalidDeploymentKey)
}

func TestCheckPyTest(t *testing.T) {
mockDeployImage := "test-image"

Expand All @@ -259,21 +244,6 @@ func TestCheckPyTest(t *testing.T) {
mockContainerHandler.On("Pytest", "", mockDeployImage).Return("exit code 1", errMock).Once()
err = checkPytest("", mockDeployImage, mockContainerHandler)
assert.Error(t, err)
assert.Contains(t, err.Error(), "pytests failed, please fix failures or rerun the command without the '--pytest' flag to deploy")
assert.Contains(t, err.Error(), "at least 1 pytest in your tests directory failed. Fix the issues listed or rerun the command without the '--pytest' flag to deploy")
mockContainerHandler.AssertExpectations(t)
}

func TestValidateWorkspace(t *testing.T) {
mockClient := new(astro_mocks.Client)
mockClient.On("ListWorkspaces").Return([]astro.Workspace{}, errMock).Once()

_, err := validateWorkspace("test-ws-id", mockClient)
assert.ErrorIs(t, err, errMock)

mockClient.On("ListWorkspaces").Return([]astro.Workspace{{ID: "test-ws-id"}}, nil).Once()
_, err = validateWorkspace("test-invalid-id", mockClient)
assert.Error(t, err)
assert.Contains(t, err.Error(), "no workspaces with id")

mockClient.AssertExpectations(t)
}
Loading

0 comments on commit 50b270b

Please sign in to comment.