diff --git a/airflow-client/mocks/Client.go b/airflow-client/mocks/Client.go index f93ba9e76..d60aa5a63 100644 --- a/airflow-client/mocks/Client.go +++ b/airflow-client/mocks/Client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package airflow_mocks @@ -168,12 +168,13 @@ func (_m *Client) UpdateVariable(airflowURL string, variable airflowclient.Varia return r0 } -// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewClient(t interface { +type mockConstructorTestingTNewClient interface { mock.TestingT Cleanup(func()) -}) *Client { +} + +// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClient(t mockConstructorTestingTNewClient) *Client { mock := &Client{} mock.Mock.Test(t) diff --git a/airflow/airflow.go b/airflow/airflow.go index f43091e12..013f5e157 100644 --- a/airflow/airflow.go +++ b/airflow/airflow.go @@ -34,6 +34,9 @@ var ( //go:embed include/dockerfile Dockerfile string + //go:embed include/test-conflicts.dockerfile + testConflictsDockerfile string + //go:embed include/dockerignore Dockerignore string @@ -131,6 +134,18 @@ func Init(path, airflowImageName, airflowImageTag string) error { return nil } +func initConflictTest(path, airflowImageName, airflowImageTag string) error { + // Map of files to create + files := map[string]string{ + "conflict-check.Dockerfile": fmt.Sprintf(testConflictsDockerfile, airflowImageName, airflowImageTag), + } + // Initialize files + if err := initFiles(path, files); err != nil { + return errors.Wrap(err, "failed to create upgrade check files") + } + return nil +} + // repositoryName creates an airflow repository name func repositoryName(name string) string { return fmt.Sprintf("%s/%s", name, componentName) diff --git a/airflow/airflow_test.go b/airflow/airflow_test.go index 3099c7733..ef1127b05 100644 --- a/airflow/airflow_test.go +++ b/airflow/airflow_test.go @@ -87,3 +87,23 @@ func TestInit(t *testing.T) { assert.True(t, exist) } } + +func TestInitConflictTest(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "temp") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + err = initConflictTest(tmpDir, "astro-runtime", "test") + assert.NoError(t, err) + + expectedFiles := []string{ + "conflict-check.Dockerfile", + } + for _, file := range expectedFiles { + exist, err := fileutil.Exists(filepath.Join(tmpDir, file), nil) + assert.NoError(t, err) + assert.True(t, exist) + } +} diff --git a/airflow/container.go b/airflow/container.go index b44596c7d..c9dbd73e0 100644 --- a/airflow/container.go +++ b/airflow/container.go @@ -10,6 +10,7 @@ import ( "time" "github.com/astronomer/astro-cli/airflow/types" + "github.com/astronomer/astro-cli/astro-client" "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/pkg/fileutil" "github.com/astronomer/astro-cli/pkg/util" @@ -33,6 +34,7 @@ type ContainerHandler interface { ComposeExport(settingsFile, composeFile string) error Pytest(pytestFile, customImageName, deployImageName, pytestArgsString string) (string, error) Parse(customImageName, deployImageName string) error + UpgradeTest(runtimeVersion, deploymentID, newImageName, customImageName string, dependencyTest, versionTest, dagTest bool, client astro.Client) error } // RegistryHandler defines methods require to handle all operations with registry @@ -42,13 +44,16 @@ type RegistryHandler interface { // ImageHandler defines methods require to handle all operations on/for container images type ImageHandler interface { - Build(config types.ImageBuildConfig) error + Build(dockerfile string, config types.ImageBuildConfig) error Push(registry, username, token, remoteImage string) error - GetLabel(labelName string) (string, error) + Pull(registry, username, token, remoteImage string) error + GetLabel(altImageName, labelName string) (string, error) ListLabels() (map[string]string, error) TagLocalImage(localImage string) error Run(dagID, envFile, settingsFile, containerName, dagFile, executionDate string, taskLogs bool) error - Pytest(pytestFile, airflowHome, envFile string, pytestArgs []string, config types.ImageBuildConfig) (string, error) + Pytest(pytestFile, airflowHome, envFile, testHomeDirectory string, pytestArgs []string, htmlReport bool, config types.ImageBuildConfig) (string, error) + ConflictTest(workingDirectory, testHomeDirectory string, buildConfig types.ImageBuildConfig) (string, error) + CreatePipFreeze(altImageName, pipFreezeFile string) error } type DockerComposeAPI interface { diff --git a/airflow/docker.go b/airflow/docker.go index ebd2ccc3e..10038e267 100644 --- a/airflow/docker.go +++ b/airflow/docker.go @@ -1,18 +1,24 @@ package airflow import ( + "bufio" "bytes" "context" "encoding/json" "fmt" + "io/fs" "os" + "path/filepath" "runtime" + "sort" "strings" "text/tabwriter" "time" semver "github.com/Masterminds/semver/v3" airflowTypes "github.com/astronomer/astro-cli/airflow/types" + "github.com/astronomer/astro-cli/astro-client" + "github.com/astronomer/astro-cli/cloud/deployment" "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/docker" "github.com/astronomer/astro-cli/pkg/ansi" @@ -32,10 +38,12 @@ import ( "github.com/docker/docker/api/types/versions" "github.com/pkg/browser" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" ) const ( + RuntimeImageLabel = "io.astronomer.docker.runtime.version" + AirflowImageLabel = "io.astronomer.docker.airflow.version" componentName = "airflow" podman = "podman" dockerStateUp = "running" @@ -47,6 +55,12 @@ const ( pytestDirectory = "tests" OpenCmd = "open" dockerCmd = "docker" + registryUsername = "cli" + unknown = "unknown" + major = "major" + patch = "patch" + minor = "minor" + partsNum = 2 composeCreateErrMsg = "error creating docker-compose project" composeStatusCheckErrMsg = "error checking docker-compose status" @@ -79,6 +93,34 @@ var ( startupTimeout time.Duration isM1 = util.IsM1 + majorUpdatesAirflowProviders = []string{} + minorUpdatesAirflowProviders = []string{} + patchUpdatesAirflowProviders = []string{} + unknownUpdatesAirflowProviders = []string{} + removedPackagesAirflowProviders = []string{} + addedPackagesAirflowProviders = []string{} + majorUpdates = []string{} + minorUpdates = []string{} + patchUpdates = []string{} + unknownUpdates = []string{} + removedPackages = []string{} + addedPackages = []string{} + airflowUpdate = []string{} + titles = []string{ + "Apache Airflow Update:\n", + "Airflow Providers Unknown Updates:\n", + "Airflow Providers Major Updates:\n", + "Airflow Providers Minor Updates:\n", + "Airflow Providers Patch Updates:\n", + "Added Airflow Providers:\n", + "Removed Airflow Providers:\n", + "Unknown Updates:\n", + "Major Updates:\n", + "Minor Updates:\n", + "Patch Updates:\n", + "Added Packages:\n", + "Removed Packages:\n", + } composeOverrideFilename = "docker-compose.override.yml" stopPostgresWaitTimeout = 10 * time.Second @@ -135,12 +177,12 @@ func DockerComposeInit(airflowHome, envFile, dockerfile, imageName string) (*Doc dockerCli, err := command.NewDockerCli() if err != nil { - log.Fatalf("error creating compose client %s", err) + logrus.Fatalf("error creating compose client %s", err) } err = dockerCli.Initialize(flags.NewClientOptions()) if err != nil { - log.Fatalf("error init compose client %s", err) + logrus.Fatalf("error init compose client %s", err) } composeService := compose.NewComposeService(dockerCli.Client(), &configfile.ConfigFile{}) @@ -194,7 +236,7 @@ func (d *DockerCompose) Start(imageName, settingsFile, composeFile string, noCac fmt.Printf("Adding 'astro-run-dag' package to requirements.txt unsuccessful: %s\nManually add package to requirements.txt", err.Error()) } } - imageBuildErr := d.imageHandler.Build(airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true, NoCache: noCache}) + imageBuildErr := d.imageHandler.Build(d.dockerfile, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true, NoCache: noCache}) if !config.CFG.DisableAstroRun.GetBool() { // remove astro-run-dag from requirments.txt err = fileutil.RemoveLineFromFile("./requirements.txt", "astro-run-dag", " # This package is needed for the astro run command. It will be removed before a deploy") @@ -325,7 +367,7 @@ func (d *DockerCompose) Stop(waitForExit bool) error { for { select { case <-timeout: - log.Debug("timed out waiting for postgres container to be in exited state") + logrus.Debug("timed out waiting for postgres container to be in exited state") return nil case <-ticker.C: psInfo, _ := d.composeService.Ps(context.Background(), d.projectName, api.PsOptions{ @@ -336,10 +378,10 @@ func (d *DockerCompose) Stop(waitForExit bool) error { // so docker compose will ensure that postgres container going in shutting down phase only after all other containers have exited if strings.Contains(psInfo[i].Name, PostgresDockerContainerName) { if psInfo[i].State == dockerExitState { - log.Debug("postgres container reached exited state") + logrus.Debug("postgres container reached exited state") return nil } - log.Debugf("postgres container is still in %s state, waiting for it to be in exited state", psInfo[i].State) + logrus.Debugf("postgres container is still in %s state, waiting for it to be in exited state", psInfo[i].State) } } } @@ -456,7 +498,7 @@ func (d *DockerCompose) Pytest(pytestFile, customImageName, deployImageName, pyt if deployImageName == "" { // build image if customImageName == "" { - err := d.imageHandler.Build(airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + err := d.imageHandler.Build(d.dockerfile, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) if err != nil { return "", err } @@ -482,7 +524,7 @@ func (d *DockerCompose) Pytest(pytestFile, customImageName, deployImageName, pyt } // run pytests - exitCode, err := d.imageHandler.Pytest(pytestFile, d.airflowHome, d.envFile, pytestArgs, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + exitCode, err := d.imageHandler.Pytest(pytestFile, d.airflowHome, d.envFile, "", pytestArgs, false, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) if err != nil { return exitCode, err } @@ -492,6 +534,504 @@ func (d *DockerCompose) Pytest(pytestFile, customImageName, deployImageName, pyt return exitCode, errors.New("something went wrong while Pytesting your DAGs") } +func (d *DockerCompose) UpgradeTest(newAirflowVersion, deploymentID, newImageName, customImage string, conflictTest, versionTest, dagTest bool, client astro.Client) error { + // figure out which tests to run + if !conflictTest && !versionTest && !dagTest { + conflictTest = true + versionTest = true + dagTest = true + } + // if user supplies deployment id pull down current image + var deploymentImage string + if deploymentID != "" { + err := d.pullImageFromDeployment(deploymentID, client) + if err != nil { + return err + } + } else { + // build image for current Airflow version to get current Airflow version + fmt.Println("\nBuilding image for current Airflow version") + imageBuildErr := d.imageHandler.Build(d.dockerfile, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + if imageBuildErr != nil { + return imageBuildErr + } + } + // get current Airflow version + currentAirflowVersion, err := d.imageHandler.GetLabel(deploymentImage, RuntimeImageLabel) + if err != nil { + return err + } + if currentAirflowVersion == "" { + currentAirflowVersion, err = d.imageHandler.GetLabel(deploymentImage, AirflowImageLabel) + if err != nil { + return err + } + } + // create test home directory + testHomeDirectory := "upgrade-test-" + currentAirflowVersion + "--" + newAirflowVersion + + destFolder := filepath.Join(d.airflowHome, testHomeDirectory) + var filePerms fs.FileMode = 0o755 + if err := os.MkdirAll(destFolder, filePerms); err != nil { + return err + } + newDockerFile := destFolder + "/Dockerfile" + + // check for dependency conflicts + if conflictTest { + err = d.conflictTest(testHomeDirectory, newImageName, newAirflowVersion) + if err != nil { + return err + } + } + var pipFreezeCompareFile string + if versionTest { + err := d.versionTest(testHomeDirectory, currentAirflowVersion, deploymentImage, newDockerFile, newAirflowVersion, customImage) + if err != nil { + return err + } + } + if dagTest { + err := d.dagTest(testHomeDirectory, newAirflowVersion, newDockerFile, customImage) + if err != nil { + return err + } + } + fmt.Println("\nTest Summary:") + fmt.Printf("\tUpgrade Test Results Directory: %s\n", testHomeDirectory) + if conflictTest { + fmt.Printf("\tDependency Conflict Test Results file: %s\n", "conflict-test-results.txt") + } + if versionTest { + fmt.Printf("\tDependency Version Comparison Results file: %s\n", pipFreezeCompareFile) + } + if dagTest { + fmt.Printf("\tDAG Parse Test HTML Report: %s\n", "dag-test-report.html") + } + + return nil +} + +func (d *DockerCompose) pullImageFromDeployment(deploymentID string, client astro.Client) error { + c, err := config.GetCurrentContext() + if err != nil { + return err + } + domain := c.Domain + if domain == "" { + return errors.New("no domain set, re-authenticate") + } + ws := c.Workspace + registry := GetRegistryURL(domain) + repository := registry + "/" + c.Organization + "/" + deploymentID + currentDeployment, err := deployment.GetDeployment(ws, deploymentID, "", true, client, nil) + if err != nil { + return err + } + currentImageTag := currentDeployment.DeploymentSpec.Image.Tag + deploymentImage := fmt.Sprintf("%s:%s", repository, currentImageTag) + token := c.Token + // Splitting out the Bearer part from the token + splittedToken := strings.Split(token, " ") + if len(splittedToken) > 1 { + token = strings.Split(token, " ")[1] + } + fmt.Printf("\nPulling image from Astro Deployment %s\n\n", currentDeployment.Label) + err = d.imageHandler.Pull(registry, registryUsername, token, deploymentImage) + if err != nil { + return err + } + return nil +} + +func (d *DockerCompose) conflictTest(testHomeDirectory, newImageName, newAirflowVersion string) error { + fmt.Println("\nChecking your 'requirments.txt' for dependency conflicts with the new version of Airflow") + fmt.Println("\nThis may take a few minutes...") + + // create files needed for conflict test + err := initConflictTest(config.WorkingPath, newImageName, newAirflowVersion) + defer os.Remove("conflict-check.Dockerfile") + if err != nil { + return err + } + + exitCode, conflictErr := d.imageHandler.ConflictTest(d.airflowHome, testHomeDirectory, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + if conflictErr != nil { + return conflictErr + } + if strings.Contains(exitCode, "0") || exitCode == "" { // if the error code is 0 the pytests passed + fmt.Println("There were no dependency conflicts found") + } else { + fmt.Println("\nSomething went wrong while compiling your dependencies check the logs above for conflicts") + fmt.Println("If there are conflicts remove them from your 'requirments.txt' and rerun this test\nYou will see the best candidate in the 'conflict-test-results.txt' file") + return err + } + return nil +} + +func (d *DockerCompose) versionTest(testHomeDirectory, currentAirflowVersion, deploymentImage, newDockerFile, newAirflowVersion, customImage string) error { + fmt.Println("\nComparing dependency versions between current and upgraded environment") + // pip freeze old Airflow image + fmt.Println("\nObtaining pip freeze for current Airflow version") + currentAirflowPipFreezeFile := d.airflowHome + "/" + testHomeDirectory + "/pip_freeze_" + currentAirflowVersion + ".txt" + err := d.imageHandler.CreatePipFreeze(deploymentImage, currentAirflowPipFreezeFile) + if err != nil { + return err + } + + // build image with the new airflow version + err = upgradeDockerfile(d.dockerfile, newDockerFile, newAirflowVersion, customImage) + if err != nil { + return err + } + fmt.Println("\nBuilding image for new Airflow version") + imageBuildErr := d.imageHandler.Build(newDockerFile, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + if imageBuildErr != nil { + return imageBuildErr + } + // pip freeze new airflow image + fmt.Println("\nObtaining pip freeze for new Airflow version") + newAirflowPipFreezeFile := d.airflowHome + "/" + testHomeDirectory + "/pip_freeze_" + newAirflowVersion + ".txt" + err = d.imageHandler.CreatePipFreeze("", newAirflowPipFreezeFile) + if err != nil { + return err + } + // compare pip freeze files + fmt.Println("\nComparing pip freeze files") + pipFreezeCompareFile := d.airflowHome + "/" + testHomeDirectory + "/dependency_compare.txt" + err = CreateVersionTestFile(currentAirflowPipFreezeFile, newAirflowPipFreezeFile, pipFreezeCompareFile) + if err != nil { + return err + } + fmt.Printf("Pip Freeze comparison can be found at \n" + pipFreezeCompareFile) + return nil +} + +func (d *DockerCompose) dagTest(testHomeDirectory, newAirflowVersion, newDockerFile, customImage string) error { + fmt.Printf("\nChecking the DAGs in this project for errors against the new Airflow version %s\n", newAirflowVersion) + + // build image with the new runtime version + err := upgradeDockerfile(d.dockerfile, newDockerFile, newAirflowVersion, customImage) + if err != nil { + return err + } + + // add pytest-html to the requirements + err = fileutil.AddLineToFile("./requirements.txt", "pytest-html", "# This package is needed for the upgrade dag test. It will be removed once the test is over") + if err != nil { + fmt.Printf("Adding 'pytest-html' package to requirements.txt unsuccessful: %s\nManually add package to requirements.txt", err.Error()) + } + fmt.Println("\nBuilding image for new Airflow version") + imageBuildErr := d.imageHandler.Build(newDockerFile, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + + // remove pytest-html to the requirements + err = fileutil.RemoveLineFromFile("./requirements.txt", "pytest-html", " # This package is needed for the upgrade dag test. It will be removed once the test is over") + if err != nil { + fmt.Printf("Removing package 'pytest-html' from requirements.txt unsuccessful: %s\n", err.Error()) + } + if imageBuildErr != nil { + return imageBuildErr + } + // check for file + path := d.airflowHome + "/" + DefaultTestPath + + fileExist, err := util.Exists(path) + if err != nil { + return err + } + if !fileExist { + fmt.Println("\nThe file " + path + " which is needed for the parse test does not exist. Please run `astro dev init` to create it") + + return err + } + // run parse test + pytestFile := DefaultTestPath + // create html report + htmlReportArgs := "--html=dag-test-report.html --self-contained-html" + // compare pip freeze files + fmt.Println("\nRunning parse test") + exitCode, err := d.imageHandler.Pytest(pytestFile, d.airflowHome, d.envFile, testHomeDirectory, strings.Fields(htmlReportArgs), true, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true}) + if err != nil { + if strings.Contains(exitCode, "1") { // exit code is 1 meaning tests failed + fmt.Println("See above for errors detected in your DAGs") + } else { + return errors.Wrap(err, "something went wrong while parsing your DAGs") + } + } else { + fmt.Println("\n" + ansi.Green("✔") + " no errors detected in your DAGs ") + } + return nil +} + +func GetRegistryURL(domain string) string { + var registry string + if domain == "localhost" { + registry = config.CFG.LocalRegistry.GetString() + } else { + registry = "images." + strings.Split(domain, ".")[0] + ".cloud" + } + return registry +} + +func upgradeDockerfile(oldDockerfilePath, newDockerfilePath, newTag, newImage string) error { + // Read the content of the old Dockerfile + content, err := os.ReadFile(oldDockerfilePath) + if err != nil { + return err + } + + lines := strings.Split(string(content), "\n") + var newContent strings.Builder + if newImage == "" { + for _, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "FROM quay.io/astronomer/astro-runtime:") { + // Replace the tag on the matching line + parts := strings.SplitN(line, ":", partsNum) + if len(parts) == partsNum { + line = parts[0] + ":" + newTag + } + } + newContent.WriteString(line + "\n") // Add a newline after each line + } + } else { + for _, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "FROM ") { + // Replace the tag on the matching line + parts := strings.SplitN(line, " ", partsNum) + if len(parts) == partsNum { + line = parts[0] + " " + newImage + } + } + newContent.WriteString(line + "\n") // Add a newline after each line + } + } + + // Write the new content to the new Dockerfile + err = os.WriteFile(newDockerfilePath, []byte(newContent.String()), 0o600) //nolint:gomnd + if err != nil { + return err + } + + return nil +} + +func CreateVersionTestFile(beforeFile, afterFile, outputFile string) error { + // Open the before file for reading + before, err := os.Open(beforeFile) + if err != nil { + return err + } + defer before.Close() + + // Open the after file for reading + after, err := os.Open(afterFile) + if err != nil { + return err + } + defer after.Close() + + // Create the output file for writing + output, err := os.Create(outputFile) + if err != nil { + return err + } + defer output.Close() + + // Create a map to store versions by package name + pgkVersions := make(map[string][2]string) + + // Read versions from the before file and store them in the map + beforeScanner := bufio.NewScanner(before) + for beforeScanner.Scan() { + line := beforeScanner.Text() + parts := strings.Split(line, "==") + if len(parts) == partsNum { + pkg := parts[0] + ver := parts[1] + pgkVersions[pkg] = [2]string{ver, ""} + } + } + + // Read versions from the after file and update the map with the new versions + afterScanner := bufio.NewScanner(after) + for afterScanner.Scan() { + line := afterScanner.Text() + parts := strings.Split(line, "==") + if len(parts) == partsNum { + pkg := parts[0] + ver := parts[1] + if v, ok := pgkVersions[pkg]; ok { + v[1] = ver + pgkVersions[pkg] = v + } else { + pgkVersions[pkg] = [2]string{"", ver} + } + } + } + // Iterate over the versions map and categorize the changes + err = iteratePkgMap(pgkVersions) + if err != nil { + return err + } + // sort lists into alphabetical order + sort.Strings(unknownUpdatesAirflowProviders) + sort.Strings(majorUpdatesAirflowProviders) + sort.Strings(minorUpdatesAirflowProviders) + sort.Strings(patchUpdatesAirflowProviders) + sort.Strings(addedPackagesAirflowProviders) + sort.Strings(removedPackagesAirflowProviders) + sort.Strings(unknownUpdates) + sort.Strings(majorUpdates) + sort.Strings(minorUpdates) + sort.Strings(patchUpdates) + sort.Strings(addedPackages) + sort.Strings(removedPackages) + pkgLists := [][]string{ + airflowUpdate, + unknownUpdatesAirflowProviders, + majorUpdatesAirflowProviders, + minorUpdatesAirflowProviders, + patchUpdatesAirflowProviders, + addedPackagesAirflowProviders, + removedPackagesAirflowProviders, + unknownUpdates, + majorUpdates, + minorUpdates, + patchUpdates, + addedPackages, + removedPackages, + } + + // Write the categorized updates to the output file + writer := bufio.NewWriter(output) + + for i, title := range titles { + writeToCompareFile(title, pkgLists[i], writer) + } + + // Flush the buffer to ensure all data is written to the file + writer.Flush() + return nil +} + +func iteratePkgMap(pgkVersions map[string][2]string) error { //nolint:gocognit + // Iterate over the versions map and categorize the changes + for pkg, ver := range pgkVersions { + beforeVer := ver[0] + afterVer := ver[1] + if beforeVer != "" && afterVer != "" && beforeVer != afterVer { + change, updateType, err := checkVersionChange(beforeVer, afterVer) + if err != nil { + if err.Error() == "Invalid Semantic Version" { + updateType = unknown + } else { + return err + } + } + if !change { + updateType = unknown + } + pkgUpdate := pkg + " " + beforeVer + " >> " + afterVer + + // Categorize the packages based on the update type + categorizeAirflowProviderPackage(pkg, pkgUpdate, updateType) + } + switch { + case strings.Contains(pkg, "apache-airflow-providers-"): + if beforeVer != "" && afterVer == "" { + pkgUpdate := pkg + "==" + beforeVer + removedPackagesAirflowProviders = append(removedPackagesAirflowProviders, pkgUpdate) + } + if beforeVer == "" && afterVer != "" { + pkgUpdate := pkg + "==" + afterVer + addedPackagesAirflowProviders = append(addedPackagesAirflowProviders, pkgUpdate) + } + default: + if beforeVer != "" && afterVer == "" { + pkgUpdate := pkg + "==" + beforeVer + removedPackages = append(removedPackages, pkgUpdate) + } + if beforeVer == "" && afterVer != "" { + pkgUpdate := pkg + "==" + afterVer + addedPackages = append(addedPackages, pkgUpdate) + } + } + } + return nil +} + +func categorizeAirflowProviderPackage(pkg, pkgUpdate, updateType string) { + // Categorize the packages based on the update type + switch { + case strings.Contains(pkg, "apache-airflow-providers-"): + switch updateType { + case major: + majorUpdatesAirflowProviders = append(majorUpdatesAirflowProviders, pkgUpdate) + case minor: + minorUpdatesAirflowProviders = append(minorUpdatesAirflowProviders, pkgUpdate) + case patch: + patchUpdatesAirflowProviders = append(patchUpdatesAirflowProviders, pkgUpdate) + case unknown: + unknownUpdatesAirflowProviders = append(unknownUpdatesAirflowProviders, pkgUpdate) + } + case pkg == "apache-airflow": + airflowUpdate = append(airflowUpdate, pkgUpdate) + default: + switch updateType { + case major: + majorUpdates = append(majorUpdates, pkgUpdate) + case minor: + minorUpdates = append(minorUpdates, pkgUpdate) + case patch: + patchUpdates = append(patchUpdates, pkgUpdate) + case unknown: + unknownUpdates = append(unknownUpdates, pkgUpdate) + } + } +} + +func writeToCompareFile(title string, pkgList []string, writer *bufio.Writer) { + if len(pkgList) > 0 { + _, err := writer.WriteString(title) + if err != nil { + logrus.Debug(err) + } + for _, pkg := range pkgList { + _, err = writer.WriteString(pkg + "\n") + if err != nil { + logrus.Debug(err) + } + } + _, err = writer.WriteString("\n") + if err != nil { + logrus.Debug(err) + } + } +} + +func checkVersionChange(before, after string) (change bool, updateType string, err error) { + beforeVer, err := semver.NewVersion(before) + if err != nil { + return false, "", err + } + + afterVer, err := semver.NewVersion(after) + if err != nil { + return false, "", err + } + + switch { + case afterVer.Major() > beforeVer.Major(): + return true, "major", nil + case afterVer.Minor() > beforeVer.Minor(): + return true, "minor", nil + case afterVer.Patch() > beforeVer.Patch(): + return true, "patch", nil + default: + return false, "", nil + } +} + func (d *DockerCompose) Parse(customImageName, deployImageName string) error { // check for file path := d.airflowHome + "/" + DefaultTestPath @@ -701,7 +1241,7 @@ func (d *DockerCompose) RunDAG(dagID, settingsFile, dagFile, executionDate strin fmt.Printf("Removing line 'astro-run-dag' package from requirements.txt unsuccessful: %s\n", err.Error()) } }() - err = d.imageHandler.Build(airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true, NoCache: noCache}) + err = d.imageHandler.Build(d.dockerfile, airflowTypes.ImageBuildConfig{Path: d.airflowHome, Output: true, NoCache: noCache}) if err != nil { return err } diff --git a/airflow/docker_image.go b/airflow/docker_image.go index ac91ed40a..c0b0d6a8c 100644 --- a/airflow/docker_image.go +++ b/airflow/docker_image.go @@ -10,6 +10,7 @@ import ( "io" "os" "os/exec" + "regexp" "strings" "github.com/astronomer/astro-cli/pkg/util" @@ -29,6 +30,8 @@ const ( EchoCmd = "echo" pushingImagePrompt = "Pushing image to Astronomer registry" astroRunContainer = "astro-run" + pullingImagePrompt = "Pulling image from Astronomer registry" + prefix = "Bearer " ) var errGetImageLabel = errors.New("error getting image label") @@ -41,9 +44,11 @@ func DockerImageInit(image string) *DockerImage { return &DockerImage{imageName: image} } -func (d *DockerImage) Build(buildConfig airflowTypes.ImageBuildConfig) error { +func (d *DockerImage) Build(dockerfile string, buildConfig airflowTypes.ImageBuildConfig) error { dockerCommand := config.CFG.DockerCommand.GetString() - + if dockerfile == "" { + dockerfile = "Dockerfile" + } err := os.Chdir(buildConfig.Path) if err != nil { return err @@ -52,6 +57,8 @@ func (d *DockerImage) Build(buildConfig airflowTypes.ImageBuildConfig) error { "build", "-t", d.imageName, + "-f", + dockerfile, ".", } if buildConfig.NoCache { @@ -77,7 +84,7 @@ func (d *DockerImage) Build(buildConfig airflowTypes.ImageBuildConfig) error { return err } -func (d *DockerImage) Pytest(pytestFile, airflowHome, envFile string, pytestArgs []string, buildConfig airflowTypes.ImageBuildConfig) (string, error) { +func (d *DockerImage) Pytest(pytestFile, airflowHome, envFile, testHomeDirectory string, pytestArgs []string, htmlReport bool, buildConfig airflowTypes.ImageBuildConfig) (string, error) { // delete container dockerCommand := config.CFG.DockerCommand.GetString() err := cmdExec(dockerCommand, nil, nil, "rm", "astro-pytest") @@ -140,7 +147,18 @@ func (d *DockerImage) Pytest(pytestFile, airflowHome, envFile string, pytestArgs if err != nil { log.Debug(err) } - + if htmlReport { + // Copy the dag-test-report.html file from the container to the destination folder + err = cmdExec(dockerCommand, nil, stderr, "cp", "astro-pytest:/usr/local/airflow/dag-test-report.html", "./"+testHomeDirectory) + if err != nil { + // Remove the temporary container + err2 := cmdExec(dockerCommand, nil, stderr, "rm", "astro-pytest") + if err2 != nil { + return outb.String(), err2 + } + return outb.String(), err + } + } // delete container err = cmdExec(dockerCommand, nil, stderr, "rm", "astro-pytest") if err != nil { @@ -150,6 +168,104 @@ func (d *DockerImage) Pytest(pytestFile, airflowHome, envFile string, pytestArgs return outb.String(), docErr } +func (d *DockerImage) ConflictTest(workingDirectory, testHomeDirectory string, buildConfig airflowTypes.ImageBuildConfig) (string, error) { + dockerCommand := config.CFG.DockerCommand.GetString() + // delete container + err := cmdExec(dockerCommand, nil, nil, "rm", "astro-temp-container") + if err != nil { + log.Debug(err) + } + // Change to location of Dockerfile + err = os.Chdir(buildConfig.Path) + if err != nil { + return "", err + } + args := []string{ + "build", + "-t", + "conflict-check:latest", + "-f", + "conflict-check.Dockerfile", + ".", + } + + // Create a buffer to capture the command output + var stdout, stderr bytes.Buffer + multiStdout := io.MultiWriter(&stdout, os.Stdout) + multiStderr := io.MultiWriter(&stderr, os.Stdout) + + // Start the command execution + err = cmdExec(dockerCommand, multiStdout, multiStderr, args...) + if err != nil { + return "", err + } + // Get the exit code + exitCode := "" + if _, ok := err.(*exec.ExitError); ok { + // The command exited with a non-zero status + exitCode = parseExitCode(stderr.String()) + } else if err != nil { + // An error occurred while running the command + return "", err + } + // Run a temporary container to copy the file from the image + err = cmdExec(dockerCommand, nil, nil, "create", "--name", "astro-temp-container", "conflict-check:latest") + if err != nil { + return exitCode, err + } + // Copy the result.txt file from the container to the destination folder + err1 := cmdExec(dockerCommand, nil, nil, "cp", "astro-temp-container:/usr/local/airflow/conflict-test-results.txt", "./"+testHomeDirectory) + if err1 != nil { + // Remove the temporary container + err = cmdExec(dockerCommand, nil, nil, "rm", "astro-temp-container") + if err != nil { + return exitCode, err + } + return exitCode, err1 + } + + // Remove the temporary container + err = cmdExec(dockerCommand, nil, nil, "rm", "astro-temp-container") + if err != nil { + return exitCode, err + } + return exitCode, nil +} + +func parseExitCode(logs string) string { + re := regexp.MustCompile(`exit code: (\d+)`) + match := re.FindStringSubmatch(logs) + if len(match) > 1 { + return match[1] + } + return "" +} + +func (d *DockerImage) CreatePipFreeze(altImageName, pipFreezeFile string) error { + dockerCommand := config.CFG.DockerCommand.GetString() + // Define the Docker command and arguments + imageName := d.imageName + if altImageName != "" { + imageName = altImageName + } + dockerArgs := []string{"run", "--rm", imageName, "pip", "freeze"} + + // Create a file to store the command output + file, err := os.Create(pipFreezeFile) + if err != nil { + return err + } + defer file.Close() + + // Run the Docker command + err = cmdExec(dockerCommand, file, os.Stderr, dockerArgs...) + if err != nil { + return err + } + + return nil +} + func (d *DockerImage) Push(registry, username, token, remoteImage string) error { dockerCommand := config.CFG.DockerCommand.GetString() err := cmdExec(dockerCommand, nil, nil, "tag", d.imageName, remoteImage) @@ -219,6 +335,29 @@ func (d *DockerImage) Push(registry, username, token, remoteImage string) error return nil } +func (d *DockerImage) Pull(registry, username, token, remoteImage string) error { + // Pulling image to registry + fmt.Println(pullingImagePrompt) + dockerCommand := config.CFG.DockerCommand.GetString() + var err error + if username != "" { // Case for cloud image push where we have both registry user & pass, for software login happens during `astro login` itself + pass := token + pass = strings.TrimPrefix(pass, prefix) + cmd := "echo \"" + pass + "\"" + " | " + dockerCommand + " login " + registry + " -u " + username + " --password-stdin" + err = cmdExec("bash", os.Stdout, os.Stderr, "-c", cmd) // This command will only work on machines that have bash. If users have issues we will revist + } + if err != nil { + return err + } + // docker pull + err = cmdExec(dockerCommand, os.Stdout, os.Stderr, "pull", remoteImage) + if err != nil { + return err + } + + return nil +} + var displayJSONMessagesToStream = func(responseBody io.ReadCloser, auxCallback func(jsonmessage.JSONMessage)) error { out := cliCommand.NewOutStream(os.Stdout) err := jsonmessage.DisplayJSONMessagesToStream(responseBody, out, nil) @@ -228,14 +367,18 @@ var displayJSONMessagesToStream = func(responseBody io.ReadCloser, auxCallback f return nil } -func (d *DockerImage) GetLabel(labelName string) (string, error) { +func (d *DockerImage) GetLabel(altImageName, labelName string) (string, error) { dockerCommand := config.CFG.DockerCommand.GetString() stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) labelFmt := fmt.Sprintf("{{ index .Config.Labels %q }}", labelName) var label string - err := cmdExec(dockerCommand, stdout, stderr, "inspect", "--format", labelFmt, d.imageName) + imageName := d.imageName + if altImageName != "" { + imageName = altImageName + } + err := cmdExec(dockerCommand, stdout, stderr, "inspect", "--format", labelFmt, imageName) if err != nil { return label, err } @@ -400,7 +543,6 @@ func useBash(authConfig *cliTypes.AuthConfig, image string) error { var err error if authConfig.Username != "" { // Case for cloud image push where we have both registry user & pass, for software login happens during `astro login` itself pass := authConfig.Password - prefix := "Bearer " pass = strings.TrimPrefix(pass, prefix) cmd := "echo \"" + pass + "\"" + " | " + dockerCommand + " login " + authConfig.ServerAddress + " -u " + authConfig.Username + " --password-stdin" err = cmdExec("bash", os.Stdout, os.Stderr, "-c", cmd) // This command will only work on machines that have bash. If users have issues we will revist diff --git a/airflow/docker_image_test.go b/airflow/docker_image_test.go index cb2184d4b..7b5766a0c 100644 --- a/airflow/docker_image_test.go +++ b/airflow/docker_image_test.go @@ -45,7 +45,7 @@ func TestDockerImageBuild(t *testing.T) { cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { return nil } - err = handler.Build(options) + err = handler.Build("", options) assert.NoError(t, err) }) @@ -56,7 +56,7 @@ func TestDockerImageBuild(t *testing.T) { assert.Contains(t, args, "--no-cache") return nil } - err = handler.Build(options) + err = handler.Build("", options) assert.NoError(t, err) }) @@ -64,10 +64,9 @@ func TestDockerImageBuild(t *testing.T) { cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { return errMock } - err = handler.Build(options) + err = handler.Build("", options) assert.Contains(t, err.Error(), errMock.Error()) }) - t.Run("unable to read file error", func(t *testing.T) { options := airflowTypes.ImageBuildConfig{ Path: "incorrect-path", @@ -76,7 +75,7 @@ func TestDockerImageBuild(t *testing.T) { Output: false, } - err = handler.Build(options) + err = handler.Build("", options) assert.Error(t, err) }) @@ -84,6 +83,7 @@ func TestDockerImageBuild(t *testing.T) { } func TestDockerImagePytest(t *testing.T) { + testUtil.InitTestConfig(testUtil.LocalPlatform) handler := DockerImage{ imageName: "testing", } @@ -108,10 +108,23 @@ func TestDockerImagePytest(t *testing.T) { cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { return nil } - _, err = handler.Pytest("", "", "", []string{}, options) + _, err = handler.Pytest("", "", "", "", []string{}, true, options) assert.NoError(t, err) }) + t.Run("copy error", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + switch { + case args[0] == "cp": + return errMock + default: + return nil + } + } + _, err = handler.Pytest("", "", "", "", []string{}, true, options) + assert.Error(t, err) + }) + t.Run("pytest error", func(t *testing.T) { options = airflowTypes.ImageBuildConfig{ Path: cwd, @@ -123,10 +136,107 @@ func TestDockerImagePytest(t *testing.T) { cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { return errMock } - _, err = handler.Pytest("", "", "", []string{}, options) + _, err = handler.Pytest("", "", "", "", []string{}, false, options) assert.Contains(t, err.Error(), errMock.Error()) }) + t.Run("unable to read file error", func(t *testing.T) { + options := airflowTypes.ImageBuildConfig{ + Path: "incorrect-path", + TargetPlatforms: []string{"linux/amd64"}, + NoCache: false, + } + + _, err = handler.Pytest("", "", "", "", []string{}, false, options) + assert.Error(t, err) + }) + + cmdExec = previousCmdExec +} + +func TestDockerImageConflictTest(t *testing.T) { + testUtil.InitTestConfig(testUtil.LocalPlatform) + handler := DockerImage{ + imageName: "testing", + } + + cwd, err := os.Getwd() + assert.NoError(t, err) + + dockerIgnoreFile := cwd + "/.dockerignore" + fileutil.WriteStringToFile(dockerIgnoreFile, "") + defer afero.NewOsFs().Remove(dockerIgnoreFile) + + options := airflowTypes.ImageBuildConfig{ + Path: cwd, + TargetPlatforms: []string{"linux/amd64"}, + NoCache: false, + Output: true, + } + + previousCmdExec := cmdExec + + t.Run("conflict test success", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return nil + } + _, err = handler.ConflictTest("", "", options) + assert.NoError(t, err) + }) + + t.Run("conflict test create error", func(t *testing.T) { + options = airflowTypes.ImageBuildConfig{ + Path: cwd, + TargetPlatforms: []string{"linux/amd64"}, + NoCache: false, + Output: false, + } + + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return errMock + } + _, err = handler.ConflictTest("", "", options) + assert.Error(t, err) + }) + + t.Run("conflict test cp error", func(t *testing.T) { + options = airflowTypes.ImageBuildConfig{ + Path: cwd, + TargetPlatforms: []string{"linux/amd64"}, + NoCache: false, + Output: false, + } + + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + switch { + case args[0] == "cp": + return errMock + default: + return nil + } + } + _, err = handler.ConflictTest("", "", options) + assert.Error(t, err) + }) + t.Run("conflict test rm error", func(t *testing.T) { + options = airflowTypes.ImageBuildConfig{ + Path: cwd, + TargetPlatforms: []string{"linux/amd64"}, + NoCache: false, + Output: false, + } + + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + switch { + case args[0] == "rm": + return errMock + default: + return nil + } + } + _, err = handler.ConflictTest("", "", options) + assert.Error(t, err) + }) t.Run("unable to read file error", func(t *testing.T) { options := airflowTypes.ImageBuildConfig{ Path: "incorrect-path", @@ -134,7 +244,90 @@ func TestDockerImagePytest(t *testing.T) { NoCache: false, } - _, err = handler.Pytest("", "", "", []string{}, options) + _, err = handler.ConflictTest("", "", options) + assert.Error(t, err) + }) + + cmdExec = previousCmdExec +} + +func TestParseExitCode(t *testing.T) { + output := "exit code: 1" + t.Run("success", func(t *testing.T) { + _ = parseExitCode(output) + _ = parseExitCode("") + }) +} + +func TestDockerCreatePipFreeze(t *testing.T) { + testUtil.InitTestConfig(testUtil.CloudPlatform) + handler := DockerImage{ + imageName: "testing", + } + + cwd, err := os.Getwd() + assert.NoError(t, err) + + pipFreeze := cwd + "/pip-freeze-test.txt" + defer afero.NewOsFs().Remove(pipFreeze) + + previousCmdExec := cmdExec + + t.Run("create pip freeze success", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return nil + } + err := handler.CreatePipFreeze("", pipFreeze) + assert.NoError(t, err) + }) + t.Run("create pip freeze error", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return errMock + } + err := handler.CreatePipFreeze("", pipFreeze) + assert.Error(t, err) + }) + t.Run("unable to read file error", func(t *testing.T) { + err := handler.CreatePipFreeze("", "") + assert.Error(t, err) + }) + + cmdExec = previousCmdExec +} + +func TestDockerPull(t *testing.T) { + testUtil.InitTestConfig(testUtil.LocalPlatform) + handler := DockerImage{ + imageName: "testing", + } + + previousCmdExec := cmdExec + + t.Run("pull image without username", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return nil + } + err := handler.Pull("", "", "", "") + assert.NoError(t, err) + }) + + t.Run("pull image with username", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return nil + } + err := handler.Pull("", "username", "", "") + assert.NoError(t, err) + }) + t.Run("pull error", func(t *testing.T) { + cmdExec = func(cmd string, stdout, stderr io.Writer, args ...string) error { + return errMock + } + err := handler.Pull("", "", "", "") + assert.Error(t, err) + }) + + t.Run("login error", func(t *testing.T) { + err := handler.Pull("", "username", "", "") assert.Error(t, err) }) @@ -203,7 +396,7 @@ func TestDockerImageGetLabel(t *testing.T) { return nil } - resp, err := handler.GetLabel(mockLabel) + resp, err := handler.GetLabel("", mockLabel) assert.NoError(t, err) assert.Equal(t, mockResp, resp) }) @@ -215,7 +408,7 @@ func TestDockerImageGetLabel(t *testing.T) { return errMockDocker } - _, err := handler.GetLabel(mockLabel) + _, err := handler.GetLabel("", mockLabel) assert.ErrorIs(t, err, errMockDocker) }) @@ -228,7 +421,7 @@ func TestDockerImageGetLabel(t *testing.T) { return nil } - _, err := handler.GetLabel(mockLabel) + _, err := handler.GetLabel("", mockLabel) assert.ErrorIs(t, err, errGetImageLabel) }) } diff --git a/airflow/docker_test.go b/airflow/docker_test.go index 15cdf29a9..28d1975bc 100644 --- a/airflow/docker_test.go +++ b/airflow/docker_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + _ "embed" "fmt" "io" "os" @@ -12,10 +13,13 @@ import ( "github.com/astronomer/astro-cli/airflow/mocks" airflowTypes "github.com/astronomer/astro-cli/airflow/types" + "github.com/astronomer/astro-cli/astro-client" + astro_mocks "github.com/astronomer/astro-cli/astro-client/mocks" "github.com/astronomer/astro-cli/config" "github.com/sirupsen/logrus" - testUtils "github.com/astronomer/astro-cli/pkg/testing" + "github.com/astronomer/astro-cli/pkg/fileutil" + testUtil "github.com/astronomer/astro-cli/pkg/testing" "github.com/compose-spec/compose-go/types" "github.com/docker/compose/v2/pkg/api" docker_types "github.com/docker/docker/api/types" @@ -28,6 +32,10 @@ import ( var ( errMockDocker = errors.New("mock docker compose error") errMockSettings = errors.New("mock Settings error") + //go:embed testfiles/pip_freeze_new-version.txt + pipFreezeFile string + //go:embed testfiles/pip_freeze_old-version.txt + pipFreezeFile2 string ) var airflowVersionLabel = "2.2.5" @@ -50,7 +58,7 @@ func TestCheckServiceStateFalse(t *testing.T) { func TestGenerateConfig(t *testing.T) { fs := afero.NewMemMapFs() - configYaml := testUtils.NewTestConfig(testUtils.LocalPlatform) + configYaml := testUtil.NewTestConfig(testUtil.LocalPlatform) err := afero.WriteFile(fs, config.HomeConfigFile, configYaml, 0o777) assert.NoError(t, err) config.InitConfig(fs) @@ -341,19 +349,19 @@ func TestCheckTriggererEnabled(t *testing.T) { } func TestDockerComposeInit(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) _, err := DockerComposeInit("./testfiles", "", "Dockerfile", "") assert.NoError(t, err) } func TestDockerComposeStart(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} waitTime := 1 * time.Second t.Run("success", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{airflowVersionLabelName: airflowVersionLabel}, nil).Times(4) imageHandler.On("TagLocalImage", mock.Anything).Return(nil).Once() @@ -384,7 +392,7 @@ func TestDockerComposeStart(t *testing.T) { defaultTimeOut := 1 * time.Minute noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{airflowVersionLabelName: airflowVersionLabel}, nil).Times(2) composeMock := new(mocks.DockerComposeAPI) @@ -418,7 +426,7 @@ func TestDockerComposeStart(t *testing.T) { expectedTimeout := 5 * time.Minute noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{airflowVersionLabelName: airflowVersionLabel}, nil).Times(2) composeMock := new(mocks.DockerComposeAPI) @@ -451,7 +459,7 @@ func TestDockerComposeStart(t *testing.T) { userProvidedTimeOut := 8 * time.Minute noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{airflowVersionLabelName: airflowVersionLabel}, nil).Times(2) composeMock := new(mocks.DockerComposeAPI) @@ -478,7 +486,7 @@ func TestDockerComposeStart(t *testing.T) { t.Run("success with invalid airflow version label", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{airflowVersionLabelName: "2.3.4.dev+astro1"}, nil).Times(4) imageHandler.On("TagLocalImage", mock.Anything).Return(nil).Once() @@ -532,7 +540,7 @@ func TestDockerComposeStart(t *testing.T) { t.Run("image build failure", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(errMockDocker).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(errMockDocker).Once() composeMock := new(mocks.DockerComposeAPI) composeMock.On("Ps", mock.Anything, mockDockerCompose.projectName, api.PsOptions{All: true}).Return([]api.ContainerSummary{}, nil).Once() @@ -556,7 +564,7 @@ func TestDockerComposeStart(t *testing.T) { t.Run("list label failure", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{}, errMockDocker).Once() composeMock := new(mocks.DockerComposeAPI) @@ -581,7 +589,7 @@ func TestDockerComposeStart(t *testing.T) { t.Run("compose up failure", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{}, nil).Once() composeMock := new(mocks.DockerComposeAPI) @@ -607,7 +615,7 @@ func TestDockerComposeStart(t *testing.T) { t.Run("webserver health check failure", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("ListLabels").Return(map[string]string{airflowVersionLabelName: airflowVersionLabel}, nil).Twice() composeMock := new(mocks.DockerComposeAPI) @@ -632,7 +640,7 @@ func TestDockerComposeStart(t *testing.T) { } func TestDockerComposeExport(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test", airflowHome: "/home/airflow", envFile: "/home/airflow/.env"} t.Run("success", func(t *testing.T) { @@ -702,7 +710,7 @@ func TestDockerComposeExport(t *testing.T) { } func TestDockerComposeStop(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("success", func(t *testing.T) { imageHandler := new(mocks.ImageHandler) @@ -828,7 +836,7 @@ func TestDockerComposeStop(t *testing.T) { } func TestDockerComposePS(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("success", func(t *testing.T) { composeMock := new(mocks.DockerComposeAPI) @@ -865,7 +873,7 @@ func TestDockerComposePS(t *testing.T) { } func TestDockerComposeKill(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("success", func(t *testing.T) { composeMock := new(mocks.DockerComposeAPI) @@ -891,7 +899,7 @@ func TestDockerComposeKill(t *testing.T) { } func TestDockerComposeLogs(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} containerNames := []string{WebserverDockerContainerName, SchedulerDockerContainerName, TriggererDockerContainerName} follow := false @@ -943,7 +951,7 @@ func TestDockerComposeLogs(t *testing.T) { } func TestDockerComposeRun(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("success", func(t *testing.T) { testCmd := []string{"test", "command"} @@ -1006,12 +1014,12 @@ func TestDockerComposeRun(t *testing.T) { } func TestDockerComposePytest(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("success", func(t *testing.T) { imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, []string{}, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() mockDockerCompose.imageHandler = imageHandler @@ -1022,10 +1030,24 @@ func TestDockerComposePytest(t *testing.T) { imageHandler.AssertExpectations(t) }) + t.Run("success custom image", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("TagLocalImage", mock.Anything).Return(nil) + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() + + mockDockerCompose.imageHandler = imageHandler + + resp, err := mockDockerCompose.Pytest("", "custom-image-name", "", "") + + assert.NoError(t, err) + assert.Equal(t, "", resp) + imageHandler.AssertExpectations(t) + }) + t.Run("unexpected exit code", func(t *testing.T) { imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("1", nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("1", nil).Once() mockResponse := "1" mockDockerCompose.imageHandler = imageHandler @@ -1038,7 +1060,7 @@ func TestDockerComposePytest(t *testing.T) { t.Run("image build failure", func(t *testing.T) { imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(errMockDocker).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(errMockDocker).Once() mockDockerCompose.imageHandler = imageHandler @@ -1048,14 +1070,237 @@ func TestDockerComposePytest(t *testing.T) { }) } +func TestDockerComposedUpgradeTest(t *testing.T) { + testUtil.InitTestConfig(testUtil.CloudPlatform) + cwd, err := os.Getwd() + assert.NoError(t, err) + mockDockerCompose := DockerCompose{projectName: "test", dockerfile: "Dockerfile", airflowHome: cwd} + + pipFreeze := "upgrade-test-old-version--new-version/pip_freeze_old-version.txt" + pipFreeze2 := "upgrade-test-old-version--new-version/pip_freeze_new-version.txt" + parseTest := cwd + "/.astro/test_dag_integrity_default.py" + oldDockerFile := cwd + "/Dockerfile" + // Write files out + err = fileutil.WriteStringToFile(pipFreeze, pipFreezeFile) + assert.NoError(t, err) + err = fileutil.WriteStringToFile(pipFreeze2, pipFreezeFile2) + assert.NoError(t, err) + err = fileutil.WriteStringToFile(parseTest, "") + assert.NoError(t, err) + err = fileutil.WriteStringToFile(oldDockerFile, "") + assert.NoError(t, err) + + defer afero.NewOsFs().Remove(pipFreeze) + defer afero.NewOsFs().Remove(pipFreeze2) + defer afero.NewOsFs().Remove(parseTest) + defer afero.NewOsFs().Remove("upgrade-test-old-version--new-version/Dockerfile") + defer afero.NewOsFs().Remove("upgrade-test-old-version--new-version/dependency_compare.txt") + defer afero.NewOsFs().Remove("upgrade-test-old-version--new-version") + defer afero.NewOsFs().Remove(oldDockerFile) + + t.Run("success no deployment id", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Times(3) + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze2).Return(nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() + + mockDockerCompose.imageHandler = imageHandler + + err := mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + + assert.NoError(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("success with deployment id", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + mockClient := new(astro_mocks.Client) + mockClient.On("ListDeployments", mock.Anything, mock.Anything).Return([]astro.Deployment{{ID: "deployment-id"}}, nil).Once() + imageHandler.On("Pull", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Times(2) + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze2).Return(nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() + + mockDockerCompose.imageHandler = imageHandler + + err := mockDockerCompose.UpgradeTest("new-version", "deployment-id", "", "", false, false, false, mockClient) + + assert.NoError(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("image build failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("GetLabel failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", errMockDocker) + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("GetLabel failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", errMockDocker) + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("ConflictTest failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("Create old pip freeze failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("build new image failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Once() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(nil).Once() + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("build new image for pytest failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Twice() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze2).Return(nil).Once() + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("pytest failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Times(3) + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze2).Return(nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("get deployments failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + mockClient := new(astro_mocks.Client) + mockClient.On("ListDeployments", mock.Anything, mock.Anything).Return([]astro.Deployment{{ID: "deployment-id"}}, errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "deployment-id", "", "", false, false, false, mockClient) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("image pull failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + mockClient := new(astro_mocks.Client) + mockClient.On("ListDeployments", mock.Anything, mock.Anything).Return([]astro.Deployment{{ID: "deployment-id"}}, nil).Once() + imageHandler.On("Pull", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errMockDocker) + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "deployment-id", "", "", false, false, false, mockClient) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("build new image failure", func(t *testing.T) { + imageHandler := new(mocks.ImageHandler) + imageHandler.On("Build", mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return(nil).Twice() + imageHandler.On("GetLabel", mock.Anything, mock.Anything).Return("old-version", nil) + imageHandler.On("ConflictTest", mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("", nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze).Return(nil).Once() + imageHandler.On("CreatePipFreeze", mock.Anything, cwd+"/"+pipFreeze2).Return(errMockDocker).Once() + + mockDockerCompose.imageHandler = imageHandler + + err = mockDockerCompose.UpgradeTest("new-version", "", "", "", false, false, false, nil) + assert.Error(t, err) + imageHandler.AssertExpectations(t) + }) + + t.Run("no domain", func(t *testing.T) { + err := config.ResetCurrentContext() + assert.NoError(t, err) + + err = mockDockerCompose.UpgradeTest("new-version", "deployment-id", "", "", false, false, false, nil) + assert.Error(t, err) + }) +} + func TestDockerComposeParse(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test", airflowHome: "./testfiles"} t.Run("success", func(t *testing.T) { DefaultTestPath = "test_dag_integrity_file.py" imageHandler := new(mocks.ImageHandler) - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("0", nil).Once() composeMock := new(mocks.DockerComposeAPI) mockDockerCompose.composeService = composeMock @@ -1071,7 +1316,7 @@ func TestDockerComposeParse(t *testing.T) { DefaultTestPath = "test_dag_integrity_file.py" imageHandler := new(mocks.ImageHandler) - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("1", nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, []string{}, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("1", nil).Once() composeMock := new(mocks.DockerComposeAPI) mockDockerCompose.composeService = composeMock @@ -1087,7 +1332,7 @@ func TestDockerComposeParse(t *testing.T) { DefaultTestPath = "test_dag_integrity_file.py" imageHandler := new(mocks.ImageHandler) - imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("2", nil).Once() + imageHandler.On("Pytest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: false}).Return("2", nil).Once() composeMock := new(mocks.DockerComposeAPI) mockDockerCompose.composeService = composeMock @@ -1125,7 +1370,7 @@ func TestDockerComposeParse(t *testing.T) { } func TestDockerComposeBash(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} container := "scheduler" t.Run("success", func(t *testing.T) { @@ -1178,7 +1423,7 @@ func TestDockerComposeBash(t *testing.T) { } func TestDockerComposeSettings(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("import success", func(t *testing.T) { composeMock := new(mocks.DockerComposeAPI) @@ -1384,7 +1629,7 @@ func TestDockerComposeSettings(t *testing.T) { } func TestDockerComposeRunDAG(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) mockDockerCompose := DockerCompose{projectName: "test"} t.Run("success with container", func(t *testing.T) { noCache := false @@ -1425,7 +1670,7 @@ func TestDockerComposeRunDAG(t *testing.T) { t.Run("success without container", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("Run", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() composeMock := new(mocks.DockerComposeAPI) @@ -1444,7 +1689,7 @@ func TestDockerComposeRunDAG(t *testing.T) { t.Run("error without container", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(nil).Once() imageHandler.On("Run", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errMockDocker).Once() composeMock := new(mocks.DockerComposeAPI) @@ -1463,7 +1708,7 @@ func TestDockerComposeRunDAG(t *testing.T) { t.Run("build error without container", func(t *testing.T) { noCache := false imageHandler := new(mocks.ImageHandler) - imageHandler.On("Build", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(errMockDocker).Once() + imageHandler.On("Build", "", airflowTypes.ImageBuildConfig{Path: mockDockerCompose.airflowHome, Output: true, NoCache: noCache}).Return(errMockDocker).Once() composeMock := new(mocks.DockerComposeAPI) composeMock.On("Ps", mock.Anything, mockDockerCompose.projectName, api.PsOptions{All: true}).Return([]api.ContainerSummary{}, nil).Once() @@ -1493,7 +1738,7 @@ func TestDockerComposeRunDAG(t *testing.T) { } func TestCheckWebserverHealth(t *testing.T) { - testUtils.InitTestConfig(testUtils.LocalPlatform) + testUtil.InitTestConfig(testUtil.LocalPlatform) t.Run("success", func(t *testing.T) { settingsFile := "docker_test.go" // any file which exists composeMock := new(mocks.DockerComposeAPI) @@ -1696,7 +1941,7 @@ func TestStartDocker(t *testing.T) { func TestCreateDockerProject(t *testing.T) { fs := afero.NewMemMapFs() - configYaml := testUtils.NewTestConfig(testUtils.LocalPlatform) + configYaml := testUtil.NewTestConfig(testUtil.LocalPlatform) err := afero.WriteFile(fs, config.HomeConfigFile, configYaml, 0o777) assert.NoError(t, err) config.InitConfig(fs) diff --git a/airflow/include/test-conflicts.dockerfile b/airflow/include/test-conflicts.dockerfile new file mode 100644 index 000000000..b04fcd805 --- /dev/null +++ b/airflow/include/test-conflicts.dockerfile @@ -0,0 +1,7 @@ +FROM quay.io/astronomer/%s:%s +USER root +RUN pip install pip-tools +RUN pip freeze > req.txt +RUN cat requirements.txt >> req.txt +RUN sed -i '/\.whl/d' req.txt +RUN python -m piptools compile --verbose req.txt -o conflict-test-results.txt diff --git a/airflow/mocks/ContainerHandler.go b/airflow/mocks/ContainerHandler.go index 96ad9e7d3..56cbe6c36 100644 --- a/airflow/mocks/ContainerHandler.go +++ b/airflow/mocks/ContainerHandler.go @@ -1,11 +1,12 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package mocks import ( - time "time" - + astro "github.com/astronomer/astro-cli/astro-client" mock "github.com/stretchr/testify/mock" + + time "time" ) // ContainerHandler is an autogenerated mock type for the ContainerHandler type @@ -212,12 +213,27 @@ func (_m *ContainerHandler) Stop(waitForExit bool) error { return r0 } -// NewContainerHandler creates a new instance of ContainerHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewContainerHandler(t interface { +// UpgradeTest provides a mock function with given fields: runtimeVersion, deploymentID, newImageName, customImageName, dependencyTest, versionTest, dagTest, client +func (_m *ContainerHandler) UpgradeTest(runtimeVersion string, deploymentID string, newImageName string, customImageName string, dependencyTest bool, versionTest bool, dagTest bool, client astro.Client) error { + ret := _m.Called(runtimeVersion, deploymentID, newImageName, customImageName, dependencyTest, versionTest, dagTest, client) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string, string, bool, bool, bool, astro.Client) error); ok { + r0 = rf(runtimeVersion, deploymentID, newImageName, customImageName, dependencyTest, versionTest, dagTest, client) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewContainerHandler interface { mock.TestingT Cleanup(func()) -}) *ContainerHandler { +} + +// NewContainerHandler creates a new instance of ContainerHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewContainerHandler(t mockConstructorTestingTNewContainerHandler) *ContainerHandler { mock := &ContainerHandler{} mock.Mock.Test(t) diff --git a/airflow/mocks/DockerCLIClient.go b/airflow/mocks/DockerCLIClient.go index 575b37133..e40729cdd 100644 --- a/airflow/mocks/DockerCLIClient.go +++ b/airflow/mocks/DockerCLIClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package mocks @@ -2642,12 +2642,13 @@ func (_m *DockerCLIClient) VolumesPrune(ctx context.Context, pruneFilter filters return r0, r1 } -// NewDockerCLIClient creates a new instance of DockerCLIClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewDockerCLIClient(t interface { +type mockConstructorTestingTNewDockerCLIClient interface { mock.TestingT Cleanup(func()) -}) *DockerCLIClient { +} + +// NewDockerCLIClient creates a new instance of DockerCLIClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDockerCLIClient(t mockConstructorTestingTNewDockerCLIClient) *DockerCLIClient { mock := &DockerCLIClient{} mock.Mock.Test(t) diff --git a/airflow/mocks/DockerComposeAPI.go b/airflow/mocks/DockerComposeAPI.go index 71bf649bb..53c51c9df 100644 --- a/airflow/mocks/DockerComposeAPI.go +++ b/airflow/mocks/DockerComposeAPI.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package mocks @@ -450,12 +450,13 @@ func (_m *DockerComposeAPI) Up(ctx context.Context, project *types.Project, opti return r0 } -// NewDockerComposeAPI creates a new instance of DockerComposeAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewDockerComposeAPI(t interface { +type mockConstructorTestingTNewDockerComposeAPI interface { mock.TestingT Cleanup(func()) -}) *DockerComposeAPI { +} + +// NewDockerComposeAPI creates a new instance of DockerComposeAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDockerComposeAPI(t mockConstructorTestingTNewDockerComposeAPI) *DockerComposeAPI { mock := &DockerComposeAPI{} mock.Mock.Test(t) diff --git a/airflow/mocks/DockerRegistryAPI.go b/airflow/mocks/DockerRegistryAPI.go index 1981b4798..1b4c4ba04 100644 --- a/airflow/mocks/DockerRegistryAPI.go +++ b/airflow/mocks/DockerRegistryAPI.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package mocks @@ -2588,12 +2588,13 @@ func (_m *DockerRegistryAPI) VolumesPrune(ctx context.Context, pruneFilter filte return r0, r1 } -// NewDockerRegistryAPI creates a new instance of DockerRegistryAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewDockerRegistryAPI(t interface { +type mockConstructorTestingTNewDockerRegistryAPI interface { mock.TestingT Cleanup(func()) -}) *DockerRegistryAPI { +} + +// NewDockerRegistryAPI creates a new instance of DockerRegistryAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDockerRegistryAPI(t mockConstructorTestingTNewDockerRegistryAPI) *DockerRegistryAPI { mock := &DockerRegistryAPI{} mock.Mock.Test(t) diff --git a/airflow/mocks/ImageHandler.go b/airflow/mocks/ImageHandler.go index 5fc875247..a70b41876 100644 --- a/airflow/mocks/ImageHandler.go +++ b/airflow/mocks/ImageHandler.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package mocks @@ -12,13 +12,13 @@ type ImageHandler struct { mock.Mock } -// Build provides a mock function with given fields: config -func (_m *ImageHandler) Build(config types.ImageBuildConfig) error { - ret := _m.Called(config) +// Build provides a mock function with given fields: dockerfile, config +func (_m *ImageHandler) Build(dockerfile string, config types.ImageBuildConfig) error { + ret := _m.Called(dockerfile, config) var r0 error - if rf, ok := ret.Get(0).(func(types.ImageBuildConfig) error); ok { - r0 = rf(config) + if rf, ok := ret.Get(0).(func(string, types.ImageBuildConfig) error); ok { + r0 = rf(dockerfile, config) } else { r0 = ret.Error(0) } @@ -26,23 +26,61 @@ func (_m *ImageHandler) Build(config types.ImageBuildConfig) error { return r0 } -// GetLabel provides a mock function with given fields: labelName -func (_m *ImageHandler) GetLabel(labelName string) (string, error) { - ret := _m.Called(labelName) +// ConflictTest provides a mock function with given fields: workingDirectory, testHomeDirectory, buildConfig +func (_m *ImageHandler) ConflictTest(workingDirectory string, testHomeDirectory string, buildConfig types.ImageBuildConfig) (string, error) { + ret := _m.Called(workingDirectory, testHomeDirectory, buildConfig) var r0 string var r1 error - if rf, ok := ret.Get(0).(func(string) (string, error)); ok { - return rf(labelName) + if rf, ok := ret.Get(0).(func(string, string, types.ImageBuildConfig) (string, error)); ok { + return rf(workingDirectory, testHomeDirectory, buildConfig) } - if rf, ok := ret.Get(0).(func(string) string); ok { - r0 = rf(labelName) + if rf, ok := ret.Get(0).(func(string, string, types.ImageBuildConfig) string); ok { + r0 = rf(workingDirectory, testHomeDirectory, buildConfig) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(labelName) + if rf, ok := ret.Get(1).(func(string, string, types.ImageBuildConfig) error); ok { + r1 = rf(workingDirectory, testHomeDirectory, buildConfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreatePipFreeze provides a mock function with given fields: altImageName, pipFreezeFile +func (_m *ImageHandler) CreatePipFreeze(altImageName string, pipFreezeFile string) error { + ret := _m.Called(altImageName, pipFreezeFile) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(altImageName, pipFreezeFile) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetLabel provides a mock function with given fields: altImageName, labelName +func (_m *ImageHandler) GetLabel(altImageName string, labelName string) (string, error) { + ret := _m.Called(altImageName, labelName) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (string, error)); ok { + return rf(altImageName, labelName) + } + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(altImageName, labelName) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(altImageName, labelName) } else { r1 = ret.Error(1) } @@ -76,6 +114,20 @@ func (_m *ImageHandler) ListLabels() (map[string]string, error) { return r0, r1 } +// Pull provides a mock function with given fields: registry, username, token, remoteImage +func (_m *ImageHandler) Pull(registry string, username string, token string, remoteImage string) error { + ret := _m.Called(registry, username, token, remoteImage) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string, string) error); ok { + r0 = rf(registry, username, token, remoteImage) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Push provides a mock function with given fields: registry, username, token, remoteImage func (_m *ImageHandler) Push(registry string, username string, token string, remoteImage string) error { ret := _m.Called(registry, username, token, remoteImage) @@ -90,23 +142,23 @@ func (_m *ImageHandler) Push(registry string, username string, token string, rem return r0 } -// Pytest provides a mock function with given fields: pytestFile, airflowHome, envFile, pytestArgs, config -func (_m *ImageHandler) Pytest(pytestFile string, airflowHome string, envFile string, pytestArgs []string, config types.ImageBuildConfig) (string, error) { - ret := _m.Called(pytestFile, airflowHome, envFile, pytestArgs, config) +// Pytest provides a mock function with given fields: pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config +func (_m *ImageHandler) Pytest(pytestFile string, airflowHome string, envFile string, testHomeDirectory string, pytestArgs []string, htmlReport bool, config types.ImageBuildConfig) (string, error) { + ret := _m.Called(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) var r0 string var r1 error - if rf, ok := ret.Get(0).(func(string, string, string, []string, types.ImageBuildConfig) (string, error)); ok { - return rf(pytestFile, airflowHome, envFile, pytestArgs, config) + if rf, ok := ret.Get(0).(func(string, string, string, string, []string, bool, types.ImageBuildConfig) (string, error)); ok { + return rf(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) } - if rf, ok := ret.Get(0).(func(string, string, string, []string, types.ImageBuildConfig) string); ok { - r0 = rf(pytestFile, airflowHome, envFile, pytestArgs, config) + if rf, ok := ret.Get(0).(func(string, string, string, string, []string, bool, types.ImageBuildConfig) string); ok { + r0 = rf(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(string, string, string, []string, types.ImageBuildConfig) error); ok { - r1 = rf(pytestFile, airflowHome, envFile, pytestArgs, config) + if rf, ok := ret.Get(1).(func(string, string, string, string, []string, bool, types.ImageBuildConfig) error); ok { + r1 = rf(pytestFile, airflowHome, envFile, testHomeDirectory, pytestArgs, htmlReport, config) } else { r1 = ret.Error(1) } @@ -142,12 +194,13 @@ func (_m *ImageHandler) TagLocalImage(localImage string) error { return r0 } -// NewImageHandler creates a new instance of ImageHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewImageHandler(t interface { +type mockConstructorTestingTNewImageHandler interface { mock.TestingT Cleanup(func()) -}) *ImageHandler { +} + +// NewImageHandler creates a new instance of ImageHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewImageHandler(t mockConstructorTestingTNewImageHandler) *ImageHandler { mock := &ImageHandler{} mock.Mock.Test(t) diff --git a/airflow/mocks/RegistryHandler.go b/airflow/mocks/RegistryHandler.go index 49eb516a9..15caacd7f 100644 --- a/airflow/mocks/RegistryHandler.go +++ b/airflow/mocks/RegistryHandler.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package mocks @@ -23,12 +23,13 @@ func (_m *RegistryHandler) Login(username string, token string) error { return r0 } -// NewRegistryHandler creates a new instance of RegistryHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewRegistryHandler(t interface { +type mockConstructorTestingTNewRegistryHandler interface { mock.TestingT Cleanup(func()) -}) *RegistryHandler { +} + +// NewRegistryHandler creates a new instance of RegistryHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRegistryHandler(t mockConstructorTestingTNewRegistryHandler) *RegistryHandler { mock := &RegistryHandler{} mock.Mock.Test(t) diff --git a/airflow/testfiles/pip_freeze_new-version.txt b/airflow/testfiles/pip_freeze_new-version.txt new file mode 100644 index 000000000..123380837 --- /dev/null +++ b/airflow/testfiles/pip_freeze_new-version.txt @@ -0,0 +1,320 @@ +adal==1.2.7 +aiobotocore==2.5.2 +aiofiles==23.1.0 +aiohttp==3.8.4 +aioitertools==0.11.0 +aiosignal==1.3.1 +alembic==1.11.1 +amqp==5.1.1 +anyio==3.7.1 +apache-airflow==2.6.3+astro.1 +apache-airflow-providers-amazon==8.3.0 +apache-airflow-providers-celery==3.2.1 +apache-airflow-providers-cncf-kubernetes==7.2.0 +apache-airflow-providers-common-sql==1.6.0 +apache-airflow-providers-datadog==3.3.1 +apache-airflow-providers-elasticsearch==4.5.1 +apache-airflow-providers-ftp==3.4.2 +apache-airflow-providers-google==10.0.0 +apache-airflow-providers-http==4.4.2 +apache-airflow-providers-imap==3.2.2 +apache-airflow-providers-microsoft-azure==6.2.0 +apache-airflow-providers-postgres==5.5.2 +apache-airflow-providers-redis==3.2.1 +apache-airflow-providers-sqlite==3.4.2 +apispec==5.2.2 +argcomplete==3.1.1 +asgiref==3.7.2 +asn1crypto==1.5.1 +astro-sdk-python==1.6.1 +astronomer-airflow-scripts @ https://github.com/astronomer/astronomer-airflow-scripts/releases/download/v0.0.6/astronomer_airflow_scripts-0.0.6-py3-none-any.whl#sha256=b099d655e4f4a85b51a03ef35df689e9bc072d1ff20d9bff5b772a76911c2a25 +astronomer-airflow-version-check==2.0.0 +astronomer-analytics-plugin @ https://github.com/astronomer/airflow-analytics-plugin/releases/download/1.0.3/astronomer_analytics_plugin-1.0.3-py3-none-any.whl#sha256=a6045c3d7c4aa44f4e82959062723046a1cfae3518504a4f190a1ab282ed2ead +astronomer-dbcleanup-plugin @ https://github.com/astronomer/airflow-dbcleanup-plugin/releases/download/1.0.1/astronomer_dbcleanup_plugin-1.0.1-py3-none-any.whl#sha256=af8f5bd57a6c0661b2bc09e3f97b43138c6b90ef26b51e57638f814f51f1b8d3 +astronomer-fab-security-manager==1.9.4 +astronomer-providers==1.17.1 +astronomer-runtime-extensions @ file:///tmp/wheels/astronomer_runtime_extensions-1.0.0-py3-none-any.whl#sha256=5ab3be6c7256cfe4de504a069654b2a4e369f231fc5c7f4bd273e0bf311b4143 +async-timeout==4.0.2 +attrs==23.1.0 +azure-batch==14.0.0 +azure-common==1.1.28 +azure-core==1.28.0 +azure-cosmos==4.4.0 +azure-datalake-store==0.0.53 +azure-identity==1.13.0 +azure-keyvault-secrets==4.7.0 +azure-kusto-data==0.0.45 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-core==1.4.0 +azure-mgmt-datafactory==1.1.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==23.0.1 +azure-nspkg==3.0.2 +azure-servicebus==7.11.0 +azure-storage-blob==12.16.0 +azure-storage-common==2.1.0 +azure-storage-file==2.1.0 +azure-storage-file-datalake==12.11.0 +azure-synapse-spark==0.7.0 +Babel==2.12.1 +backoff==1.10.0 +bcrypt==4.0.1 +beautifulsoup4==4.12.2 +billiard==4.1.0 +blinker==1.6.2 +boto3==1.26.161 +botocore==1.29.161 +cachelib==0.9.0 +cachetools==5.3.1 +cattrs==23.1.2 +celery==5.3.1 +certifi==2023.5.7 +cffi==1.15.1 +chardet==5.1.0 +charset-normalizer==3.1.0 +click==8.1.4 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 +clickclick==20.10.2 +colorama==0.4.6 +colorlog==4.8.0 +ConfigUpdater==3.1.1 +connexion==2.14.2 +cron-descriptor==1.4.0 +croniter==1.4.1 +cryptography==40.0.2 +datadog==0.45.0 +db-dtypes==1.1.1 +decorator==5.1.1 +Deprecated==1.2.14 +dill==0.3.1.1 +distlib==0.3.6 +distro==1.8.0 +dnspython==2.3.0 +docutils==0.20.1 +elasticsearch==7.13.4 +elasticsearch-dbapi==0.2.10 +elasticsearch-dsl==7.4.1 +email-validator==1.3.1 +exceptiongroup==1.1.2 +exchange-calendars==4.2.8 +filelock==3.12.2 +Flask==2.2.5 +Flask-AppBuilder==4.3.1 +Flask-Babel==2.0.0 +Flask-Bcrypt==1.0.1 +Flask-Caching==2.0.2 +Flask-JWT-Extended==4.5.2 +Flask-Limiter==3.3.1 +Flask-Login==0.6.2 +Flask-Session==0.5.0 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==1.1.1 +flower==2.0.0 +frozenlist==1.3.3 +fsspec==2023.6.0 +future==0.18.3 +gcloud-aio-auth==4.2.3 +gcloud-aio-bigquery==6.3.0 +gcloud-aio-storage==8.2.0 +google-ads==21.2.0 +google-api-core==2.8.2 +google-api-python-client==1.12.11 +google-auth==2.21.0 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.8.0 +google-cloud-aiplatform==1.16.1 +google-cloud-appengine-logging==1.1.3 +google-cloud-audit-log==0.2.4 +google-cloud-automl==2.8.0 +google-cloud-bigquery==2.34.4 +google-cloud-bigquery-datatransfer==3.7.0 +google-cloud-bigquery-storage==2.14.1 +google-cloud-bigtable==2.11.1 +google-cloud-build==3.9.0 +google-cloud-compute==0.7.0 +google-cloud-container==2.11.1 +google-cloud-core==2.3.3 +google-cloud-datacatalog==3.9.0 +google-cloud-dataflow-client==0.5.4 +google-cloud-dataform==0.2.0 +google-cloud-dataplex==1.1.0 +google-cloud-dataproc==5.0.0 +google-cloud-dataproc-metastore==1.6.0 +google-cloud-dlp==3.8.0 +google-cloud-kms==2.12.0 +google-cloud-language==1.3.2 +google-cloud-logging==3.2.1 +google-cloud-memcache==1.4.1 +google-cloud-monitoring==2.11.0 +google-cloud-orchestration-airflow==1.4.1 +google-cloud-os-login==2.7.1 +google-cloud-pubsub==2.13.5 +google-cloud-redis==2.9.0 +google-cloud-resource-manager==1.6.0 +google-cloud-secret-manager==1.0.2 +google-cloud-spanner==1.19.3 +google-cloud-speech==1.3.4 +google-cloud-storage==2.10.0 +google-cloud-tasks==2.10.1 +google-cloud-texttospeech==1.0.3 +google-cloud-translate==1.7.2 +google-cloud-videointelligence==1.16.3 +google-cloud-vision==1.0.2 +google-cloud-workflows==1.7.1 +google-crc32c==1.5.0 +google-re2==1.0 +google-resumable-media==2.5.0 +googleapis-common-protos==1.56.4 +graphviz==0.20.1 +greenlet==2.0.2 +grpc-google-iam-v1==0.12.4 +grpcio==1.56.0 +grpcio-gcp==0.2.2 +grpcio-status==1.48.2 +gunicorn==20.1.0 +h11==0.14.0 +httpcore==0.16.3 +httplib2==0.22.0 +httpx==0.23.3 +humanize==4.7.0 +idna==3.4 +importlib-resources==5.12.0 +inflection==0.5.1 +iniconfig==2.0.0 +isodate==0.6.1 +itsdangerous==2.1.2 +Jinja2==3.1.2 +jmespath==0.10.0 +json-merge-patch==0.2 +jsonpath-ng==1.5.3 +jsonschema==4.18.0 +jsonschema-specifications==2023.6.1 +jwcrypto==1.5.0 +kombu==5.3.1 +korean-lunar-calendar==0.3.1 +kubernetes==23.6.0 +kubernetes-asyncio==24.2.3 +lazy-object-proxy==1.9.0 +limits==3.5.0 +linkify-it-py==2.0.2 +lockfile==0.12.2 +looker-sdk==23.10.0 +lxml==4.9.3 +Mako==1.2.4 +Markdown==3.4.3 +markdown-it-py==3.0.0 +MarkupSafe==2.1.3 +marshmallow==3.19.0 +marshmallow-enum==1.5.1 +marshmallow-oneofschema==3.0.1 +marshmallow-sqlalchemy==0.26.1 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +msal==1.22.0 +msal-extensions==1.0.0 +msrest==0.7.1 +msrestazure==0.6.4 +multidict==6.0.4 +mypy-boto3-appflow==1.28.0 +mypy-boto3-rds==1.28.0 +mypy-boto3-redshift-data==1.28.0 +mypy-boto3-s3==1.28.0 +numpy==1.24.4 +oauthlib==3.2.2 +openlineage-airflow==0.29.2 +openlineage-integration-common==0.29.2 +openlineage-python==0.29.2 +openlineage_sql==0.29.2 +ordered-set==4.1.0 +packaging==21.3 +pandas==1.5.3 +pandas-gbq==0.17.9 +pandas-market-calendars==3.5 +pathspec==0.9.0 +pendulum==2.1.2 +platformdirs==3.8.1 +pluggy==1.2.0 +ply==3.11 +portalocker==2.7.0 +prison==0.2.1 +prometheus-client==0.17.0 +prompt-toolkit==3.0.39 +proto-plus==1.19.6 +protobuf==3.20.0 +psutil==5.9.5 +psycopg2-binary==2.9.6 +pyarrow==9.0.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.21 +pydantic==1.10.11 +pydata-google-auth==1.8.0 +Pygments==2.15.1 +PyJWT==2.4.0 +pyluach==2.2.0 +pyOpenSSL==23.2.0 +pyparsing==3.1.0 +pytest==7.4.0 +python-daemon==3.0.1 +python-dateutil==2.8.2 +python-frontmatter==1.0.0 +python-nvd3==0.15.0 +python-slugify==8.0.1 +pytz==2023.3 +pytzdata==2020.1 +PyYAML==6.0 +redis==4.6.0 +redshift-connector==2.0.912 +referencing==0.29.1 +requests==2.31.0 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +rfc3339-validator==0.1.4 +rfc3986==1.5.0 +rich==13.4.2 +rich-argparse==1.2.0 +rpds-py==0.8.8 +rsa==4.9 +s3fs==2023.6.0 +s3transfer==0.6.1 +scramp==1.4.4 +semver==3.0.1 +setproctitle==1.3.2 +Shapely==1.8.5.post1 +six==1.16.0 +smart-open==6.3.0 +sniffio==1.3.0 +soupsieve==2.4.1 +SQLAlchemy==1.4.49 +sqlalchemy-bigquery==1.6.1 +SQLAlchemy-JSONField==1.0.1.post0 +sqlalchemy-redshift==0.8.14 +SQLAlchemy-Utils==0.41.1 +sqlparse==0.4.4 +statsd==4.0.1 +tabulate==0.9.0 +tenacity==8.2.2 +termcolor==2.3.0 +text-unidecode==1.3 +tomli==2.0.1 +toolz==0.12.0 +tornado==6.3.2 +typing_extensions==4.7.1 +tzdata==2023.3 +uc-micro-py==1.0.2 +unicodecsv==0.14.1 +uritemplate==3.0.1 +urllib3==1.26.16 +vine==5.0.0 +virtualenv==20.23.1 +watchtower==2.0.1 +wcwidth==0.2.6 +websocket-client==1.6.1 +Werkzeug==2.2.3 +wrapt==1.15.0 +WTForms==3.0.1 +yarl==1.9.2 diff --git a/airflow/testfiles/pip_freeze_old-version.txt b/airflow/testfiles/pip_freeze_old-version.txt new file mode 100644 index 000000000..bc753fc86 --- /dev/null +++ b/airflow/testfiles/pip_freeze_old-version.txt @@ -0,0 +1,268 @@ +aiobotocore==2.4.1 +aiofiles==0.8.0 +aiohttp==3.8.3 +aioitertools==0.11.0 +aiosignal==1.3.1 +alembic==1.7.7 +amqp==5.1.0 +anyio==3.5.0 +apache-airflow==2.2.5+astro.6 +apache-airflow-providers-amazon==3.2.0 +apache-airflow-providers-celery==2.1.3 +apache-airflow-providers-cncf-kubernetes==3.0.0 +apache-airflow-providers-common-sql==1.3.1 +apache-airflow-providers-databricks==3.3.0 +apache-airflow-providers-elasticsearch==2.2.0 +apache-airflow-providers-ftp==2.1.2 +apache-airflow-providers-google==6.7.0 +apache-airflow-providers-http==2.1.2 +apache-airflow-providers-imap==2.2.3 +apache-airflow-providers-postgres==4.1.0 +apache-airflow-providers-redis==2.0.4 +apache-airflow-providers-snowflake==3.3.0 +apache-airflow-providers-sqlite==2.1.3 +apispec==3.3.2 +argcomplete==1.12.3 +asgiref==3.5.2 +asn1crypto==1.5.1 +astronomer-airflow-scripts @ https://github.com/astronomer/astronomer-airflow-scripts/releases/download/v0.0.5/astronomer_airflow_scripts-0.0.5-py3-none-any.whl +astronomer-fab-security-manager==1.9.3 +astronomer-providers==1.1.0 +astronomer-runtime-extensions @ file:///tmp/wheels/astronomer_runtime_extensions-1.0.0-py3-none-any.whl +async-timeout==4.0.2 +attrs==20.3.0 +Babel==2.9.1 +backoff==2.2.1 +bcrypt==3.2.0 +beautifulsoup4==4.10.0 +billiard==3.6.4.0 +blinker==1.4 +boto3==1.24.59 +botocore==1.27.59 +cachelib==0.6.0 +cachetools==4.2.2 +cattrs==1.10.0 +celery==5.2.3 +certifi==2020.12.5 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.12 +click==8.1.0 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.2.0 +clickclick==20.10.2 +colorama==0.4.4 +colorlog==4.8.0 +commonmark==0.9.1 +connexion==2.13.0 +croniter==1.3.4 +cryptography==36.0.2 +databricks-sql-connector==2.2.1 +db-dtypes==1.0.5 +decorator==5.1.1 +defusedxml==0.7.1 +Deprecated==1.2.13 +dill==0.3.1.1 +distlib==0.3.4 +dnspython==2.2.1 +docutils==0.16 +elasticsearch==7.13.4 +elasticsearch-dbapi==0.2.9 +elasticsearch-dsl==7.4.0 +email-validator==1.1.3 +exceptiongroup==1.0.4 +exchange-calendars==4.2.4 +filelock==3.6.0 +Flask==1.1.2 +Flask-AppBuilder==3.4.5 +Flask-Babel==2.0.0 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.10.1 +Flask-JWT-Extended==3.25.1 +Flask-Login==0.4.1 +Flask-OpenID==1.3.0 +Flask-Session==0.4.0 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==0.14.3 +flower==1.0.0 +frozenlist==1.3.3 +future==0.18.2 +gcloud-aio-auth==4.0.1 +gcloud-aio-bigquery==6.1.2 +gcloud-aio-storage==7.0.1 +google-ads==14.0.0 +google-api-core==1.31.5 +google-api-python-client==1.12.11 +google-auth==1.35.0 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.5.1 +google-cloud-aiplatform==1.11.0 +google-cloud-appengine-logging==1.1.1 +google-cloud-audit-log==0.2.0 +google-cloud-automl==2.7.2 +google-cloud-bigquery==2.34.2 +google-cloud-bigquery-datatransfer==3.6.1 +google-cloud-bigquery-storage==2.13.0 +google-cloud-bigtable==1.7.0 +google-cloud-build==3.8.1 +google-cloud-container==1.0.1 +google-cloud-core==1.7.2 +google-cloud-datacatalog==3.7.1 +google-cloud-dataplex==0.2.1 +google-cloud-dataproc==4.0.1 +google-cloud-dataproc-metastore==1.5.0 +google-cloud-dlp==1.0.0 +google-cloud-kms==2.11.1 +google-cloud-language==1.3.0 +google-cloud-logging==3.0.0 +google-cloud-memcache==1.3.1 +google-cloud-monitoring==2.9.1 +google-cloud-orchestration-airflow==1.3.1 +google-cloud-os-login==2.6.1 +google-cloud-pubsub==2.11.0 +google-cloud-redis==2.8.0 +google-cloud-secret-manager==1.0.0 +google-cloud-spanner==1.19.1 +google-cloud-speech==1.3.2 +google-cloud-storage==1.44.0 +google-cloud-tasks==2.8.1 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==1.7.0 +google-cloud-videointelligence==1.16.1 +google-cloud-vision==1.0.0 +google-cloud-workflows==1.6.1 +google-crc32c==1.3.0 +google-resumable-media==2.3.2 +googleapis-common-protos==1.56.0 +graphviz==0.19.1 +grpc-google-iam-v1==0.12.3 +grpcio==1.45.0 +grpcio-gcp==0.2.2 +grpcio-status==1.45.0 +gunicorn==20.1.0 +h11==0.12.0 +httpcore==0.15.0 +httplib2==0.19.1 +httpx==0.23.0 +humanize==4.0.0 +idna==3.3 +importlib-metadata==4.11.3 +inflection==0.5.1 +iniconfig==1.1.1 +iso8601==1.0.2 +itsdangerous==1.1.0 +Jinja2==3.0.3 +jmespath==0.10.0 +json-merge-patch==0.2 +jsonpath-ng==1.5.3 +jsonschema==3.2.0 +jwcrypto==1.4.2 +kombu==5.2.4 +korean-lunar-calendar==0.3.1 +kubernetes==11.0.0 +kubernetes-asyncio==24.2.2 +lazy-object-proxy==1.4.3 +lockfile==0.12.2 +looker-sdk==22.4.0 +lxml==4.9.1 +lz4==4.0.2 +Mako==1.2.2 +Markdown==3.3.6 +MarkupSafe==2.0.1 +marshmallow==3.15.0 +marshmallow-enum==1.5.1 +marshmallow-oneofschema==3.0.1 +marshmallow-sqlalchemy==0.26.1 +multidict==6.0.3 +mypy-boto3-rds==1.21.27 +mypy-boto3-redshift-data==1.21.27 +nox==2020.12.31 +numpy==1.23.4 +oauthlib==3.2.0 +openlineage-airflow==0.6.2 +openlineage-integration-common==0.6.2 +openlineage-python==0.6.2 +oscrypto==1.3.0 +packaging==21.3 +pandas==1.3.5 +pandas-gbq==0.17.9 +pandas-market-calendars==3.5 +pendulum==2.1.2 +platformdirs==2.5.1 +pluggy==1.0.0 +ply==3.11 +prison==0.2.1 +prometheus-client==0.13.1 +prompt-toolkit==3.0.28 +proto-plus==1.18.1 +protobuf==3.19.4 +psutil==5.9.0 +psycopg2-binary==2.9.3 +py==1.11.0 +pyarrow==9.0.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.21 +pycryptodomex==3.16.0 +pydata-google-auth==1.4.0 +Pygments==2.11.2 +PyJWT==1.7.1 +pyluach==2.0.2 +pyOpenSSL==21.0.0 +pyparsing==2.4.7 +pyrsistent==0.18.1 +pytest==7.2.0 +python-daemon==2.3.0 +python-dateutil==2.8.2 +python-nvd3==0.15.0 +python-slugify==4.0.1 +python3-openid==3.2.0 +pytz==2021.3 +pytzdata==2020.1 +PyYAML==5.4.1 +redis==3.5.3 +redshift-connector==2.0.905 +requests==2.27.1 +requests-oauthlib==1.3.1 +rfc3986==1.5.0 +rich==12.0.1 +rsa==4.8 +s3transfer==0.6.0 +scramp==1.4.1 +setproctitle==1.2.2 +six==1.16.0 +sniffio==1.2.0 +snowflake-connector-python==2.8.3 +snowflake-sqlalchemy==1.2.4 +soupsieve==2.3.1 +SQLAlchemy==1.3.24 +sqlalchemy-bigquery==1.5.0 +SQLAlchemy-JSONField==1.0.0 +sqlalchemy-redshift==0.8.9 +SQLAlchemy-Utils==0.38.2 +sqlparse==0.4.3 +statsd==3.3.0 +swagger-ui-bundle==0.0.9 +tabulate==0.8.9 +tenacity==8.0.1 +termcolor==1.1.0 +text-unidecode==1.3 +thrift==0.16.0 +tomli==2.0.1 +toolz==0.12.0 +tornado==6.1 +typing_extensions==4.4.0 +unicodecsv==0.14.1 +uritemplate==3.0.1 +urllib3==1.26.9 +vine==5.0.0 +virtualenv==20.14.0 +watchtower==2.0.1 +wcwidth==0.2.5 +websocket-client==1.3.2 +Werkzeug==1.0.1 +wrapt==1.14.0 +WTForms==2.3.3 +yarl==1.8.2 +zipp==3.7.0 diff --git a/astro-client-core/mocks/client.go b/astro-client-core/mocks/client.go index 4c2743611..231f8e898 100644 --- a/astro-client-core/mocks/client.go +++ b/astro-client-core/mocks/client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package astrocore_mocks @@ -3845,12 +3845,13 @@ func (_m *ClientWithResponsesInterface) VerifyManagedDomainWithResponse(ctx cont return r0, r1 } -// NewClientWithResponsesInterface creates a new instance of ClientWithResponsesInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewClientWithResponsesInterface(t interface { +type mockConstructorTestingTNewClientWithResponsesInterface interface { mock.TestingT Cleanup(func()) -}) *ClientWithResponsesInterface { +} + +// NewClientWithResponsesInterface creates a new instance of ClientWithResponsesInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClientWithResponsesInterface(t mockConstructorTestingTNewClientWithResponsesInterface) *ClientWithResponsesInterface { mock := &ClientWithResponsesInterface{} mock.Mock.Test(t) diff --git a/astro-client/mocks/Client.go b/astro-client/mocks/Client.go index 90680be96..c93df6fd1 100644 --- a/astro-client/mocks/Client.go +++ b/astro-client/mocks/Client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package astro_mocks @@ -385,12 +385,13 @@ func (_m *Client) UpdateDeployment(input *astro.UpdateDeploymentInput) (astro.De return r0, r1 } -// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewClient(t interface { +type mockConstructorTestingTNewClient interface { mock.TestingT Cleanup(func()) -}) *Client { +} + +// NewClient creates a new instance of Client. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClient(t mockConstructorTestingTNewClient) *Client { mock := &Client{} mock.Mock.Test(t) diff --git a/cloud/deploy/deploy.go b/cloud/deploy/deploy.go index 2f79032d5..e20fcae72 100644 --- a/cloud/deploy/deploy.go +++ b/cloud/deploy/deploy.go @@ -33,7 +33,7 @@ const ( parse = "parse" astroDomain = "astronomer.io" registryUsername = "cli" - runtimeImageLabel = "io.astronomer.docker.runtime.version" + runtimeImageLabel = airflow.RuntimeImageLabel defaultRuntimeVersion = "4.2.5" dagParseAllowedVersion = "4.1.0" @@ -107,17 +107,6 @@ type InputDeploy struct { Description string } -func getRegistryURL(domain string) string { - var registry string - if domain == "localhost" { - registry = config.CFG.LocalRegistry.GetString() - } else { - registry = "images." + strings.Split(domain, ".")[0] + ".cloud" - } - - return registry -} - func removeDagsFromDockerIgnore(fullpath string) error { f, err := os.Open(fullpath) if err != nil { @@ -358,7 +347,7 @@ func Deploy(deployInput InputDeploy, client astro.Client, coreClient astrocore.C } nextTag := "deploy-" + time.Now().UTC().Format("2006-01-02T15-04") - registry := getRegistryURL(domain) + registry := airflow.GetRegistryURL(domain) repository := registry + "/" + deployInfo.organizationID + "/" + deployInfo.deploymentID // TODO: Resolve the edge case where two people push the same nextTag at the same time remoteImage := fmt.Sprintf("%s:%s", repository, nextTag) @@ -589,7 +578,7 @@ func buildImageWithoutDags(path string, imageHandler airflow.ImageHandler) error dagsIgnoreSet = true } - err = imageHandler.Build(types.ImageBuildConfig{Path: path, Output: true, TargetPlatforms: deployImagePlatformSupport}) + err = imageHandler.Build("", types.ImageBuildConfig{Path: path, Output: true, TargetPlatforms: deployImagePlatformSupport}) if err != nil { return err } @@ -618,7 +607,7 @@ func buildImage(path, currentVersion, deployImage, imageName string, dagDeployEn return "", err } } else { - err := imageHandler.Build(types.ImageBuildConfig{Path: path, Output: true, TargetPlatforms: deployImagePlatformSupport}) + err := imageHandler.Build("", types.ImageBuildConfig{Path: path, Output: true, TargetPlatforms: deployImagePlatformSupport}) if err != nil { return "", err } @@ -641,7 +630,7 @@ func buildImage(path, currentVersion, deployImage, imageName string, dagDeployEn DockerfileImage := docker.GetImageFromParsedFile(cmds) - version, err = imageHandler.GetLabel(runtimeImageLabel) + version, err = imageHandler.GetLabel("", runtimeImageLabel) if err != nil { fmt.Println("unable get runtime version from image") } diff --git a/cloud/deploy/deploy_test.go b/cloud/deploy/deploy_test.go index f6eae0af6..32945fdf5 100644 --- a/cloud/deploy/deploy_test.go +++ b/cloud/deploy/deploy_test.go @@ -68,9 +68,9 @@ func TestDeployWithoutDagsDeploySuccess(t *testing.T) { mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("", nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("", nil) mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) return mockImageHandler } @@ -235,9 +235,9 @@ func TestDeployWithDagsDeploySuccess(t *testing.T) { mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("", nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("", nil) mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) return mockImageHandler } @@ -404,9 +404,9 @@ func TestDagsDeploySuccess(t *testing.T) { // Test pytest with dags deploy mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("", nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("", nil) mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) return mockImageHandler } @@ -557,8 +557,8 @@ func TestDagsDeployFailed(t *testing.T) { mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("4.2.5", nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("4.2.5", nil) return mockImageHandler } @@ -632,8 +632,8 @@ func TestDeployFailure(t *testing.T) { mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("4.2.5", nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("4.2.5", nil) return mockImageHandler } @@ -759,9 +759,9 @@ func TestDeployMonitoringDAGNonHosted(t *testing.T) { // Test pytest with dags deploy mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("", nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("", nil) mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) return mockImageHandler } @@ -872,9 +872,9 @@ func TestDeployNoMonitoringDAGHosted(t *testing.T) { // Test pytest with dags deploy mockImageHandler := new(mocks.ImageHandler) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("", nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("", nil) mockImageHandler.On("TagLocalImage", mock.Anything).Return(nil) return mockImageHandler } @@ -921,8 +921,8 @@ func TestBuildImageFailure(t *testing.T) { assert.ErrorIs(t, err, errMock) airflowImageHandler = func(image string) airflow.ImageHandler { - mockImageHandler.On("Build", mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("4.2.5", nil) + mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) + mockImageHandler.On("GetLabel", mock.Anything, runtimeImageLabel).Return("4.2.5", nil) return mockImageHandler } diff --git a/cmd/airflow.go b/cmd/airflow.go index 7066b895f..8d7970232 100644 --- a/cmd/airflow.go +++ b/cmd/airflow.go @@ -9,6 +9,7 @@ import ( "github.com/astronomer/astro-cli/airflow" airflowversions "github.com/astronomer/astro-cli/airflow_versions" + astro "github.com/astronomer/astro-cli/astro-client" "github.com/astronomer/astro-cli/cmd/utils" "github.com/astronomer/astro-cli/config" "github.com/astronomer/astro-cli/context" @@ -35,6 +36,7 @@ var ( exportComposeFile string pytestArgs string pytestFile string + deploymentID string followLogs bool schedulerLogs bool webserverLogs bool @@ -50,6 +52,9 @@ var ( envExport bool noBrowser bool compose bool + conflictTest bool + versionTest bool + dagTest bool waitTime time.Duration RunExample = ` # Create default admin user. @@ -98,7 +103,7 @@ astro dev init --airflow-version 2.2.3 errPytestArgs = errors.New("") ) -func newDevRootCmd() *cobra.Command { +func newDevRootCmd(astroClient astro.Client) *cobra.Command { cmd := &cobra.Command{ Use: "dev", Aliases: []string{"d"}, @@ -119,6 +124,7 @@ func newDevRootCmd() *cobra.Command { newAirflowUpgradeCheckCmd(), newAirflowBashCmd(), newAirflowObjectRootCmd(), + newAirflowUpgradeTestCmd(astroClient), ) return cmd } @@ -162,6 +168,49 @@ func newAirflowInitCmd() *cobra.Command { return cmd } +func newAirflowUpgradeTestCmd(astroClient astro.Client) *cobra.Command { + cmd := &cobra.Command{ + Use: "upgrade-test", + Short: "Run tests to see if your environment and DAGs are compatible with a new version of Airflow or Astro Runtime. This test will produce a series of reports where you can see the test resluts.", + Long: "Run tests to see if your environment and DAGs are compatible with a new version of Airflow or Astro Runtime. This test will produce a series of reports where you can see the test resluts.", + // ignore PersistentPreRunE of root command + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return airflowUpgradeTest(cmd, astroClient) + }, + } + cmd.Flags().StringVarP(&airflowVersion, "airflow-version", "a", "", "The version of Airflow you want to upgrade to. The default is the latest available version. Tests are run against the equivalent Astro Runtime version. ") + cmd.Flags().BoolVarP(&conflictTest, "conflict-test", "c", false, "Only run conflict tests. These tests check whether you will have dependency conflicts after you upgrade.") + cmd.Flags().BoolVarP(&versionTest, "version-test", "", false, "Only run version tests. These tests show you how the versions of your dependencies will change after you upgrade.") + cmd.Flags().BoolVarP(&dagTest, "dag-test", "d", false, "Only run DAG tests. These tests check whether your DAGs will generate import errors after you upgrade.") + cmd.Flags().StringVarP(&deploymentID, "deployment-id", "i", "", "ID of the Deployment you want run dependency tests against.") + cmd.Flags().StringVarP(&customImageName, "image-name", "n", "", "Name of the upgraded image. Updates the FROM line in your Dockerfile to pull this image for the upgrade.") + var err error + var avoidACFlag bool + + // In case user is connected to Astronomer Platform and is connected to older version of platform + if context.IsCloudContext() || houstonVersion == "" || (!context.IsCloudContext() && houston.VerifyVersionMatch(houstonVersion, houston.VersionRestrictions{GTE: "0.29.0"})) { + cmd.Flags().StringVarP(&runtimeVersion, "runtime-version", "v", "", "The version of Astro Runtime you want to upgrade to. The default is the latest available version.") + } else { // default to using AC flag, since runtime is not available for these cases + useAstronomerCertified = true + avoidACFlag = true + } + + if !context.IsCloudContext() && !avoidACFlag { + cmd.Flags().BoolVarP(&useAstronomerCertified, "use-astronomer-certified", "", false, "Use an Astronomer Certified image instead of Astro Runtime. Use the airflow-version flag to specify your AC version.") + } + + _, err = context.GetCurrentContext() + if err != nil && !avoidACFlag { // Case when user is not logged in to any platform + cmd.Flags().BoolVarP(&useAstronomerCertified, "use-astronomer-certified", "", false, "Use an Astronomer Certified image instead of Astro Runtime. Use the airflow-version flag to specify your AC version.") + _ = cmd.Flags().MarkHidden("use-astronomer-certified") + } + + return cmd +} + func newAirflowStartCmd() *cobra.Command { cmd := &cobra.Command{ Use: "start", @@ -501,6 +550,51 @@ func airflowInit(cmd *cobra.Command, args []string) error { return nil } +func airflowUpgradeTest(cmd *cobra.Command, astroClient astro.Client) error { + // Validate runtimeVersion and airflowVersion + if airflowVersion != "" && runtimeVersion != "" { + return errInvalidBothAirflowAndRuntimeVersions + } + // If user provides a runtime version, use it, otherwise retrieve the latest one (matching Airflow Version if provided) + var err error + defaultImageTag := runtimeVersion + if defaultImageTag == "" { + httpClient := airflowversions.NewClient(httputil.NewHTTPClient(), useAstronomerCertified) + defaultImageTag = prepareDefaultAirflowImageTag(airflowVersion, httpClient) + } + + defaultImageName := airflow.AstroRuntimeImageName + if useAstronomerCertified { + defaultImageName = airflow.AstronomerCertifiedImageName + fmt.Printf("Testing an upgrade to Astronomer Certified Airflow version %s\n\n", defaultImageTag) + } else { + fmt.Printf("Testing an upgrade to Astro Runtime %s\n", defaultImageTag) + } + + // Silence Usage as we have now validated command input + cmd.SilenceUsage = true + + imageName := "tmp-upgrade-test" + + containerHandler, err := containerHandlerInit(config.WorkingPath, envFile, dockerfile, imageName) + if err != nil { + return err + } + + // add upgrade-test* to the gitignore + err = fileutil.AddLineToFile("./.gitignore", "upgrade-test*", "") + if err != nil { + fmt.Printf("failed to add 'upgrade-test*' to .gitignore: %s", err.Error()) + } + + err = containerHandler.UpgradeTest(defaultImageTag, deploymentID, defaultImageName, customImageName, conflictTest, versionTest, dagTest, astroClient) + if err != nil { + return err + } + + return nil +} + // Start an airflow cluster func airflowStart(cmd *cobra.Command, args []string) error { // Silence Usage as we have now validated command input diff --git a/cmd/airflow_test.go b/cmd/airflow_test.go index 6005660c8..91d20ff65 100644 --- a/cmd/airflow_test.go +++ b/cmd/airflow_test.go @@ -15,6 +15,7 @@ import ( testUtil "github.com/astronomer/astro-cli/pkg/testing" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) var errMock = errors.New("mock error") @@ -442,6 +443,48 @@ func TestAirflowStart(t *testing.T) { }) } +func TestAirflowUpgradeTest(t *testing.T) { + testUtil.InitTestConfig(testUtil.CloudPlatform) + t.Run("success", func(t *testing.T) { + cmd := newAirflowUpgradeTestCmd(nil) + + mockContainerHandler := new(mocks.ContainerHandler) + containerHandlerInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.ContainerHandler, error) { + mockContainerHandler.On("UpgradeTest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false, false, false, nil).Return(nil).Once() + return mockContainerHandler, nil + } + + err := airflowUpgradeTest(cmd, nil) + assert.NoError(t, err) + mockContainerHandler.AssertExpectations(t) + }) + + t.Run("failure", func(t *testing.T) { + cmd := newAirflowUpgradeTestCmd(nil) + + mockContainerHandler := new(mocks.ContainerHandler) + containerHandlerInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.ContainerHandler, error) { + mockContainerHandler.On("UpgradeTest", mock.Anything, mock.Anything, mock.Anything, mock.Anything, false, false, false, nil).Return(errMock).Once() + return mockContainerHandler, nil + } + + err := airflowUpgradeTest(cmd, nil) + assert.ErrorIs(t, err, errMock) + mockContainerHandler.AssertExpectations(t) + }) + + t.Run("containerHandlerInit failure", func(t *testing.T) { + cmd := newAirflowUpgradeTestCmd(nil) + + containerHandlerInit = func(airflowHome, envFile, dockerfile, imageName string) (airflow.ContainerHandler, error) { + return nil, errMock + } + + err := airflowUpgradeTest(cmd, nil) + assert.ErrorIs(t, err, errMock) + }) +} + func TestAirflowRun(t *testing.T) { t.Run("success", func(t *testing.T) { cmd := newAirflowRunCmd() diff --git a/cmd/root.go b/cmd/root.go index 778487083..2c947cd90 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -100,7 +100,7 @@ Welcome to the Astro CLI, the modern command line interface for data orchestrati newLoginCommand(astroClient, astroCoreClient, os.Stdout), newLogoutCommand(os.Stdout), newVersionCommand(), - newDevRootCmd(), + newDevRootCmd(astroClient), newContextCmd(os.Stdout), newConfigRootCmd(os.Stdout), newRunCommand(), diff --git a/houston/mocks/ClientInterface.go b/houston/mocks/ClientInterface.go index 2e39a38b0..03acfe3c5 100644 --- a/houston/mocks/ClientInterface.go +++ b/houston/mocks/ClientInterface.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package houston_mocks @@ -1428,12 +1428,13 @@ func (_m *ClientInterface) ValidateWorkspaceID(workspaceID string) (*houston.Wor return r0, r1 } -// NewClientInterface creates a new instance of ClientInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewClientInterface(t interface { +type mockConstructorTestingTNewClientInterface interface { mock.TestingT Cleanup(func()) -}) *ClientInterface { +} + +// NewClientInterface creates a new instance of ClientInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewClientInterface(t mockConstructorTestingTNewClientInterface) *ClientInterface { mock := &ClientInterface{} mock.Mock.Test(t) diff --git a/pkg/azure/mocks/Azure.go b/pkg/azure/mocks/Azure.go index 66853a879..d6bb35a5f 100644 --- a/pkg/azure/mocks/Azure.go +++ b/pkg/azure/mocks/Azure.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.20.2. DO NOT EDIT. package azure_mocks @@ -37,12 +37,13 @@ func (_m *Azure) Upload(sasLink string, dagFileReader io.Reader) (string, error) return r0, r1 } -// NewAzure creates a new instance of Azure. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewAzure(t interface { +type mockConstructorTestingTNewAzure interface { mock.TestingT Cleanup(func()) -}) *Azure { +} + +// NewAzure creates a new instance of Azure. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewAzure(t mockConstructorTestingTNewAzure) *Azure { mock := &Azure{} mock.Mock.Test(t) diff --git a/software/deploy/deploy.go b/software/deploy/deploy.go index 57e2d595e..ddb516a60 100644 --- a/software/deploy/deploy.go +++ b/software/deploy/deploy.go @@ -223,7 +223,7 @@ func buildPushDockerImage(houstonClient houston.ClientInterface, c *config.Conte TargetPlatforms: deployImagePlatformSupport, Output: true, } - err = imageHandler.Build(buildConfig) + err = imageHandler.Build("", buildConfig) if err != nil { return err } @@ -244,8 +244,8 @@ func buildPushDockerImage(houstonClient houston.ClientInterface, c *config.Conte } if byoRegistryEnabled { - runtimeVersion, _ := imageHandler.GetLabel(runtimeImageLabel) - airflowVersion, _ := imageHandler.GetLabel(airflowImageLabel) + runtimeVersion, _ := imageHandler.GetLabel("", runtimeImageLabel) + airflowVersion, _ := imageHandler.GetLabel("", airflowImageLabel) req := houston.UpdateDeploymentImageRequest{ReleaseName: name, Image: remoteImage, AirflowVersion: airflowVersion, RuntimeVersion: runtimeVersion} _, err := houston.Call(houstonClient.UpdateDeploymentImage)(req) return err diff --git a/software/deploy/deploy_test.go b/software/deploy/deploy_test.go index 1bfa7b466..50a8907ef 100644 --- a/software/deploy/deploy_test.go +++ b/software/deploy/deploy_test.go @@ -164,8 +164,8 @@ func TestBuildPushDockerImageSuccessWithBYORegistry(t *testing.T) { imageHandlerInit = func(image string) airflow.ImageHandler { mockImageHandler.On("Build", mock.Anything, mock.Anything).Return(nil) mockImageHandler.On("Push", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - mockImageHandler.On("GetLabel", runtimeImageLabel).Return("", nil).Once() - mockImageHandler.On("GetLabel", airflowImageLabel).Return("1.10.12", nil).Once() + mockImageHandler.On("GetLabel", "", runtimeImageLabel).Return("", nil).Once() + mockImageHandler.On("GetLabel", "", airflowImageLabel).Return("1.10.12", nil).Once() return mockImageHandler }