Merge pull request #420 from astronomer/main

sync release-0.25 with main

airflow/airflow.go

@@ -64,15 +64,14 @@ func Init(path string, airflowImageTag string) error {
 
 	// Map of files to create
 	files := map[string]string{
-		".dockerignore":             include.Dockerignore,
-		"Dockerfile":                fmt.Sprintf(include.Dockerfile, airflowImageTag),
-		".gitignore":                include.Gitignore,
-		"packages.txt":              "",
-		"requirements.txt":          "",
-		".env":                      "",
-		"airflow_settings.yaml":     include.Settingsyml,
-		"dags/example-dag.py":       include.Exampledag,
-		"plugins/example-plugin.py": include.ExamplePlugin,
+		".dockerignore":         include.Dockerignore,
+		"Dockerfile":            fmt.Sprintf(include.Dockerfile, airflowImageTag),
+		".gitignore":            include.Gitignore,
+		"packages.txt":          "",
+		"requirements.txt":      "",
+		".env":                  "",
+		"airflow_settings.yaml": include.Settingsyml,
+		"dags/example-dag.py":   include.Exampledag,
 	}
 
 	// Initailize directories

airflow/airflow_test.go

@@ -81,7 +81,6 @@ func TestInit(t *testing.T) {
 		".env",
 		"airflow_settings.yaml",
 		"dags/example-dag.py",
-		"plugins/example-plugin.py",
 	}
 	for _, file := range expectedFiles {
 		exist, err := fileutil.Exists(filepath.Join(tmpDir, file))

airflow/include/settingsyml.go

@@ -4,13 +4,12 @@ import "strings"
 
 // Settingsyml is the settings template
 var Settingsyml = strings.TrimSpace(`
-# This feature is in Beta.
-
-# Please report any bugs to [email protected]
 
+# This file allows you to configure Airflow Connections, Pools, and Variables in a single place for local development only.
 # NOTE: If putting a dict in conn_extra, please wrap in single quotes.
 
-# More details you can find https://github.com/astronomer/docs/blob/main/v0.10/cli-airflow-configuration.md
+# For more information, refer to our docs: https://www.astronomer.io/docs/cloud/stable/develop/customize-image#configure-airflowsettingsyaml
+# For issues or questions, reach out to [email protected]
 
 airflow:
   connections:

airflow_versions/airflow_versions.go

@@ -0,0 +1,26 @@
+package airflowversions
+
+import (
+	"fmt"
+	"sort"
+)
+
+// GetDefaultImageTag returns default airflow image tag
+func GetDefaultImageTag(httpClient *Client, airflowVersion string) (string, error) {
+	r := Request{}
+
+	resp, err := r.DoWithClient(httpClient)
+	if err != nil {
+		return "", err
+	}
+
+	vs := make(AirflowVersions, len(resp.AvailableReleases))
+	for i, r := range resp.AvailableReleases {
+		v, _ := NewAirflowVersion(r.Version, r.Tags)
+		vs[i] = v
+	}
+
+	sort.Sort(vs)
+	maxAvailableVersion := vs[len(vs)-1]
+	return fmt.Sprintf("%s-buster-onbuild", maxAvailableVersion.Coerce()), nil
+}

airflow_versions/airflow_versions_test.go

@@ -0,0 +1,88 @@
+package airflowversions
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	testUtil "github.com/astronomer/astro-cli/pkg/testing"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetDefaultImageTag(t *testing.T) {
+	testUtil.InitTestConfig()
+	okResponse := `{
+  "version": "1.0",
+  "available_releases": [
+    {
+      "version": "1.10.5",
+      "level": "new_feature",
+      "url": "https://github.com/astronomer/airflow/releases/tag/1.10.5-11",
+      "release_date": "2020-10-05T20:03:00+00:00",
+      "tags": [
+        "1.10.5-alpine3.10-onbuild",
+        "1.10.5-buster-onbuild",
+        "1.10.5-alpine3.10",
+        "1.10.5-buster"
+      ],
+      "channel": "stable"
+    },
+    {
+      "version": "1.10.5-11",
+      "level": "bug_fix",
+      "url": "https://github.com/astronomer/airflow/releases/tag/1.10.5-11",
+      "release_date": "2020-10-05T20:03:00+00:00",
+      "tags": [
+        "1.10.5-11-alpine3.10-onbuild",
+        "1.10.5-11-buster-onbuild",
+        "1.10.5-11-alpine3.10",
+        "1.10.5-11-buster"
+      ],
+      "channel": "stable"
+    },
+    {
+      "version": "1.10.4-11",
+      "level": "bug_fix",
+      "url": "https://github.com/astronomer/airflow/releases/tag/1.10.4-11",
+      "release_date": "2020-9-05T20:03:00+00:00",
+      "tags": [
+        "1.10.4-11-alpine3.10-onbuild",
+        "1.10.4-11-buster-onbuild",
+        "1.10.4-11-alpine3.10",
+        "1.10.4-11-buster"
+      ],
+      "channel": "stable"
+    }
+  ]
+}`
+	client := testUtil.NewTestClient(func(req *http.Request) *http.Response {
+		return &http.Response{
+			StatusCode: 200,
+			Body:       ioutil.NopCloser(bytes.NewBufferString(okResponse)),
+			Header:     make(http.Header),
+		}
+	})
+	httpClient := NewClient(client)
+
+	defaultImageTag, err := GetDefaultImageTag(httpClient, "")
+	assert.NoError(t, err)
+	assert.Equal(t, "1.10.5-buster-onbuild", defaultImageTag)
+}
+
+func TestGetDefaultImageTagError(t *testing.T) {
+	testUtil.InitTestConfig()
+	okResponse := `Page not found`
+	client := testUtil.NewTestClient(func(req *http.Request) *http.Response {
+		return &http.Response{
+			StatusCode: 404,
+			Body:       ioutil.NopCloser(bytes.NewBufferString(okResponse)),
+			Header:     make(http.Header),
+		}
+	})
+	httpClient := NewClient(client)
+
+	defaultImageTag, err := GetDefaultImageTag(httpClient, "")
+	assert.Error(t, err)
+	assert.Equal(t, "", defaultImageTag)
+}

airflow_versions/http_client.go

@@ -0,0 +1,71 @@
+package airflowversions
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/astronomer/astro-cli/config"
+	"github.com/astronomer/astro-cli/pkg/httputil"
+	"github.com/pkg/errors"
+)
+
+// Client containers the logger and HTTPClient used to communicate with the HoustonAPI
+type Client struct {
+	HTTPClient *httputil.HTTPClient
+}
+
+// NewClient returns a new Client with the logger and HTTP client setup.
+func NewClient(c *httputil.HTTPClient) *Client {
+	return &Client{
+		HTTPClient: c,
+	}
+}
+
+// Request represents empty request
+type Request struct{}
+
+// DoWithClient (request) is a wrapper to more easily pass variables to a client.Do request
+func (r *Request) DoWithClient(api *Client) (*Response, error) {
+	doOpts := httputil.DoOptions{
+		Headers: map[string]string{
+			"Accept": "application/json",
+		},
+	}
+
+	return api.Do(doOpts)
+}
+
+// Do executes the given HTTP request and returns the HTTP Response
+func (r *Request) Do() (*Response, error) {
+	return r.DoWithClient(NewClient(httputil.NewHTTPClient()))
+}
+
+// Do executes a query against the updates astronomer API, logging out any errors contained in the response object
+func (c *Client) Do(doOpts httputil.DoOptions) (*Response, error) {
+	var response httputil.HTTPResponse
+	url := config.CFG.AirflowReleasesURL.GetString()
+	httpResponse, err := c.HTTPClient.Do("GET", url, &doOpts)
+	if err != nil {
+		return nil, err
+	}
+	defer httpResponse.Body.Close()
+
+	body, err := ioutil.ReadAll(httpResponse.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	response = httputil.HTTPResponse{
+		Raw:  httpResponse,
+		Body: string(body),
+	}
+	decode := Response{}
+	err = json.NewDecoder(strings.NewReader(response.Body)).Decode(&decode)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("Failed to JSON decode %s response", url))
+	}
+
+	return &decode, nil
+}