add tests

.github/workflows/build.yaml

@@ -84,28 +84,29 @@ jobs:
           tags: ghcr.io/castai/cluster-controller:${{ github.sha }}
 
       - name: Build and push main
-        if: github.ref_name == 'main' && github.event_name == 'push'
+        #if: github.ref_name == 'main' && github.event_name == 'push'
+        if: github.ref_name == 'KUBE-632/trigger-e2e-tests' && github.event_name == 'push'
         uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/arm64,linux/amd64
           push: ${{ github.event_name != 'pull_request' }}
           tags: us-docker.pkg.dev/castai-hub/library/cluster-controller:${{ github.sha }}
 
-      - name: Trigger e2e tests
-        if: github.ref_name == 'main' && github.event_name == 'push'
+      - name: Run e2e tests
+        #if: github.ref_name == 'main' && github.event_name == 'push'
+        if: github.ref_name == 'KUBE-632/trigger-e2e-tests' && github.event_name == 'push'
+        env:
+        CLUSTER_CONTROLLER_IMAGE_REPOSITORY: us-docker.pkg.dev/castai-hub/library/cluster-controller
+        CLUSTER_CONTROLLER_IMAGE_TAG: ${{ github.sha }}
+        CLUSTER_NAME: e2e-cluster-controller
+        CLUSTER_REGION: us-central1
+        CASTAI_API_TOKEN: ${{ secrets.CASTAI_API_TOKEN }}
+        GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
         run: |
-          git clone "https://git:${{ secrets.GITLAB_TEST_REPO_TOKEN }}@${{ secrets.GITLAB_TEST_REPO }}" test-repo
-
-          cd test-repo/charts/cluster-controller-tests
-          git config user.email "[email protected]"
-          git config user.name "cluster-controller-github"
-
-          ./set-cluster-controller.sh "${{ github.sha }}"
+          go test ./e2e
 
-          git add .
-          git commit -m 'cluster-controller-tests: trigger tests for ${{ github.sha }}'
-          git push origin master
+      # TODO: Create Github release after e2e tests on main
 
       - name: Build and push release
         if: github.event_name == 'release'

.gitignore

@@ -3,3 +3,4 @@
 *.iml
 bin
 .env
+e2e/**/castai-*.json

Makefile

@@ -1,3 +1,6 @@
+export API_TAGS ?= ExternalClusterAPI,AuthTokenAPI,OperationsAPI,AutoscalerAPI
+export SWAGGER_LOCATION ?= https://api.cast.ai/v1/spec/openapi.json
+
 build:
 	CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -o bin/castai-cluster-controller-amd64 .
 	docker build -t us-docker.pkg.dev/castai-hub/library/cluster-controller:$(VERSION) .
@@ -17,4 +20,8 @@ fix:
 
 test:
 	go test ./... -race
-.PHONY: test
+.PHONY: test
+
+generate-e2e-client:
+	go generate ./e2e/client
+.PHONY: generate-e2e-client

e2e/client/client.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging"
+)
+
+func CreateClient(apiURL, apiToken, userAgent string) (*ClientWithResponses, error) {
+	httpClientOption := func(client *Client) error {
+		client.Client = &http.Client{
+			Transport: logging.NewSubsystemLoggingHTTPTransport("CAST.AI", http.DefaultTransport),
+			Timeout:   1 * time.Minute,
+		}
+		client.RequestEditors = append(client.RequestEditors, func(_ context.Context, req *http.Request) error {
+			req.Header.Set("user-agent", userAgent)
+			return nil
+		})
+		return nil
+	}
+
+	apiTokenOption := WithRequestEditorFn(func(ctx context.Context, req *http.Request) error {
+		req.Header.Set("X-API-Key", apiToken)
+		return nil
+	})
+
+	apiClient, err := NewClientWithResponses(apiURL, httpClientOption, apiTokenOption)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp, err := apiClient.AuthTokenAPIListAuthTokensWithResponse(context.Background(), &AuthTokenAPIListAuthTokensParams{}); err != nil {
+		return nil, fmt.Errorf("validating api token (by listing auth tokens): %w", err)
+	} else if resp.StatusCode() != http.StatusOK {
+		return nil, fmt.Errorf("expected status code %d, received %d", http.StatusOK, resp.StatusCode())
+	}
+
+	return apiClient, nil
+}

e2e/client/codegen/templates/client-with-responses.tmpl

@@ -0,0 +1,129 @@
+// ClientWithResponses builds on ClientInterface to offer response payloads
+type ClientWithResponses struct {
+    ClientInterface
+}
+
+// NewClientWithResponses creates a new ClientWithResponses, which wraps
+// Client with return type handling
+func NewClientWithResponses(server string, opts ...ClientOption) (*ClientWithResponses, error) {
+    client, err := NewClient(server, opts...)
+    if err != nil {
+        return nil, err
+    }
+    return &ClientWithResponses{client}, nil
+}
+
+// WithBaseURL overrides the baseURL.
+func WithBaseURL(baseURL string) ClientOption {
+	return func(c *Client) error {
+		newBaseURL, err := url.Parse(baseURL)
+		if err != nil {
+			return err
+		}
+		c.Server = newBaseURL.String()
+		return nil
+	}
+}
+
+// ClientWithResponsesInterface is the interface specification for the client with responses above.
+type ClientWithResponsesInterface interface {
+{{range . -}}
+{{$hasParams := .RequiresParamObject -}}
+{{$pathParams := .PathParams -}}
+{{$opid := .OperationId -}}
+    // {{$opid}} request {{if .HasBody}} with any body{{end}}
+    {{$opid}}{{if .HasBody}}WithBody{{end}}WithResponse(ctx context.Context{{genParamArgs .PathParams}}{{if .RequiresParamObject}}, params *{{$opid}}Params{{end}}{{if .HasBody}}, contentType string, body io.Reader{{end}}) (*{{genResponseTypeName $opid}}, error)
+{{range .Bodies}}
+    {{$opid}}{{.Suffix}}WithResponse(ctx context.Context{{genParamArgs $pathParams}}{{if $hasParams}}, params *{{$opid}}Params{{end}}, body {{$opid}}{{.NameTag}}RequestBody) (*{{genResponseTypeName $opid}}, error)
+{{end}}{{/* range .Bodies */}}
+{{end}}{{/* range . $opid := .OperationId */}}
+}
+
+// TODO: <castai customization> to have common interface. https://github.com/deepmap/oapi-codegen/issues/240
+type Response interface {
+    Status()     string
+    StatusCode() int
+    GetBody()    []byte
+}
+// TODO: </castai customization> to have common interface. https://github.com/deepmap/oapi-codegen/issues/240
+
+{{range .}}{{$opid := .OperationId}}{{$op := .}}
+type {{$opid | ucFirst}}Response struct {
+    Body         []byte
+	HTTPResponse *http.Response
+    {{- range getResponseTypeDefinitions .}}
+    {{.TypeName}} *{{.Schema.TypeDecl}}
+    {{- end}}
+}
+
+// Status returns HTTPResponse.Status
+func (r {{$opid | ucFirst}}Response) Status() string {
+    if r.HTTPResponse != nil {
+        return r.HTTPResponse.Status
+    }
+    return http.StatusText(0)
+}
+
+// StatusCode returns HTTPResponse.StatusCode
+func (r {{$opid | ucFirst}}Response) StatusCode() int {
+    if r.HTTPResponse != nil {
+        return r.HTTPResponse.StatusCode
+    }
+    return 0
+}
+
+// TODO: <castai customization> to have common interface. https://github.com/deepmap/oapi-codegen/issues/240
+// Body returns body of byte array
+func (r {{$opid | ucFirst}}Response) GetBody() []byte {
+     return r.Body
+}
+// TODO: </castai customization> to have common interface. https://github.com/deepmap/oapi-codegen/issues/240
+{{end}}
+
+{{range .}}
+{{$opid := .OperationId -}}
+{{/* Generate client methods (with responses)*/}}
+
+// {{$opid}}{{if .HasBody}}WithBody{{end}}WithResponse request{{if .HasBody}} with arbitrary body{{end}} returning *{{$opid}}Response
+func (c *ClientWithResponses) {{$opid}}{{if .HasBody}}WithBody{{end}}WithResponse(ctx context.Context{{genParamArgs .PathParams}}{{if .RequiresParamObject}}, params *{{$opid}}Params{{end}}{{if .HasBody}}, contentType string, body io.Reader{{end}}) (*{{genResponseTypeName $opid}}, error){
+    rsp, err := c.{{$opid}}{{if .HasBody}}WithBody{{end}}(ctx{{genParamNames .PathParams}}{{if .RequiresParamObject}}, params{{end}}{{if .HasBody}}, contentType, body{{end}})
+    if err != nil {
+        return nil, err
+    }
+    return Parse{{genResponseTypeName $opid | ucFirst}}(rsp)
+}
+
+{{$hasParams := .RequiresParamObject -}}
+{{$pathParams := .PathParams -}}
+{{$bodyRequired := .BodyRequired -}}
+{{range .Bodies}}
+func (c *ClientWithResponses) {{$opid}}{{.Suffix}}WithResponse(ctx context.Context{{genParamArgs $pathParams}}{{if $hasParams}}, params *{{$opid}}Params{{end}}, body {{$opid}}{{.NameTag}}RequestBody) (*{{genResponseTypeName $opid}}, error) {
+    rsp, err := c.{{$opid}}{{.Suffix}}(ctx{{genParamNames $pathParams}}{{if $hasParams}}, params{{end}}, body)
+    if err != nil {
+        return nil, err
+    }
+    return Parse{{genResponseTypeName $opid | ucFirst}}(rsp)
+}
+{{end}}
+
+{{end}}{{/* operations */}}
+
+{{/* Generate parse functions for responses*/}}
+{{range .}}{{$opid := .OperationId}}
+
+// Parse{{genResponseTypeName $opid | ucFirst}} parses an HTTP response from a {{$opid}}WithResponse call
+func Parse{{genResponseTypeName $opid | ucFirst}}(rsp *http.Response) (*{{genResponseTypeName $opid}}, error) {
+    bodyBytes, err := io.ReadAll(rsp.Body)
+    defer rsp.Body.Close()
+    if err != nil {
+        return nil, err
+    }
+
+    response := {{genResponsePayload $opid}}
+
+    {{genResponseUnmarshal .}}
+
+    return response, nil
+}
+{{end}}{{/* range . $opid := .OperationId */}}
+

e2e/client/generate.go

@@ -0,0 +1,5 @@
+package client
+
+//go:generate go install github.com/deepmap/oapi-codegen/cmd/[email protected]
+//go:generate oapi-codegen -o api.gen.go --old-config-style -generate types -include-tags $API_TAGS -package client $SWAGGER_LOCATION
+//go:generate oapi-codegen -o client.gen.go --old-config-style -templates codegen/templates -generate client -include-tags $API_TAGS -package client $SWAGGER_LOCATION

e2e/main_test.go

@@ -0,0 +1,39 @@
+package e2e
+
+import (
+	"context"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/kelseyhightower/envconfig"
+	"github.com/stretchr/testify/require"
+
+	"github.com/castai/cluster-controller/e2e/suites"
+)
+
+var cfg suites.Config
+
+func TestMain(m *testing.M) {
+	if err := envconfig.Process("", &cfg); err != nil {
+		log.Fatalf("failed to load config: %v", err)
+	}
+
+	exitCode := m.Run()
+	os.Exit(exitCode)
+}
+
+func TestClusterController_GKEUpgrade(t *testing.T) {
+	t.Parallel()
+
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
+
+	ctx := context.Background()
+
+	ts, err := suites.NewGKETestSuite(t, &cfg)
+	require.NoError(t, err)
+
+	ts.Run(ctx, t)
+}