From 9ce82f00b2b669775ea90cc725cccf243ca60d56 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Mon, 27 Jun 2022 14:48:58 +0200 Subject: [PATCH] e2e test for vault --- .github/workflows/_e2e-test.yml | 39 ++++++++ .github/workflows/e2e-test-on-pr.yml | 34 ------- .github/workflows/e2e-trousseau-on-pr.yml | 6 ++ .github/workflows/e2e-vault-on-pr.yml | 6 ++ Taskfile.yml | 91 +++++++++++++++---- localdev.md | 7 +- providers/awskms/localdev.md | 11 ++- providers/debug/localdev.md | 11 --- providers/vault/go.mod | 5 +- providers/vault/localdev.md | 11 +-- providers/vault/pkg/vault/vault_test.go | 22 ----- tests/e2e/kuttl/kube-v1.23/kind.yaml | 2 +- .../kuttl/tests/01-restart-components.yaml | 4 +- 13 files changed, 146 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/_e2e-test.yml delete mode 100644 .github/workflows/e2e-test-on-pr.yml delete mode 100644 providers/vault/pkg/vault/vault_test.go diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml new file mode 100644 index 0000000..7a8ae61 --- /dev/null +++ b/.github/workflows/_e2e-test.yml @@ -0,0 +1,39 @@ +name: e2e test on pr + +on: + workflow_call: + inputs: + provider: + required: true + default: "debug" + type: string + +env: + DOCKER_REGISTRY: local + IMAGE_NAME: trousseau + IMAGE_VERSION: e2e + +jobs: + e2e: + name: kuttl e2e + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install Taskfile + run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task + - name: fetch dependencies + run: ./bin/task fetch:kind fetch:kuttl + - name: build and start proxy + run: ./bin/task docker:build:proxy docker:run:proxy + - name: build and start components for debug + if: ${{ inputs.provider == 'debug' }} + run: ./bin/task docker:build:debug docker:run:debug docker:build:trousseau docker:run:trousseau + env: + ENABLED_PROVIDERS: --enabled-providers debug + - name: build and start components for vault + if: ${{ inputs.provider == 'vault' }} + run: ./bin/task docker:build:vault docker:run:vault docker:build:trousseau docker:run:trousseau + env: + ENABLED_PROVIDERS: --enabled-providers vault + - name: run e2e tests + run: ./bin/kubectl-kuttl test --config tests/e2e/kuttl/kube-v1.23/kuttl.yaml diff --git a/.github/workflows/e2e-test-on-pr.yml b/.github/workflows/e2e-test-on-pr.yml deleted file mode 100644 index c936d82..0000000 --- a/.github/workflows/e2e-test-on-pr.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: e2e test on pr - -on: - pull_request: - branches: [ main, v2* ] - -permissions: - contents: read - pull-requests: read - actions: read - security-events: write - packages: write - -concurrency: - group: ci-e2e-test-${{ github.ref }}-1 - cancel-in-progress: true - -env: - DOCKER_REGISTRY: local - IMAGE_NAME: trousseau - IMAGE_VERSION: e2e - -jobs: - e2e: - name: kuttl e2e - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: install Taskfile - run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task - - name: fetch dependencies - run: ./bin/task fetch:kind fetch:kuttl - - name: run e2e tests - run: ./bin/task go:e2e-tests diff --git a/.github/workflows/e2e-trousseau-on-pr.yml b/.github/workflows/e2e-trousseau-on-pr.yml index e84e9f3..9dfd968 100644 --- a/.github/workflows/e2e-trousseau-on-pr.yml +++ b/.github/workflows/e2e-trousseau-on-pr.yml @@ -35,6 +35,12 @@ jobs: project: trousseau needs: gosec-scanning + e2e: + uses: ./.github/workflows/_e2e-test.yml + with: + provider: debug + needs: image-build + image-vulnerability-scan: uses: ./.github/workflows/_trivy.yml with: diff --git a/.github/workflows/e2e-vault-on-pr.yml b/.github/workflows/e2e-vault-on-pr.yml index e1a490e..5bb7f3d 100644 --- a/.github/workflows/e2e-vault-on-pr.yml +++ b/.github/workflows/e2e-vault-on-pr.yml @@ -35,6 +35,12 @@ jobs: project: providers/vault needs: gosec-scanning + e2e: + uses: ./.github/workflows/_e2e-test.yml + with: + provider: vault + needs: image-build + image-vulnerability-scan: uses: ./.github/workflows/_trivy.yml with: diff --git a/Taskfile.yml b/Taskfile.yml index 079f166..f64a93a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -24,9 +24,17 @@ tasks: desc: create bin directory cmds: - mkdir -p ./bin + - mkdir -m 777 bin/run + - mkdir -m 777 bin/run/debug + - mkdir -m 777 bin/run/vault + - mkdir -m 777 bin/run/awskms - mkdir -p tests/e2e/generated_manifests status: - test -d ./bin + - test -d ./bin/run + - test -d ./bin/run/debug + - test -d ./bin/run/vault + - test -d ./bin/run/awskms - test -d tests/e2e/generated_manifests fetch:golangci: deps: @@ -219,34 +227,41 @@ tasks: - task: docker:run:awskms - task: docker:run:trousseau docker:run:proxy: + deps: + - bin-dir:init cmds: - rm -rf bin/run/proxy.socket - docker rm -f trousseau-proxy || true - docker run -d --name trousseau-proxy --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION docker:run:debug: + deps: + - bin-dir:init cmds: - - mkdir -m 777 -p bin/run/debug - rm -rf bin/run/debug/debug.socket - docker rm -f trousseau-debug || true - docker run -d --name trousseau-debug --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION docker:run:vault: + deps: + - bin-dir:init cmds: - - mkdir -m 777 -p bin/run/vault - rm -rf bin/run/vault/vault.socket - docker rm -f dev-vault || true - - docker run --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' -p 8200:8200 -d --name=dev-vault vault + - docker run -d --name=dev-vault --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' vault - sleep 5 - - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -it dev-vault vault login vault-kms-demo - - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -it dev-vault vault secrets enable transit + - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 dev-vault vault login vault-kms-demo + - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 dev-vault vault secrets enable transit - docker rm -f trousseau-vault || true - docker run -d --name trousseau-vault --rm --network=container:dev-vault -v $PWD/tests/e2e/kuttl/kube-v1.23/vault.yaml:/opt/vault-kms/vault/config.yaml -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION docker:run:awskms: + deps: + - bin-dir:init cmds: - - mkdir -m 777 -p bin/run/awskms - rm -rf bin/run/awskms/awskms.socket - docker rm -f trousseau-awskms || true - docker run -d --name trousseau-awskms --rm -v $HOME/.aws/credentials:/.aws/credentials -v $PWD/scripts/hcvault/archives/localdev/awskms.yaml:/opt/vault-kms/awskms/config.yaml -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION docker:run:trousseau: + deps: + - bin-dir:init cmds: - rm -rf bin/run/trousseau.socket - docker rm -f trousseau-core || true @@ -398,34 +413,78 @@ tasks: - task: go:unit-tests:debug - task: go:unit-tests:vault - task: go:unit-tests:awskms - - task: go:trousseau:unit-tests + - task: go:unit-tests:trousseau go:unit-tests:proxy: dir: proxy cmds: - - go test -race -timeout 30s ./... + - go test -coverprofile cover.out -race -timeout 30s ./... go:unit-tests:debug: dir: providers/debug cmds: - - go test -race -timeout 30s ./... + - go test -coverprofile cover.out -race -timeout 30s ./... go:unit-tests:vault: dir: providers/vault cmds: - - go test -race -timeout 30s ./... + - go test -coverprofile cover.out -race -timeout 30s ./... go:unit-tests:awskms: dir: providers/awskms cmds: - - go test -race -timeout 30s ./... - go:trousseau:unit-tests: + - go test -coverprofile cover.out -race -timeout 30s ./... + go:unit-tests:trousseau: dir: trousseau cmds: - - go test -race -timeout 30s ./... - go:integration-tests: + - go test -coverprofile cover.out -race -timeout 30s ./... + go:run: + desc: go run + cmds: + - task: go:run:proxy + - task: go:run:debug + - task: go:run:vault + - task: go:run:awskms + - task: go:run:trousseau + go:run:proxy: + dir: proxy + deps: + - bin-dir:init + - go:tidy:proxy + cmds: + - rm -rf ../bin/run/proxy.socket + - go run main.go --listen-addr unix://../bin/run/proxy.socket --trousseau-addr ../bin/run/trousseau.socket + go:run:debug: + dir: providers/debug + deps: + - bin-dir:init + - go:tidy:debug + cmds: + - rm -rf ../../bin/run/debug/debug.socket + - go run main.go --listen-addr unix://../../bin/run/debug/debug.socket + go:run:vault: + dir: providers/vault + deps: + - bin-dir:init + - go:tidy:vault + cmds: + - rm -rf ../../bin/run/vault/vault.socket + - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go --config-file-path ../../scripts/hcvault/archives/localdev/vault.yaml --listen-addr unix://../../bin/run/vault/vault.socket --zap-encoder=console --v=5 + go:run:awskms: + dir: providers/awskms + deps: + - bin-dir:init + - go:tidy:awskms + cmds: + - rm -rf ../../bin/run/awskms/awskms.socket + - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go --config-file-path ../../scripts/hcvault/archives/localdev/awskms.yaml --listen-addr unix://../../bin/run/awskms/awskms.socket --zap-encoder=console --v=5 + go:run:trousseau: + dir: trousseau + deps: + - bin-dir:init + - go:tidy:trousseau cmds: - - KUBECTL_CONTEXT=kind-{{.KIND_CLUSTER_NAME}} go test --tags=integration ./... + - rm -rf ../bin/run/trousseau.socket + - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go {{.ENABLED_PROVIDERS}} --socket-location ../bin/run --listen-addr unix://../bin/run/trousseau.socket --zap-encoder=console --v=5 go:e2e-tests: desc: e2e tests cmds: - - rm -rf bin/run ; mkdir -m 777 bin/run - task: docker:build:proxy - task: docker:build:debug - task: docker:build:trousseau diff --git a/localdev.md b/localdev.md index 69c35a5..f0ca293 100644 --- a/localdev.md +++ b/localdev.md @@ -18,10 +18,9 @@ task fetch:all ## Run Trousseau components ```bash -mkdir bin/debug -(cd proxy ; go mod tidy && go run main.go --listen-addr unix://../bin/proxy.socket --trousseau-addr ../bin/trousseau.socket) -(cd providers/debug ; go mod tidy && go run main.go --listen-addr unix://../../bin/debug/debug.socket) -(cd trousseau ; go mod tidy && go run main.go --enabled-providers debug --socket-location ../bin --listen-addr unix://../bin/trousseau.socket --zap-encoder=console --v=5) +task go:run:proxy +task go:run:debug +task go:run:trousseau ``` ## Start cluster with encryption support diff --git a/providers/awskms/localdev.md b/providers/awskms/localdev.md index 282246f..e62ecbc 100644 --- a/providers/awskms/localdev.md +++ b/providers/awskms/localdev.md @@ -4,6 +4,10 @@ This document describes how to develop Trousseau AWS KMS provider on your local Please follow base documentation at [localdev.md](../localdev.md) +## Login to AWS + +Log in and create profile file at `~/.aws/credentials`. + ## Create AWS KMS config Edit config file at [awskms.yaml](../scripts/hcvault/archives/localdev/awskms.yaml): @@ -19,8 +23,7 @@ roleArn: roleArn Use command line or our favorite IDE to start Trousseau components on your machine: ```bash -mkdir bin/awskms -(cd proxy ; go mod tidy && go run main.go --listen-addr unix://../bin/proxy.socket --trousseau-addr ../bin/trousseau.socket) -(cd providers/awskms ; go mod tidy && go run main.go --config-file-path ../../scripts/hcvault/archives/localdev/awskms.yaml --listen-addr unix://../../bin/awskms/awskms.socket --zap-encoder=console --v=5) -(cd trousseau ; go mod tidy && go run main.go --enabled-providers awskms --socket-location ../bin --listen-addr unix://../bin/trousseau.socket --zap-encoder=console --v=5) +task go:run:proxy +task go:run:awskms +ENABLED_PROVIDERS="--enabled-providers awskms" task go:run:trousseau ``` diff --git a/providers/debug/localdev.md b/providers/debug/localdev.md index 81f82fb..b6730a0 100644 --- a/providers/debug/localdev.md +++ b/providers/debug/localdev.md @@ -3,14 +3,3 @@ This document describes how to develop Trousseau Debug provider on your local machine. Please follow base documentation at [localdev.md](../localdev.md) - -## Run Trousseau components - -Use command line or our favorite IDE to start Trousseau components on your machine: - -```bash -mkdir bin/debug -(cd proxy ; go mod tidy && go run main.go --listen-addr unix://../bin/proxy.socket --trousseau-addr ../bin/trousseau.socket) -(cd providers/debug ; go mod tidy && go run main.go --listen-addr unix://../../bin/debug/debug.socket) -(cd trousseau ; go mod tidy && go run main.go --enabled-providers debug --socket-location ../bin --listen-addr unix://../bin/trousseau.socket --zap-encoder=console --v=5) -``` diff --git a/providers/vault/go.mod b/providers/vault/go.mod index 1bd62db..83b1879 100644 --- a/providers/vault/go.mod +++ b/providers/vault/go.mod @@ -7,7 +7,6 @@ replace github.com/ondat/trousseau => ../.. require ( github.com/hashicorp/vault/api v1.7.2 github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 - github.com/stretchr/testify v1.7.2 google.golang.org/grpc v1.47.0 k8s.io/apiserver v0.24.1 k8s.io/klog/v2 v2.60.1 @@ -17,7 +16,6 @@ require ( github.com/armon/go-metrics v0.3.9 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.7.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/zapr v1.2.3 // indirect @@ -51,8 +49,8 @@ require ( github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pierrec/lz4 v2.5.2+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/stretchr/testify v1.7.2 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.0 // indirect @@ -65,5 +63,4 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/providers/vault/localdev.md b/providers/vault/localdev.md index 2f956c0..96c0e07 100644 --- a/providers/vault/localdev.md +++ b/providers/vault/localdev.md @@ -19,12 +19,12 @@ docker run --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' -p 820 You can validate your Vault instance by performing a login: ```bash -docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -it dev-vault vault login vault-kms-demo +docker exec -e VAULT_ADDR=http://127.0.0.1:8200 dev-vault vault login vault-kms-demo ``` Enable transit engine: ```bash -docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -it dev-vault vault secrets enable transit +docker exec -e VAULT_ADDR=http://127.0.0.1:8200 dev-vault vault secrets enable transit ``` ## Run Trousseau components @@ -32,8 +32,7 @@ docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -it dev-vault vault secrets enab Use command line or our favorite IDE to start Trousseau components on your machine: ```bash -mkdir bin/vault -(cd proxy ; go mod tidy && go run main.go --listen-addr unix://../bin/proxy.socket --trousseau-addr ../bin/trousseau.socket) -(cd providers/vault ; go mod tidy && go run main.go --config-file-path ../../scripts/hcvault/archives/localdev/vault.yaml --listen-addr unix://../../bin/vault/vault.socket --zap-encoder=console --v=5) -(cd trousseau ; go mod tidy && go run main.go --enabled-providers vault --socket-location ../bin --listen-addr unix://../bin/trousseau.socket --zap-encoder=console --v=5) +task go:run:proxy +rask go:run:vault +ENABLED_PROVIDERS="--enabled-providers vault" task go:run:trousseau ``` diff --git a/providers/vault/pkg/vault/vault_test.go b/providers/vault/pkg/vault/vault_test.go deleted file mode 100644 index f2a6174..0000000 --- a/providers/vault/pkg/vault/vault_test.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build !integration -// +build !integration - -package vault - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func getVaultConfig() *Config { - return &Config{ - Address: "http://localhost:9200", - Token: "test", - } -} - -func TestCreatingVaultClient(t *testing.T) { - _, err := New(getVaultConfig()) - assert.NoError(t, err) -} diff --git a/tests/e2e/kuttl/kube-v1.23/kind.yaml b/tests/e2e/kuttl/kube-v1.23/kind.yaml index 3ca0ea0..d9ff62b 100644 --- a/tests/e2e/kuttl/kube-v1.23/kind.yaml +++ b/tests/e2e/kuttl/kube-v1.23/kind.yaml @@ -1,6 +1,6 @@ --- kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 +apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane image: kindest/node:v1.23.6 diff --git a/tests/e2e/kuttl/tests/01-restart-components.yaml b/tests/e2e/kuttl/tests/01-restart-components.yaml index aa241db..77bce3e 100644 --- a/tests/e2e/kuttl/tests/01-restart-components.yaml +++ b/tests/e2e/kuttl/tests/01-restart-components.yaml @@ -1,6 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - command: docker restart trousseau-debug + - command: docker restart trousseau-debug || true + - command: docker restart trousseau-vault || true + - command: docker restart trousseau-awskms || true - command: docker restart trousseau-core - command: sleep 10 \ No newline at end of file