From b63d107dd70e0cb49f9dec30a2f20ea2485bd9cd Mon Sep 17 00:00:00 2001 From: Harrison Katz Date: Tue, 17 Dec 2024 17:51:12 -0500 Subject: [PATCH] Test starting an ngrok agent in CI (#523) * Add ngrok-agent ci step * Add bindings e2e tests * Move more e2e tests to Makefile targets * Add bindings chainsaw tests * Add chainsaw tests for bindings services * Try with more dollar signs * Add make to action * Try with alpine and make * Try with github.workspace volume mount * Add static assets to e2e-fixtures ; Update Makefile * Add tcp bindings e2e test * With /bin/sh * With different dir * Try with volume mount * With debian * With different envar approach * With --detach * With 60s * With chainsaw timeouts instead * Test with 10m timeout * With different assertion * With ubuntu apt ngrok rather than docker * With ubuntu apt ngrok rather than docker * With ubuntu apt ngrok rather than docker * No directory to cd * With direct asserts * With debug step * With trues * With sleep * With dump logs * Even more logs * With dump Secret/ngrok-operator-default-tls * Adjust chainsaw tests to ensure tls.crt is valid * Remove superfluous docker-build step * Remove accidental extra steps * With specific IMG * With specific IMG * With error on ngrok-op doesn't start * With handling registry formatting * With different k3s action * Cleanup * Ensure tlsSecret is set to the found/created secret * Remove rebase artifacts --- .github/actions/build-and-test/action.yaml | 74 ++++++++++-- Makefile | 79 ++++++++----- e2e-fixtures/ngrok-assets/hello_world.txt | 1 + helm/ngrok-operator/templates/_helpers.tpl | 4 + .../ngrok/kubernetesoperator_controller.go | 6 +- tests/chainsaw/bindings/chainsaw-test.yaml | 111 ++++++++++++++++++ .../operator-registration/chainsaw-test.yaml | 8 +- 7 files changed, 236 insertions(+), 47 deletions(-) create mode 100644 e2e-fixtures/ngrok-assets/hello_world.txt create mode 100644 tests/chainsaw/bindings/chainsaw-test.yaml diff --git a/.github/actions/build-and-test/action.yaml b/.github/actions/build-and-test/action.yaml index a3fce655..3739749a 100644 --- a/.github/actions/build-and-test/action.yaml +++ b/.github/actions/build-and-test/action.yaml @@ -21,10 +21,11 @@ inputs: runs: using: "composite" steps: - - uses: debianmaster/actions-k3s@master + - uses: jupyterhub/action-k3s-helm@v4 id: k3s with: - version: 'latest' + k3s-channel: 'latest' + docker-enabled: true - shell: bash run: | @@ -52,10 +53,6 @@ runs: shell: bash run: make test - - name: Build the Docker image - shell: bash - run: make docker-build - - name: Deploy controller to local cluster shell: bash env: @@ -65,23 +62,42 @@ runs: NGROK_AUTHTOKEN: ${{ inputs.ngrok-authtoken }} E2E_BINDING_NAME: k8s/e2e-${{ github.run_id }} # use the latest image built in this run - IMG: ngrok/ngrok-operator + IMG: ngrok-operator-${{ github.run_id }} run: | - # create some namespaces for bindings tests - kubectl create ns e2e || true - - # deploy ngrok-op for e2e tests - make deploy_for_e2e + make e2e-deploy - name: Check if operator is up shell: bash run: | kubectl get nodes kubectl get pods -A - kubectl -n ngrok-operator wait --for=condition=ready pod -l app.kubernetes.io/name=ngrok-operator --timeout=1m || true + kubectl -n ngrok-operator wait --for=condition=ready pod -l app.kubernetes.io/name=ngrok-operator --timeout=1m || { echo "Failed to start ngrok-operator!"; exit 1; } kubectl get pods -A kubectl -n ngrok-operator describe pod -l app.kubernetes.io/name=ngrok-operator + - name: Start an ngrok agent + if: ${{ inputs.run-e2e == 'true' }} + shell: bash + run: | + # envars needed for ngrok-agent + export NGROK_API_KEY=${{ inputs.ngrok-api-key }} + export NGROK_AUTHTOKEN=${{ inputs.ngrok-authtoken }} + export E2E_BINDING_NAME=k8s/e2e-${{ github.run_id }} + export E2E_TCP_ENDPOINT_NAME=${{ github.run_id }}-tcp + + # add the ngrok apt source + curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc \ + | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null \ + && echo "deb https://ngrok-agent.s3.amazonaws.com buster main" \ + | sudo tee /etc/apt/sources.list.d/ngrok.list \ + + sudo apt-get update + sudo apt-get install -y \ + make \ + ngrok \ + + make e2e-start-ngrok + - name: Install cosign if: ${{ inputs.run-e2e == 'true' }} uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 @@ -97,6 +113,38 @@ runs: run: | make e2e-tests + # best effort to dump ngrok k8s resources from cluster + # this helps us debug pipelines + - name: Dump k8s resources + shell: bash + if: ${{ always() }} + run: | + echo "Dumping k8s resources" + echo "=== Pods ===" + kubectl -n ngrok-operator describe pods || true + echo "=== Services ===" + kubectl -n ngrok-operator describe services || true + echo "=== Deployments ===" + kubectl -n ngrok-operator describe deployments || true + echo "=== TLS Cert ===" + kubectl -n ngrok-operator describe secret ngrok-operator-default-tls || true + echo "=== KubernetesOperators ===" + kubectl -n ngrok-operator describe KubernetesOperators || true + echo "=== BoundEndpoints ===" + kubectl -n ngrok-operator describe BoundEndpoints || true + + # best effort to dump some logs + # this helps us debug pipelines + - name: Dump logs + shell: bash + if: ${{ always() }} + run: | + echo "Dumping logs" + echo "=== Forwarder ===" + kubectl -n ngrok-operator logs -l app.kubernetes.io/component=bindings-forwarder --tail=-1 || true + echo "=== API ===" + kubectl -n ngrok-operator logs -l app.kubernetes.io/component=controller --tail=-1 || true + # best effort to remove ngrok k8s resources from cluster # this allows our finalizers to delete upstream ngrok API resources too # that hopefully helps not pollute our ngrok-operator-ci account diff --git a/Makefile b/Makefile index 6b10c9a9..b480ceb9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ +# absolute path to cwd relative to Makefile +CWD := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) + # Image URL to use all building/pushing image targets IMG ?= ngrok-operator +IMG_REGISTRY ?= "docker.io" # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.29.0 @@ -37,6 +41,13 @@ CONTROLLER_GEN_PATHS = {./api/..., ./internal/controller/...} # when true, deploy with --set oneClickDemoMode=true DEPLOY_ONE_CLICK_DEMO_MODE ?= false +# example assets +# Note: contents must match chainsaw tests +E2E_ASSETS_DIR ?= $(CWD)/e2e-fixtures/ngrok-assets/ + +# default name for e2e-deploy k8s binding.name +E2E_BINDING_NAME ?= k8s/e2e-from-makefile + # Targets .PHONY: all @@ -195,29 +206,6 @@ deploy_with_bindings: _deploy-check-env-vars docker-build manifests kustomize _h &&\ kubectl rollout restart deployment $(KUBE_DEPLOYMENT_NAME) -n $(KUBE_NAMESPACE) -.PHONY: deploy_for_e2e -deploy_for_e2e: _deploy-check-env-vars docker-build manifests kustomize _helm_setup ## Deploy controller to the K8s cluster specified in ~/.kube/config. - helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \ - --namespace $(KUBE_NAMESPACE) \ - --create-namespace \ - --set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \ - --set image.repository=$(IMG) \ - --set image.tag="latest" \ - --set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \ - --set credentials.apiKey=$(NGROK_API_KEY) \ - --set credentials.authtoken=$(NGROK_AUTHTOKEN) \ - --set log.format=console \ - --set log.level=debug \ - --set log.stacktraceLevel=panic \ - --set metaData.env=local,metaData.from=makefile \ - --set bindings.enabled=true \ - --set bindings.name=$(E2E_BINDING_NAME) \ - --set bindings.description="Example binding for CI e2e tests" \ - --set bindings.allowedURLs='{*.e2e}' \ - --set bindings.serviceAnnotations.annotation1="val1" \ - --set bindings.serviceAnnotations.annotation2="val2" \ - --set bindings.serviceLabels.label1="val1" - .PHONY: _deploy-check-env-vars _deploy-check-env-vars: ifndef NGROK_API_KEY @@ -308,14 +296,49 @@ helm-update-snapshots-no-deps: ## Update helm unittest snapshots without rebuild ##@ E2E tests +# NOTE: You will also need to load ngrok-operator:latest image into your local cluster for this to work +.PHONY: e2e-deploy ## Deploy the operator for e2e tests +e2e-deploy: _deploy-check-env-vars docker-build manifests kustomize _helm_setup + # create some namespaces for bindings tests + kubectl create ns e2e || true + + # deploy for e2e tests + helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \ + --namespace $(KUBE_NAMESPACE) \ + --create-namespace \ + --set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \ + --set image.repository=$(IMG) \ + --set image.tag="latest" \ + --set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \ + --set credentials.apiKey=$(NGROK_API_KEY) \ + --set credentials.authtoken=$(NGROK_AUTHTOKEN) \ + --set log.format=console \ + --set log.level=debug \ + --set log.stacktraceLevel=panic \ + --set metaData.env=local,metaData.from=makefile \ + --set bindings.enabled=true \ + --set bindings.name=$(E2E_BINDING_NAME) \ + --set bindings.description="Example binding for CI e2e tests" \ + --set bindings.allowedURLs='{*.e2e}' \ + --set bindings.serviceAnnotations.annotation1="val1" \ + --set bindings.serviceAnnotations.annotation2="val2" \ + --set bindings.serviceLabels.label1="val1" + +.PHONY: e2e-start-ngrok +e2e-start-ngrok: ## Start the ngrok-agent for e2e tests + # start an agent + ngrok http file://$(E2E_ASSETS_DIR) --url http://assets-allowed.e2e --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!" + ngrok http file://$(E2E_ASSETS_DIR) --url http://assets-denied.example --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!" + ngrok tcp tcpbin.com:4242 --url tcp://tcp-echo.e2e:4242 --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!" + .PHONY: e2e-tests e2e-tests: ## Run e2e tests chainsaw test ./tests/chainsaw .PHONY: e2e-clean e2e-clean: ## Clean up e2e tests - kubectl delete ns e2e - kubectl delete --all boundendpoints -n ngrok-operator - kubectl delete --all services -n ngrok-operator - kubectl delete --all kubernetesoperators -n ngrok-operator - helm --namespace ngrok-operator uninstall ngrok-operator + kubectl delete ns e2e || true + kubectl delete --all boundendpoints -n ngrok-operator || true + kubectl delete --all services -n ngrok-operator || true + kubectl delete --all kubernetesoperators -n ngrok-operator || true + helm --namespace ngrok-operator uninstall ngrok-operator || true diff --git a/e2e-fixtures/ngrok-assets/hello_world.txt b/e2e-fixtures/ngrok-assets/hello_world.txt new file mode 100644 index 00000000..edfeed44 --- /dev/null +++ b/e2e-fixtures/ngrok-assets/hello_world.txt @@ -0,0 +1 @@ +Hello from ngrok-operator diff --git a/helm/ngrok-operator/templates/_helpers.tpl b/helm/ngrok-operator/templates/_helpers.tpl index c2e4def0..aac40a8a 100644 --- a/helm/ngrok-operator/templates/_helpers.tpl +++ b/helm/ngrok-operator/templates/_helpers.tpl @@ -122,5 +122,9 @@ Return the ngrok operator image name {{- $registryName := .Values.image.registry -}} {{- $repositoryName := .Values.image.repository -}} {{- $tag := .Values.image.tag | default .Chart.AppVersion | toString -}} +{{- if eq $registryName "" }} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- else }} {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end }} {{- end -}} diff --git a/internal/controller/ngrok/kubernetesoperator_controller.go b/internal/controller/ngrok/kubernetesoperator_controller.go index 5e0cf20e..f28bff39 100644 --- a/internal/controller/ngrok/kubernetesoperator_controller.go +++ b/internal/controller/ngrok/kubernetesoperator_controller.go @@ -147,11 +147,14 @@ func (r *KubernetesOperatorReconciler) create(ctx context.Context, ko *ngrokv1al var tlsSecret *v1.Secret if bindingsEnabled { - tlsSecret, err := r.findOrCreateTLSSecret(ctx, ko) + foundSecret, err := r.findOrCreateTLSSecret(ctx, ko) if err != nil { return err } + // remember to set the outer secret + tlsSecret = foundSecret + createParams.Binding = &ngrok.KubernetesOperatorBindingCreate{ Name: ko.Spec.Binding.Name, AllowedURLs: ko.Spec.Binding.AllowedURLs, @@ -469,7 +472,6 @@ func extractNamespaceUIDFromMetadata(metadata string) (string, error) { return uid, nil } -// nolint:unused func generateCSR(privKey *ecdsa.PrivateKey) ([]byte, error) { subj := pkix.Name{} diff --git a/tests/chainsaw/bindings/chainsaw-test.yaml b/tests/chainsaw/bindings/chainsaw-test.yaml new file mode 100644 index 00000000..9aec9322 --- /dev/null +++ b/tests/chainsaw/bindings/chainsaw-test.yaml @@ -0,0 +1,111 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: bindings +spec: + timeouts: + assert: 3m + steps: + - name: assert BoundEndpoint http://assets-denied.example is denied + try: + - assert: + resource: + apiVersion: bindings.k8s.ngrok.com/v1alpha1 + kind: BoundEndpoint + metadata: + name: ngrok-238b1294-28ba-5de5-8713-8a2928d8a2f9 # stable hash + namespace: ngrok-operator + spec: + allowed: false + endpointURI: "http://assets-denied.example:80" + scheme: "http" + # port: <-- port is allocated and may be out of order, do not assert + target: + service: "assets-denied" + namespace: "example" + protocol: TCP + port: 80 + status: + ~.(endpoints): + status: "denied" + + - name: assert BoundEndpoint http://assets-allowed.e2e is bound + try: + - assert: + resource: + apiVersion: bindings.k8s.ngrok.com/v1alpha1 + kind: BoundEndpoint + metadata: + name: ngrok-adb90775-7749-5b56-92f4-d52ee756975b # stable hash + namespace: ngrok-operator + spec: + allowed: true + endpointURI: "http://assets-allowed.e2e:80" + scheme: "http" + # port: <-- port is allocated and may be out of order, do not assert + target: + service: "assets-allowed" + namespace: "e2e" + protocol: TCP + port: 80 + status: + ~.(endpoints): + status: "bound" + # TargetService + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: assets-allowed + namespace: e2e + spec: + type: ExternalName + externalName: ngrok-adb90775-7749-5b56-92f4-d52ee756975b.ngrok-operator.svc.cluster.local # stable hash + ~.(ports): + name: http + port: 80 + targetPort: 80 + protocol: TCP + # UpstreamService + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: ngrok-adb90775-7749-5b56-92f4-d52ee756975b # stable hash + namespace: ngrok-operator + spec: + type: ClusterIP + ~.(ports): + name: http + port: 80 + # targetPort: < -- port is allocated and may be out of order, do not assert + protocol: TCP + + - name: test assets retrieval via BoundEndpoint TargetService + try: + - script: + content: | + # See contents in `make e2e-start-ngrok` task + WANT="Hello from ngrok-operator" + + # 2>/dev/null to suppress kubectl default output + # first line is the contents, last line in "deleted pod" message + # remove newline to compare strings directly + GOT=$(kubectl run --restart=Never --rm --attach --image=dersimn/netutils net-utils -- curl -s http://assets-allowed.e2e/hello_world.txt 2>/dev/null | head -n1 | tr -d '\n') + [ $? -eq 0 ] || { echo "Failed to retrieve assets"; exit 1; } + [ "$GOT" = "$WANT" ] || { echo "Incorrect assets content: want '$WANT', got '$GOT'"; exit 1; } + + - name: test tcp connection via BoundEndpoint UpstreamService + try: + - script: + content: | + WANT_REGEX="Connection to tcp-echo\.e2e .* 4242 port .* succeeded" + # 2>/dev/null to suppress kubectl default output + # first line is the contents, last line in "deleted pod" message + GOT=$(kubectl run --restart=Never --rm --attach --image=dersimn/netutils net-utils -- nc -zv tcp-echo.e2e 4242 2>/dev/null | head -n1) + + [ $? -eq 0 ] || { echo "Failed to establish tcp connection"; exit 1; } + (echo "$GOT" | grep -q "$WANT_REGEX") || { echo "Unexpected nc output: want '$WANT_REGEX' got '$GOT'"; exit 1; } diff --git a/tests/chainsaw/operator-registration/chainsaw-test.yaml b/tests/chainsaw/operator-registration/chainsaw-test.yaml index f22727a0..b7baffdc 100644 --- a/tests/chainsaw/operator-registration/chainsaw-test.yaml +++ b/tests/chainsaw/operator-registration/chainsaw-test.yaml @@ -41,9 +41,9 @@ spec: namespace: ngrok-operator type: kubernetes.io/tls data: - ("tls.crt" != null): true - ("tls.csr" != null): true - ("tls.key" != null): true + ("tls.crt" != null && "tls.crt" != ''): true + ("tls.csr" != null && "tls.csr" != ''): true + ("tls.key" != null && "tls.key" != ''): true - name: assert Configmap/ngrok-intermediate-ca exists (tunnels/forwarders will work) try: @@ -55,4 +55,4 @@ spec: name: ngrok-intermediate-ca namespace: ngrok-operator data: - ("root.crt" != null): true + ("root.crt" != null && "root.crt" != ''): true