Test starting an ngrok agent in CI (#523)

* Add ngrok-agent ci step * Add bindings e2e tests * Move more e2e tests to Makefile targets * Add bindings chainsaw tests * Add chainsaw tests for bindings services * Try with more dollar signs * Add make to action * Try with alpine and make * Try with github.workspace volume mount * Add static assets to e2e-fixtures ; Update Makefile * Add tcp bindings e2e test * With /bin/sh * With different dir * Try with volume mount * With debian * With different envar approach * With --detach * With 60s * With chainsaw timeouts instead * Test with 10m timeout * With different assertion * With ubuntu apt ngrok rather than docker * With ubuntu apt ngrok rather than docker * With ubuntu apt ngrok rather than docker * No directory to cd * With direct asserts * With debug step * With trues * With sleep * With dump logs * Even more logs * With dump Secret/ngrok-operator-default-tls * Adjust chainsaw tests to ensure tls.crt is valid * Remove superfluous docker-build step * Remove accidental extra steps * With specific IMG * With specific IMG * With error on ngrok-op doesn't start * With handling registry formatting * With different k3s action * Cleanup * Ensure tlsSecret is set to the found/created secret * Remove rebase artifacts * With sleep 30s * Add tcp-echo asserts * With disabled metrics-server * With different k3s provider * With inputs.github-token * Without k3d args * With k3d load image
ngrok · Dec 18, 2024 · 8921b67 · 8921b67
1 parent 3038976
commit 8921b67
Show file tree

Hide file tree

Showing 8 changed files with 301 additions and 46 deletions.
diff --git a/.github/actions/build-and-test/action.yaml b/.github/actions/build-and-test/action.yaml
@@ -2,6 +2,9 @@ name: Action - Build and Test
 description: "Build and test the operator (with optional e2e tests)"
 
 inputs:
+  github-token:
+    description: "GitHub token"
+    required: true
   run-e2e:
     description: "Run e2e tests"
     required: false
@@ -21,10 +24,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: debianmaster/actions-k3s@master
+    - uses: nolar/setup-k3d-k3s@v1
       id: k3s
       with:
-        version: 'latest'
+        version: "latest"
+        github-token: ${{ inputs.github-token }}
 
     - shell: bash
       run: |
@@ -52,10 +56,6 @@ runs:
       shell: bash
       run: make test
 
-    - name: Build the Docker image
-      shell: bash
-      run: make docker-build
-
     - name: Deploy controller to local cluster
       shell: bash
       env:
@@ -65,23 +65,49 @@ runs:
         NGROK_AUTHTOKEN: ${{ inputs.ngrok-authtoken }}
         E2E_BINDING_NAME: k8s/e2e-${{ github.run_id }}
         # use the latest image built in this run
-        IMG: ngrok/ngrok-operator
+        IMG: ngrok-operator-${{ github.run_id }}
       run: |
-        # create some namespaces for bindings tests
-        kubectl create ns e2e || true
+        make e2e-deploy
 
-        # deploy ngrok-op for e2e tests
-        make deploy_for_e2e
+        # also load the image so the deploy can succeed
+        k3d image import $IMG
 
     - name: Check if operator is up
       shell: bash
       run: |
         kubectl get nodes
         kubectl get pods -A
-        kubectl -n ngrok-operator wait --for=condition=ready pod -l app.kubernetes.io/name=ngrok-operator --timeout=1m || true
+        kubectl -n ngrok-operator wait --for=condition=ready pod -l app.kubernetes.io/name=ngrok-operator --timeout=1m || { echo "Failed to start ngrok-operator!"; exit 1; }
         kubectl get pods -A
         kubectl -n ngrok-operator describe pod -l app.kubernetes.io/name=ngrok-operator
 
+    - name: Start an ngrok agent
+      if: ${{ inputs.run-e2e == 'true' }}
+      shell: bash
+      run: |
+        # envars needed for ngrok-agent
+        export NGROK_API_KEY=${{ inputs.ngrok-api-key }}
+        export NGROK_AUTHTOKEN=${{ inputs.ngrok-authtoken }}
+        export E2E_BINDING_NAME=k8s/e2e-${{ github.run_id }}
+        export E2E_TCP_ENDPOINT_NAME=${{ github.run_id }}-tcp
+
+        # add the ngrok apt source
+        curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc \
+          | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null \
+          && echo "deb https://ngrok-agent.s3.amazonaws.com buster main" \
+          | sudo tee /etc/apt/sources.list.d/ngrok.list \
+
+        sudo apt-get update
+        sudo apt-get install -y \
+          make \
+          ngrok \
+
+        make e2e-start-ngrok
+
+        # allow the ngrok-operator to settle
+        echo "Sleeping 30s to allow ngrok-operator to settle..."
+        sleep 30
+
     - name: Install cosign
       if: ${{ inputs.run-e2e == 'true' }}
       uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0
@@ -97,6 +123,38 @@ runs:
       run: |
         make e2e-tests
 
+    # best effort to dump ngrok k8s resources from cluster
+    # this helps us debug pipelines
+    - name: Dump k8s resources
+      shell: bash
+      if: ${{ always() }}
+      run: |
+        echo "Dumping k8s resources"
+        echo "=== Pods ==="
+        kubectl -n ngrok-operator describe pods || true
+        echo "=== Services ==="
+        kubectl -n ngrok-operator describe services || true
+        echo "=== Deployments ==="
+        kubectl -n ngrok-operator describe deployments || true
+        echo "=== TLS Cert ==="
+        kubectl -n ngrok-operator describe secret ngrok-operator-default-tls || true
+        echo "=== KubernetesOperators ==="
+        kubectl -n ngrok-operator describe KubernetesOperators || true
+        echo "=== BoundEndpoints ==="
+        kubectl -n ngrok-operator describe BoundEndpoints || true
+
+    # best effort to dump some logs
+    # this helps us debug pipelines
+    - name: Dump logs
+      shell: bash
+      if: ${{ always() }}
+      run: |
+        echo "Dumping logs"
+        echo "=== Forwarder ==="
+        kubectl -n ngrok-operator logs -l app.kubernetes.io/component=bindings-forwarder --tail=-1 || true
+        echo "=== API ==="
+        kubectl -n ngrok-operator logs -l app.kubernetes.io/component=controller --tail=-1 || true
+
     # best effort to remove ngrok k8s resources from cluster
     # this allows our finalizers to delete upstream ngrok API resources too
     # that hopefully helps not pollute our ngrok-operator-ci account

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -111,6 +111,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: "./.github/actions/build-and-test"
       with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
         # this workflow is for incoming PRs, so we want to skip e2e tests
         # and deploy the demo mode because our api keys are not available
         # on contributor's forks
@@ -161,6 +162,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: "./.github/actions/build-and-test"
       with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
         run-e2e: true
         go-version: ${{ env.GO_VERSION }}
         ngrok-api-key: ${{ secrets.NGROK_CI_API_KEY }}

diff --git a/Makefile b/Makefile
@@ -1,5 +1,9 @@
+# absolute path to cwd relative to Makefile
+CWD := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
 # Image URL to use all building/pushing image targets
 IMG ?= ngrok-operator
+IMG_REGISTRY ?= "docker.io"
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.29.0
@@ -37,6 +41,13 @@ CONTROLLER_GEN_PATHS = {./api/..., ./internal/controller/...}
 # when true, deploy with --set oneClickDemoMode=true
 DEPLOY_ONE_CLICK_DEMO_MODE ?= false
 
+# example assets
+# Note: contents must match chainsaw tests
+E2E_ASSETS_DIR ?= $(CWD)/e2e-fixtures/ngrok-assets/
+
+# default name for e2e-deploy k8s binding.name
+E2E_BINDING_NAME ?= k8s/e2e-from-makefile
+
 # Targets
 
 .PHONY: all
@@ -195,29 +206,6 @@ deploy_with_bindings: _deploy-check-env-vars docker-build manifests kustomize _h
 		&&\
 	kubectl rollout restart deployment $(KUBE_DEPLOYMENT_NAME) -n $(KUBE_NAMESPACE)
 
-.PHONY: deploy_for_e2e
-deploy_for_e2e: _deploy-check-env-vars docker-build manifests kustomize _helm_setup ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \
-		--namespace $(KUBE_NAMESPACE) \
-		--create-namespace \
-		--set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \
-		--set image.repository=$(IMG) \
-		--set image.tag="latest" \
-		--set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \
-		--set credentials.apiKey=$(NGROK_API_KEY) \
-		--set credentials.authtoken=$(NGROK_AUTHTOKEN) \
-		--set log.format=console \
-		--set log.level=debug \
-		--set log.stacktraceLevel=panic \
-		--set metaData.env=local,metaData.from=makefile \
-		--set bindings.enabled=true \
-		--set bindings.name=$(E2E_BINDING_NAME) \
-		--set bindings.description="Example binding for CI e2e tests" \
-		--set bindings.allowedURLs='{*.e2e}' \
-		--set bindings.serviceAnnotations.annotation1="val1" \
-		--set bindings.serviceAnnotations.annotation2="val2" \
-		--set bindings.serviceLabels.label1="val1"
-
 .PHONY: _deploy-check-env-vars
 _deploy-check-env-vars:
 ifndef NGROK_API_KEY
@@ -308,14 +296,49 @@ helm-update-snapshots-no-deps: ## Update helm unittest snapshots without rebuild
 
 ##@ E2E tests
 
+# NOTE: You will also need to load ngrok-operator:latest image into your local cluster for this to work
+.PHONY: e2e-deploy ## Deploy the operator for e2e tests
+e2e-deploy: _deploy-check-env-vars docker-build manifests kustomize _helm_setup
+	# create some namespaces for bindings tests
+	kubectl create ns e2e || true
+
+    # deploy for e2e tests
+	helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \
+		--namespace $(KUBE_NAMESPACE) \
+		--create-namespace \
+		--set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \
+		--set image.repository=$(IMG) \
+		--set image.tag="latest" \
+		--set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \
+		--set credentials.apiKey=$(NGROK_API_KEY) \
+		--set credentials.authtoken=$(NGROK_AUTHTOKEN) \
+		--set log.format=console \
+		--set log.level=debug \
+		--set log.stacktraceLevel=panic \
+		--set metaData.env=local,metaData.from=makefile \
+		--set bindings.enabled=true \
+		--set bindings.name=$(E2E_BINDING_NAME) \
+		--set bindings.description="Example binding for CI e2e tests" \
+		--set bindings.allowedURLs='{*.e2e}' \
+		--set bindings.serviceAnnotations.annotation1="val1" \
+		--set bindings.serviceAnnotations.annotation2="val2" \
+		--set bindings.serviceLabels.label1="val1"
+
+.PHONY: e2e-start-ngrok
+e2e-start-ngrok: ## Start the ngrok-agent for e2e tests
+	# start an agent
+	ngrok http file://$(E2E_ASSETS_DIR) --url http://assets-allowed.e2e --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!"
+	ngrok http file://$(E2E_ASSETS_DIR) --url http://assets-denied.example --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!"
+	ngrok tcp tcpbin.com:4242 --url tcp://tcp-echo.e2e:4242 --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!"
+
 .PHONY: e2e-tests
 e2e-tests: ## Run e2e tests
 	chainsaw test ./tests/chainsaw
 
 .PHONY: e2e-clean
 e2e-clean: ## Clean up e2e tests
-	kubectl delete ns e2e
-	kubectl delete --all boundendpoints -n ngrok-operator
-	kubectl delete --all services -n ngrok-operator
-	kubectl delete --all kubernetesoperators -n ngrok-operator
-	helm --namespace ngrok-operator uninstall ngrok-operator
+	kubectl delete ns e2e || true
+	kubectl delete --all boundendpoints -n ngrok-operator || true
+	kubectl delete --all services -n ngrok-operator || true
+	kubectl delete --all kubernetesoperators -n ngrok-operator || true
+	helm --namespace ngrok-operator uninstall ngrok-operator || true
diff --git a/e2e-fixtures/ngrok-assets/hello_world.txt b/e2e-fixtures/ngrok-assets/hello_world.txt
@@ -0,0 +1 @@
+Hello from ngrok-operator
diff --git a/helm/ngrok-operator/templates/_helpers.tpl b/helm/ngrok-operator/templates/_helpers.tpl
@@ -122,5 +122,9 @@ Return the ngrok operator image name
 {{- $registryName := .Values.image.registry -}}
 {{- $repositoryName := .Values.image.repository -}}
 {{- $tag := .Values.image.tag | default .Chart.AppVersion | toString -}}
+{{- if eq $registryName "" }}
+{{- printf "%s:%s" $repositoryName $tag -}}
+{{- else }}
 {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
+{{- end }}
 {{- end -}}
diff --git a/internal/controller/ngrok/kubernetesoperator_controller.go b/internal/controller/ngrok/kubernetesoperator_controller.go
@@ -147,11 +147,14 @@ func (r *KubernetesOperatorReconciler) create(ctx context.Context, ko *ngrokv1al
 	var tlsSecret *v1.Secret
 
 	if bindingsEnabled {
-		tlsSecret, err := r.findOrCreateTLSSecret(ctx, ko)
+		foundSecret, err := r.findOrCreateTLSSecret(ctx, ko)
 		if err != nil {
 			return err
 		}
 
+		// remember to set the outer secret
+		tlsSecret = foundSecret
+
 		createParams.Binding = &ngrok.KubernetesOperatorBindingCreate{
 			Name:        ko.Spec.Binding.Name,
 			AllowedURLs: ko.Spec.Binding.AllowedURLs,
@@ -469,7 +472,6 @@ func extractNamespaceUIDFromMetadata(metadata string) (string, error) {
 	return uid, nil
 }
 
-// nolint:unused
 func generateCSR(privKey *ecdsa.PrivateKey) ([]byte, error) {
 	subj := pkix.Name{}