Skip to content

Commit

Permalink
Test starting an ngrok agent in CI (#523)
Browse files Browse the repository at this point in the history
* Add ngrok-agent ci step

* Add bindings e2e tests

* Move more e2e tests to Makefile targets

* Add bindings chainsaw tests

* Add chainsaw tests for bindings services

* Try with more dollar signs

* Add make to action

* Try with alpine and make

* Try with github.workspace volume mount

* Add static assets to e2e-fixtures ; Update Makefile

* Add tcp bindings e2e test

* With /bin/sh

* With different dir

* Try with volume mount

* With debian

* With different envar approach

* With --detach

* With 60s

* With chainsaw timeouts instead

* Test with 10m timeout

* With different assertion

* With ubuntu apt ngrok rather than docker

* With ubuntu apt ngrok rather than docker

* With ubuntu apt ngrok rather than docker

* No directory to cd

* With direct asserts

* With debug step

* With trues

* With sleep

* With dump logs

* Even more logs

* With dump Secret/ngrok-operator-default-tls

* Adjust chainsaw tests to ensure tls.crt is valid

* Remove superfluous docker-build step

* Remove accidental extra steps

* With specific IMG

* With specific IMG

* With error on ngrok-op doesn't start

* With handling registry formatting

* With different k3s action

* Cleanup

* Ensure tlsSecret is set to the found/created secret

* Remove rebase artifacts
  • Loading branch information
hjkatz authored Dec 17, 2024
1 parent f7dd866 commit b63d107
Show file tree
Hide file tree
Showing 7 changed files with 236 additions and 47 deletions.
74 changes: 61 additions & 13 deletions .github/actions/build-and-test/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,11 @@ inputs:
runs:
using: "composite"
steps:
- uses: debianmaster/actions-k3s@master
- uses: jupyterhub/action-k3s-helm@v4
id: k3s
with:
version: 'latest'
k3s-channel: 'latest'
docker-enabled: true

- shell: bash
run: |
Expand Down Expand Up @@ -52,10 +53,6 @@ runs:
shell: bash
run: make test

- name: Build the Docker image
shell: bash
run: make docker-build

- name: Deploy controller to local cluster
shell: bash
env:
Expand All @@ -65,23 +62,42 @@ runs:
NGROK_AUTHTOKEN: ${{ inputs.ngrok-authtoken }}
E2E_BINDING_NAME: k8s/e2e-${{ github.run_id }}
# use the latest image built in this run
IMG: ngrok/ngrok-operator
IMG: ngrok-operator-${{ github.run_id }}
run: |
# create some namespaces for bindings tests
kubectl create ns e2e || true
# deploy ngrok-op for e2e tests
make deploy_for_e2e
make e2e-deploy
- name: Check if operator is up
shell: bash
run: |
kubectl get nodes
kubectl get pods -A
kubectl -n ngrok-operator wait --for=condition=ready pod -l app.kubernetes.io/name=ngrok-operator --timeout=1m || true
kubectl -n ngrok-operator wait --for=condition=ready pod -l app.kubernetes.io/name=ngrok-operator --timeout=1m || { echo "Failed to start ngrok-operator!"; exit 1; }
kubectl get pods -A
kubectl -n ngrok-operator describe pod -l app.kubernetes.io/name=ngrok-operator
- name: Start an ngrok agent
if: ${{ inputs.run-e2e == 'true' }}
shell: bash
run: |
# envars needed for ngrok-agent
export NGROK_API_KEY=${{ inputs.ngrok-api-key }}
export NGROK_AUTHTOKEN=${{ inputs.ngrok-authtoken }}
export E2E_BINDING_NAME=k8s/e2e-${{ github.run_id }}
export E2E_TCP_ENDPOINT_NAME=${{ github.run_id }}-tcp
# add the ngrok apt source
curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc \
| sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null \
&& echo "deb https://ngrok-agent.s3.amazonaws.com buster main" \
| sudo tee /etc/apt/sources.list.d/ngrok.list \
sudo apt-get update
sudo apt-get install -y \
make \
ngrok \
make e2e-start-ngrok
- name: Install cosign
if: ${{ inputs.run-e2e == 'true' }}
uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0
Expand All @@ -97,6 +113,38 @@ runs:
run: |
make e2e-tests
# best effort to dump ngrok k8s resources from cluster
# this helps us debug pipelines
- name: Dump k8s resources
shell: bash
if: ${{ always() }}
run: |
echo "Dumping k8s resources"
echo "=== Pods ==="
kubectl -n ngrok-operator describe pods || true
echo "=== Services ==="
kubectl -n ngrok-operator describe services || true
echo "=== Deployments ==="
kubectl -n ngrok-operator describe deployments || true
echo "=== TLS Cert ==="
kubectl -n ngrok-operator describe secret ngrok-operator-default-tls || true
echo "=== KubernetesOperators ==="
kubectl -n ngrok-operator describe KubernetesOperators || true
echo "=== BoundEndpoints ==="
kubectl -n ngrok-operator describe BoundEndpoints || true
# best effort to dump some logs
# this helps us debug pipelines
- name: Dump logs
shell: bash
if: ${{ always() }}
run: |
echo "Dumping logs"
echo "=== Forwarder ==="
kubectl -n ngrok-operator logs -l app.kubernetes.io/component=bindings-forwarder --tail=-1 || true
echo "=== API ==="
kubectl -n ngrok-operator logs -l app.kubernetes.io/component=controller --tail=-1 || true
# best effort to remove ngrok k8s resources from cluster
# this allows our finalizers to delete upstream ngrok API resources too
# that hopefully helps not pollute our ngrok-operator-ci account
Expand Down
79 changes: 51 additions & 28 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# absolute path to cwd relative to Makefile
CWD := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))

# Image URL to use all building/pushing image targets
IMG ?= ngrok-operator
IMG_REGISTRY ?= "docker.io"

# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
ENVTEST_K8S_VERSION = 1.29.0
Expand Down Expand Up @@ -37,6 +41,13 @@ CONTROLLER_GEN_PATHS = {./api/..., ./internal/controller/...}
# when true, deploy with --set oneClickDemoMode=true
DEPLOY_ONE_CLICK_DEMO_MODE ?= false

# example assets
# Note: contents must match chainsaw tests
E2E_ASSETS_DIR ?= $(CWD)/e2e-fixtures/ngrok-assets/

# default name for e2e-deploy k8s binding.name
E2E_BINDING_NAME ?= k8s/e2e-from-makefile

# Targets

.PHONY: all
Expand Down Expand Up @@ -195,29 +206,6 @@ deploy_with_bindings: _deploy-check-env-vars docker-build manifests kustomize _h
&&\
kubectl rollout restart deployment $(KUBE_DEPLOYMENT_NAME) -n $(KUBE_NAMESPACE)

.PHONY: deploy_for_e2e
deploy_for_e2e: _deploy-check-env-vars docker-build manifests kustomize _helm_setup ## Deploy controller to the K8s cluster specified in ~/.kube/config.
helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \
--namespace $(KUBE_NAMESPACE) \
--create-namespace \
--set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \
--set image.repository=$(IMG) \
--set image.tag="latest" \
--set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \
--set credentials.apiKey=$(NGROK_API_KEY) \
--set credentials.authtoken=$(NGROK_AUTHTOKEN) \
--set log.format=console \
--set log.level=debug \
--set log.stacktraceLevel=panic \
--set metaData.env=local,metaData.from=makefile \
--set bindings.enabled=true \
--set bindings.name=$(E2E_BINDING_NAME) \
--set bindings.description="Example binding for CI e2e tests" \
--set bindings.allowedURLs='{*.e2e}' \
--set bindings.serviceAnnotations.annotation1="val1" \
--set bindings.serviceAnnotations.annotation2="val2" \
--set bindings.serviceLabels.label1="val1"

.PHONY: _deploy-check-env-vars
_deploy-check-env-vars:
ifndef NGROK_API_KEY
Expand Down Expand Up @@ -308,14 +296,49 @@ helm-update-snapshots-no-deps: ## Update helm unittest snapshots without rebuild

##@ E2E tests

# NOTE: You will also need to load ngrok-operator:latest image into your local cluster for this to work
.PHONY: e2e-deploy ## Deploy the operator for e2e tests
e2e-deploy: _deploy-check-env-vars docker-build manifests kustomize _helm_setup
# create some namespaces for bindings tests
kubectl create ns e2e || true

# deploy for e2e tests
helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \
--namespace $(KUBE_NAMESPACE) \
--create-namespace \
--set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \
--set image.repository=$(IMG) \
--set image.tag="latest" \
--set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \
--set credentials.apiKey=$(NGROK_API_KEY) \
--set credentials.authtoken=$(NGROK_AUTHTOKEN) \
--set log.format=console \
--set log.level=debug \
--set log.stacktraceLevel=panic \
--set metaData.env=local,metaData.from=makefile \
--set bindings.enabled=true \
--set bindings.name=$(E2E_BINDING_NAME) \
--set bindings.description="Example binding for CI e2e tests" \
--set bindings.allowedURLs='{*.e2e}' \
--set bindings.serviceAnnotations.annotation1="val1" \
--set bindings.serviceAnnotations.annotation2="val2" \
--set bindings.serviceLabels.label1="val1"

.PHONY: e2e-start-ngrok
e2e-start-ngrok: ## Start the ngrok-agent for e2e tests
# start an agent
ngrok http file://$(E2E_ASSETS_DIR) --url http://assets-allowed.e2e --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!"
ngrok http file://$(E2E_ASSETS_DIR) --url http://assets-denied.example --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!"
ngrok tcp tcpbin.com:4242 --url tcp://tcp-echo.e2e:4242 --binding $(E2E_BINDING_NAME) & echo "WARN: Started ngrok-agent with PID $$!"

.PHONY: e2e-tests
e2e-tests: ## Run e2e tests
chainsaw test ./tests/chainsaw

.PHONY: e2e-clean
e2e-clean: ## Clean up e2e tests
kubectl delete ns e2e
kubectl delete --all boundendpoints -n ngrok-operator
kubectl delete --all services -n ngrok-operator
kubectl delete --all kubernetesoperators -n ngrok-operator
helm --namespace ngrok-operator uninstall ngrok-operator
kubectl delete ns e2e || true
kubectl delete --all boundendpoints -n ngrok-operator || true
kubectl delete --all services -n ngrok-operator || true
kubectl delete --all kubernetesoperators -n ngrok-operator || true
helm --namespace ngrok-operator uninstall ngrok-operator || true
1 change: 1 addition & 0 deletions e2e-fixtures/ngrok-assets/hello_world.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Hello from ngrok-operator
4 changes: 4 additions & 0 deletions helm/ngrok-operator/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -122,5 +122,9 @@ Return the ngrok operator image name
{{- $registryName := .Values.image.registry -}}
{{- $repositoryName := .Values.image.repository -}}
{{- $tag := .Values.image.tag | default .Chart.AppVersion | toString -}}
{{- if eq $registryName "" }}
{{- printf "%s:%s" $repositoryName $tag -}}
{{- else }}
{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
{{- end }}
{{- end -}}
6 changes: 4 additions & 2 deletions internal/controller/ngrok/kubernetesoperator_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,14 @@ func (r *KubernetesOperatorReconciler) create(ctx context.Context, ko *ngrokv1al
var tlsSecret *v1.Secret

if bindingsEnabled {
tlsSecret, err := r.findOrCreateTLSSecret(ctx, ko)
foundSecret, err := r.findOrCreateTLSSecret(ctx, ko)
if err != nil {
return err
}

// remember to set the outer secret
tlsSecret = foundSecret

createParams.Binding = &ngrok.KubernetesOperatorBindingCreate{
Name: ko.Spec.Binding.Name,
AllowedURLs: ko.Spec.Binding.AllowedURLs,
Expand Down Expand Up @@ -469,7 +472,6 @@ func extractNamespaceUIDFromMetadata(metadata string) (string, error) {
return uid, nil
}

// nolint:unused
func generateCSR(privKey *ecdsa.PrivateKey) ([]byte, error) {
subj := pkix.Name{}

Expand Down
111 changes: 111 additions & 0 deletions tests/chainsaw/bindings/chainsaw-test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
name: bindings
spec:
timeouts:
assert: 3m
steps:
- name: assert BoundEndpoint http://assets-denied.example is denied
try:
- assert:
resource:
apiVersion: bindings.k8s.ngrok.com/v1alpha1
kind: BoundEndpoint
metadata:
name: ngrok-238b1294-28ba-5de5-8713-8a2928d8a2f9 # stable hash
namespace: ngrok-operator
spec:
allowed: false
endpointURI: "http://assets-denied.example:80"
scheme: "http"
# port: <-- port is allocated and may be out of order, do not assert
target:
service: "assets-denied"
namespace: "example"
protocol: TCP
port: 80
status:
~.(endpoints):
status: "denied"

- name: assert BoundEndpoint http://assets-allowed.e2e is bound
try:
- assert:
resource:
apiVersion: bindings.k8s.ngrok.com/v1alpha1
kind: BoundEndpoint
metadata:
name: ngrok-adb90775-7749-5b56-92f4-d52ee756975b # stable hash
namespace: ngrok-operator
spec:
allowed: true
endpointURI: "http://assets-allowed.e2e:80"
scheme: "http"
# port: <-- port is allocated and may be out of order, do not assert
target:
service: "assets-allowed"
namespace: "e2e"
protocol: TCP
port: 80
status:
~.(endpoints):
status: "bound"
# TargetService
- assert:
resource:
apiVersion: v1
kind: Service
metadata:
name: assets-allowed
namespace: e2e
spec:
type: ExternalName
externalName: ngrok-adb90775-7749-5b56-92f4-d52ee756975b.ngrok-operator.svc.cluster.local # stable hash
~.(ports):
name: http
port: 80
targetPort: 80
protocol: TCP
# UpstreamService
- assert:
resource:
apiVersion: v1
kind: Service
metadata:
name: ngrok-adb90775-7749-5b56-92f4-d52ee756975b # stable hash
namespace: ngrok-operator
spec:
type: ClusterIP
~.(ports):
name: http
port: 80
# targetPort: < -- port is allocated and may be out of order, do not assert
protocol: TCP

- name: test assets retrieval via BoundEndpoint TargetService
try:
- script:
content: |
# See contents in `make e2e-start-ngrok` task
WANT="Hello from ngrok-operator"
# 2>/dev/null to suppress kubectl default output
# first line is the contents, last line in "deleted pod" message
# remove newline to compare strings directly
GOT=$(kubectl run --restart=Never --rm --attach --image=dersimn/netutils net-utils -- curl -s http://assets-allowed.e2e/hello_world.txt 2>/dev/null | head -n1 | tr -d '\n')
[ $? -eq 0 ] || { echo "Failed to retrieve assets"; exit 1; }
[ "$GOT" = "$WANT" ] || { echo "Incorrect assets content: want '$WANT', got '$GOT'"; exit 1; }
- name: test tcp connection via BoundEndpoint UpstreamService
try:
- script:
content: |
WANT_REGEX="Connection to tcp-echo\.e2e .* 4242 port .* succeeded"
# 2>/dev/null to suppress kubectl default output
# first line is the contents, last line in "deleted pod" message
GOT=$(kubectl run --restart=Never --rm --attach --image=dersimn/netutils net-utils -- nc -zv tcp-echo.e2e 4242 2>/dev/null | head -n1)
[ $? -eq 0 ] || { echo "Failed to establish tcp connection"; exit 1; }
(echo "$GOT" | grep -q "$WANT_REGEX") || { echo "Unexpected nc output: want '$WANT_REGEX' got '$GOT'"; exit 1; }
8 changes: 4 additions & 4 deletions tests/chainsaw/operator-registration/chainsaw-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ spec:
namespace: ngrok-operator
type: kubernetes.io/tls
data:
("tls.crt" != null): true
("tls.csr" != null): true
("tls.key" != null): true
("tls.crt" != null && "tls.crt" != ''): true
("tls.csr" != null && "tls.csr" != ''): true
("tls.key" != null && "tls.key" != ''): true

- name: assert Configmap/ngrok-intermediate-ca exists (tunnels/forwarders will work)
try:
Expand All @@ -55,4 +55,4 @@ spec:
name: ngrok-intermediate-ca
namespace: ngrok-operator
data:
("root.crt" != null): true
("root.crt" != null && "root.crt" != ''): true

0 comments on commit b63d107

Please sign in to comment.