diff --git a/.github/actions/build-and-test/action.yaml b/.github/actions/build-and-test/action.yaml new file mode 100644 index 00000000..58008498 --- /dev/null +++ b/.github/actions/build-and-test/action.yaml @@ -0,0 +1,102 @@ +name: Action - Build and Test +description: "Build and test the operator (with optional e2e tests)" + +inputs: + run-e2e: + description: "Run e2e tests" + required: false + default: "false" + go-version: + description: "Go version to use" + required: true + ngrok-api-key: + description: "NGROK_API_KEY for e2e tests, if enabled" + required: false + default: "fake-api-key" + ngrok-authtoken: + description: "NGROK_AUTHTOKEN for e2e tests, if enabled" + required: false + default: "fake-authtoken" + +runs: + using: "composite" + steps: + - uses: debianmaster/actions-k3s@master + id: k3s + with: + version: 'latest' + + - shell: bash + run: | + kubectl get nodes + kubectl get pods -A + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ inputs.go-version }} + + - name: Build + shell: bash + run: make build + + - name: Lint + shell: bash + run: make lint + + - name: Setup Envtest + shell: bash + run: make envtest + + - name: Test + shell: bash + run: make test + + - name: Build the Docker image + shell: bash + run: make docker-build + + - name: Deploy controller to local cluster + shell: bash + env: + # deploy with 1-click demo mode when not running e2e tests + DEPLOY_ONE_CLICK_DEMO_MODE: ${{ inputs.run-e2e == 'true' && 'false' || 'true' }} + NGROK_API_KEY: ${{ inputs.ngrok-api-key }} + NGROK_AUTHTOKEN: ${{ inputs.ngrok-authtoken }} + E2E_BINDING_NAME: k8s/e2e-${{ github.run_id }} + run: | + # create some namespaces for bindings tests + kubectl create ns e2e || true + + # deploy ngrok-op for e2e tests + make deploy_for_e2e + + - name: Check if controller is up + shell: bash + run: | + kubectl get nodes + kubectl get pods -A + + - name: Install cosign + if: ${{ inputs.run-e2e == 'true' }} + uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 + - name: Install chainsaw + if: ${{ inputs.run-e2e == 'true' }} + uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11 + with: + verify: true + + - name: Run e2e tests + shell: bash + if: ${{ inputs.run-e2e == 'true' }} + run: | + make e2e-tests + + # best effort to remove ngrok k8s resources from cluster + # this allows our finalizers to delete upstream ngrok API resources too + # that hopefully helps not pollute our ngrok-operator-ci account + - name: Cleanup e2e tests + shell: bash + if: ${{ inputs.run-e2e == 'true' }} + run: | + make e2e-clean diff --git a/.github/actions/changes/action.yaml b/.github/actions/changes/action.yaml new file mode 100644 index 00000000..f444b6aa --- /dev/null +++ b/.github/actions/changes/action.yaml @@ -0,0 +1,52 @@ +name: Action - Changes +description: "Detect changes in the repository" + +outputs: + charts: + description: "If any part of Helm charts have changed" + value: ${{ steps.filter.outputs.charts }} + chartyaml: + description: "If the Helm Chart.yaml has changed" + value: ${{ steps.filter.outputs.chartyaml }} + go: + description: "If the go (build) files have changed" + value: ${{ steps.filter.outputs.go }} + tag: + description: "If the tag (VERSION) has changed" + value: ${{ steps.filter.outputs.tag }} + tests: + description: "If the tests have changed" + value: ${{ steps.filter.outputs.tests }} + make: + description: "If the Makefile has changed" + value: ${{ steps.filter.outputs.make }} + +runs: + using: "composite" + steps: + - name: filter + id: filter + uses: dorny/paths-filter@v2.11.1 + with: + filters: | + chartyaml: + - 'helm/ngrok-operator/Chart.yaml' + charts: + - 'helm/ngrok-operator/**' + - 'scripts/e2e.sh' + go: + - '**.go' + - 'go.mod' + - 'go.sum' + - 'cmd/**' + - 'internal/**' + - 'pkg/**' + - 'Dockerfile' + - 'scripts/e2e.sh' + - 'VERSION' + tests: + - 'test/**' + make: + - 'Makefile' + tag: + - 'VERSION' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 641464fd..c55dcc47 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,8 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + pull_request_target: + branches: [ "main" ] env: GO_VERSION: '1.23' @@ -13,38 +15,20 @@ jobs: changes: runs-on: ubuntu-latest - outputs: - charts: ${{ steps.filter.outputs.charts }} - chartyaml: ${{ steps.filter.outputs.chartyaml }} - go: ${{ steps.filter.outputs.go }} - tag: ${{ steps.filter.outputs.tag }} permissions: contents: read pull-requests: read + outputs: + charts: ${{ steps.changes.outputs.charts }} + chartyaml: ${{ steps.changes.outputs.chartyaml }} + go: ${{ steps.changes.outputs.go }} + tag: ${{ steps.changes.outputs.tag }} + tests: ${{ steps.changes.outputs.tests }} + make: ${{ steps.changes.outputs.make }} steps: - - name: Checkout repo - uses: actions/checkout@v3 - - id: filter - uses: dorny/paths-filter@v2.11.1 - with: - filters: | - chartyaml: - - 'helm/ngrok-operator/Chart.yaml' - charts: - - 'helm/ngrok-operator/**' - - 'scripts/e2e.sh' - go: - - '**.go' - - 'go.mod' - - 'go.sum' - - 'cmd/**' - - 'internal/**' - - 'pkg/**' - - 'Dockerfile' - - 'scripts/e2e.sh' - - 'VERSION' - tag: - - 'VERSION' + - uses: actions/checkout@v3 + - uses: "./.github/actions/changes" + id: changes # Make sure that Kubebuilder autogenerated files are up to date. kubebuilder-diff: @@ -105,54 +89,42 @@ jobs: - run: git diff --exit-code go.mod - run: git diff --exit-code go.sum + echo: + runs-on: ubuntu-latest + needs: [changes] + steps: + - uses: actions/checkout@v3 + - run: | + echo "go: ${{ needs.changes.outputs.go }}" + echo "charts: ${{ needs.changes.outputs.charts }}" + echo "chartyaml: ${{ needs.changes.outputs.chartyaml }}" + echo "tests: ${{ needs.changes.outputs.tests }}" + echo "make: ${{ needs.changes.outputs.make }}" - build: + build-and-test: runs-on: ubuntu-latest needs: - changes - kubebuilder-diff if: | (needs.changes.outputs.go == 'true') || - (needs.changes.outputs.charts == 'true') + (needs.changes.outputs.charts == 'true') || + (needs.changes.outputs.chartyaml == 'true') || + (needs.changes.outputs.tests == 'true') || + (needs.changes.outputs.make == 'true') + permissions: + contents: read + pull-requests: read steps: - uses: actions/checkout@v3 - - - uses: debianmaster/actions-k3s@master - id: k3s - with: - version: 'latest' - - run: | - kubectl get nodes - kubectl get pods -A - - - name: Set up Go - uses: actions/setup-go@v3 + - uses: "./.github/actions/build-and-test" with: + # this workflow is for incoming PRs, so we want to skip e2e tests + # and deploy the demo mode because our api keys are not available + # on contributor's forks + run-e2e: false go-version: ${{ env.GO_VERSION }} - - name: Build - run: make build - - - name: Lint - run: make lint - - - name: Setup Envtest - run: make envtest - - - name: Test - run: make test - - - name: Build the Docker image - run: make docker-build - - - name: Deploy controller to local cluster - run: make deploy NGROK_API_KEY=fake-ci-key NGROK_AUTHTOKEN=fake-ci-token - - - name: Check if controller is up - run: | - kubectl get nodes - kubectl get pods -A - helm: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..477eed5e --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,51 @@ +name: Tests +on: + push: + branches: [ "main" ] + pull_request_target: + branches: [ "main" ] + +env: + GO_VERSION: '1.23' + DOCKER_BUILDX_PLATFORMS: linux/amd64,linux/arm64 + +jobs: + changes: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + charts: ${{ steps.changes.outputs.charts }} + chartyaml: ${{ steps.changes.outputs.chartyaml }} + go: ${{ steps.changes.outputs.go }} + tag: ${{ steps.changes.outputs.tag }} + tests: ${{ steps.changes.outputs.tests }} + make: ${{ steps.changes.outputs.make }} + steps: + - uses: actions/checkout@v3 + - uses: "./.github/actions/changes" + id: changes + + build-and-test: + runs-on: ubuntu-latest + needs: + - changes + - kubebuilder-diff + if: | + (needs.changes.outputs.go == 'true') || + (needs.changes.outputs.charts == 'true') || + (needs.changes.outputs.chartyaml == 'true') || + (needs.changes.outputs.tests == 'true') || + (needs.changes.outputs.make == 'true') + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v3 + - uses: "./.github/actions/build-and-test" + with: + run-e2e: true + go-version: ${{ env.GO_VERSION }} + ngrok-api-key: ${{ secrets.NGROK_CI_API_KEY }} + ngrok-authtoken: ${{ secrets.NGROK_CI_AUTHTOKEN }} diff --git a/Makefile b/Makefile index b2872395..fe16b477 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ - # Image URL to use all building/pushing image targets IMG ?= ngrok-operator @@ -32,6 +31,11 @@ HELM_TEMPLATES_DIR = $(HELM_CHART_DIR)/templates CONTROLLER_GEN_PATHS = {./api/..., ./internal/controller/...} +# Default Environment Variables + +# when true, deploy with --set oneClickDemoMode=true +DEPLOY_ONE_CLICK_DEMO_MODE ?= false + # Targets .PHONY: all @@ -190,6 +194,29 @@ deploy_with_bindings: _deploy-check-env-vars docker-build manifests kustomize _h &&\ kubectl rollout restart deployment $(KUBE_DEPLOYMENT_NAME) -n $(KUBE_NAMESPACE) +.PHONY: deploy_for_e2e +deploy_for_e2e: _deploy-check-env-vars docker-build manifests kustomize _helm_setup ## Deploy controller to the K8s cluster specified in ~/.kube/config. + helm upgrade $(HELM_RELEASE_NAME) $(HELM_CHART_DIR) --install \ + --namespace $(KUBE_NAMESPACE) \ + --create-namespace \ + --set oneClickDemoMode=$(DEPLOY_ONE_CLICK_DEMO_MODE) \ + --set image.repository=$(IMG) \ + --set image.tag="latest" \ + --set podAnnotations."k8s\.ngrok\.com/test"="\{\"env\": \"e2e\"\}" \ + --set credentials.apiKey=$(NGROK_API_KEY) \ + --set credentials.authtoken=$(NGROK_AUTHTOKEN) \ + --set log.format=console \ + --set log.level=debug \ + --set log.stacktraceLevel=panic \ + --set metaData.env=local,metaData.from=makefile \ + --set bindings.enabled=true \ + --set bindings.name=$(E2E_BINDING_NAME) \ + --set bindings.description="Example binding for CI e2e tests" \ + --set bindings.allowedURLs='{*.e2e}' \ + --set bindings.serviceAnnotations.annotation1="val1" \ + --set bindings.serviceAnnotations.annotation2="val2" \ + --set bindings.serviceLabels.label1="val1" + .PHONY: _deploy-check-env-vars _deploy-check-env-vars: ifndef NGROK_API_KEY @@ -277,3 +304,17 @@ helm-update-snapshots: _helm_setup ## Update helm unittest snapshots helm-update-snapshots-no-deps: ## Update helm unittest snapshots without rebuilding dependencies $(MAKE) -C $(HELM_CHART_DIR) update-snapshots + +##@ E2E tests + +.PHONY: e2e-tests +e2e-tests: ## Run e2e tests + chainsaw test ./tests/chainsaw + +.PHONY: e2e-clean +e2e-clean: ## Clean up e2e tests + kubectl delete ns e2e + kubectl delete --all boundendpoints -n ngrok-operator + kubectl delete --all services -n ngrok-operator + kubectl delete --all kubernetesoperators -n ngrok-operator + helm uninstall ngrok-operator diff --git a/flake.lock b/flake.lock index c0137b3a..324c54e6 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724395761, - "narHash": "sha256-zRkDV/nbrnp3Y8oCADf5ETl1sDrdmAW6/bBVJ8EbIdQ=", + "lastModified": 1731531548, + "narHash": "sha256-sz8/v17enkYmfpgeeuyzniGJU0QQBfmAjlemAUYhfy8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ae815cee91b417be55d43781eb4b73ae1ecc396c", + "rev": "24f0d4acd634792badd6470134c387a3b039dace", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index f481eb1f..8f238164 100644 --- a/flake.nix +++ b/flake.nix @@ -19,6 +19,7 @@ kubebuilder jq yq + kyverno-chainsaw ]; }; })); diff --git a/tests/chainsaw/finalizers/chainsaw-test.yaml b/tests/chainsaw/finalizers/chainsaw-test.yaml new file mode 100644 index 00000000..b04cb421 --- /dev/null +++ b/tests/chainsaw/finalizers/chainsaw-test.yaml @@ -0,0 +1,21 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: finalizers +spec: + steps: + - name: create an ingress + try: + - create: + file: ./ingress.yaml + - name: verify finalizers + try: + - assert: + resource: + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: minimal-ingress-https + finalizers: + - k8s.ngrok.com/finalizer diff --git a/tests/chainsaw/finalizers/ingress.yaml b/tests/chainsaw/finalizers/ingress.yaml new file mode 100644 index 00000000..b7516209 --- /dev/null +++ b/tests/chainsaw/finalizers/ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minimal-ingress-https +spec: + ingressClassName: ngrok + rules: + - host: foo.bar.com + http: + paths: + - path: /https-echo-plain + pathType: Prefix + backend: + service: + name: https-echo-svc + port: + number: 80 + - path: /https-echo-tls + pathType: Prefix + backend: + service: + name: https-echo-svc + port: + number: 443 diff --git a/tests/chainsaw/sanity-checks/agent-pod.yaml b/tests/chainsaw/sanity-checks/agent-pod.yaml new file mode 100644 index 00000000..1d4e0304 --- /dev/null +++ b/tests/chainsaw/sanity-checks/agent-pod.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app.kubernetes.io/component: agent + namespace: ngrok-operator +status: + (conditions[?type == 'Ready']): + - status: "True" + ~.containerStatuses: + ready: true + restartCount: 0 + phase: Running diff --git a/tests/chainsaw/sanity-checks/chainsaw-test.yaml b/tests/chainsaw/sanity-checks/chainsaw-test.yaml new file mode 100644 index 00000000..e6f4fb75 --- /dev/null +++ b/tests/chainsaw/sanity-checks/chainsaw-test.yaml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: sanity-checks +spec: + steps: + - name: check ingress class exists + try: + - assert: + file: ./ingress-class.yaml + - name: check credentials exist + try: + - assert: + file: ./ngrok-operator-credentials.yaml + - name: check operator manager config + try: + - assert: + file: ./ngrok-operator-manager-config.yaml + - name: check operator pods are running + try: + - assert: + file: ./operator-pod.yaml + - name: check agent pods are running + try: + - assert: + file: ./agent-pod.yaml diff --git a/tests/chainsaw/sanity-checks/ingress-class.yaml b/tests/chainsaw/sanity-checks/ingress-class.yaml new file mode 100644 index 00000000..eca19ece --- /dev/null +++ b/tests/chainsaw/sanity-checks/ingress-class.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: ngrok +spec: + controller: k8s.ngrok.com/ingress-controller diff --git a/tests/chainsaw/sanity-checks/ngrok-operator-credentials.yaml b/tests/chainsaw/sanity-checks/ngrok-operator-credentials.yaml new file mode 100644 index 00000000..566c5bc8 --- /dev/null +++ b/tests/chainsaw/sanity-checks/ngrok-operator-credentials.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ngrok-operator-credentials + namespace: ngrok-operator +type: Opaque +data: + API_KEY: {} + AUTHTOKEN: {} diff --git a/tests/chainsaw/sanity-checks/ngrok-operator-manager-config.yaml b/tests/chainsaw/sanity-checks/ngrok-operator-manager-config.yaml new file mode 100644 index 00000000..b29a2c4b --- /dev/null +++ b/tests/chainsaw/sanity-checks/ngrok-operator-manager-config.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ngrok-operator-manager-config + namespace: ngrok-operator +data: + controller_manager_config.yaml: | + apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 + kind: ControllerManagerConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + leaderElection: + leaderElect: true + resourceName: ngrok-operator-leader diff --git a/tests/chainsaw/sanity-checks/operator-pod.yaml b/tests/chainsaw/sanity-checks/operator-pod.yaml new file mode 100644 index 00000000..2ae589c9 --- /dev/null +++ b/tests/chainsaw/sanity-checks/operator-pod.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app.kubernetes.io/component: controller + namespace: ngrok-operator +status: + (conditions[?type == 'Ready']): + - status: "True" + ~.containerStatuses: + ready: true + restartCount: 0 + phase: Running