test: add tests for repo and policy add,remove

.github/workflows/remote-controller.yaml

@@ -94,8 +94,8 @@ jobs:
         
     - name: Configure node IP in kind-config.yaml
       run: |
-        docker network create remote-controller
-        LAGOON_KIND_CIDR_BLOCK=$(docker network inspect remote-controller | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"') 
+        docker network create kind
+        LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"') 
         export KIND_NODE_IP=$(echo ${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $1,$2,$3,240}' OFS='.')
         envsubst < test-resources/test-suite.kind-config.yaml.tpl > test-resources/test-suite.kind-config.yaml
 
@@ -110,7 +110,7 @@ jobs:
 
     - name: Check node IP matches kind configuration
       run: |
-        LAGOON_KIND_CIDR_BLOCK=$(docker network inspect remote-controller | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"')
+        LAGOON_KIND_CIDR_BLOCK=$(docker network inspect kind | jq '. [0].IPAM.Config[0].Subnet' | tr -d '"')
         NODE_IP=$(echo ${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $1,$2,$3,240}' OFS='.')
         echo Checking for NODE_IP "$NODE_IP"
         grep $NODE_IP test-resources/test-suite.kind-config.yaml
@@ -127,4 +127,4 @@ jobs:
 
     - name: Run github/test-e2e
       run: |
-        make github/test-e2e HARBOR_VERSION=${{matrix.harbor}} OVERRIDE_BUILD_DEPLOY_DIND_IMAGE="${{matrix.lagoon_build_image}}"
+        make github/test-e2e HARBOR_VERSION=${{matrix.harbor}} OVERRIDE_BUILD_DEPLOY_DIND_IMAGE="${{matrix.lagoon_build_image}}" KIND_NETWORK=kind

.gitignore

@@ -24,4 +24,6 @@ bin
 *~
 
 test-resources/test-suite.kind-config.yaml
-test-resources/test-suite.metallb-pool.yaml
+test-resources/test-suite.metallb-pool.yaml
+
+local-dev

Makefile

@@ -18,12 +18,33 @@ INGRESS_VERSION=4.9.1
 HARBOR_VERSION=1.14.3
 
 KIND_CLUSTER ?= remote-controller
+KIND_NETWORK ?= remote-controller
 
 TIMEOUT = 30m
 HELM = helm
 KUBECTL = kubectl
 JQ = jq
 
+ARCH := $(shell uname | tr '[:upper:]' '[:lower:]')
+
+KIND = $(realpath ./local-dev/kind)
+KIND_VERSION = v0.25.0
+
+.PHONY: local-dev/kind
+local-dev/kind:
+ifeq ($(KIND_VERSION), $(shell kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p'))
+	$(info linking local kind version $(KIND_VERSION))
+	ln -sf $(shell command -v kind) ./local-dev/kind
+else
+ifneq ($(KIND_VERSION), $(shell ./local-dev/kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p'))
+	$(info downloading kind version $(KIND_VERSION) for $(ARCH))
+	mkdir -p local-dev
+	rm local-dev/kind || true
+	curl -sSLo local-dev/kind https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(ARCH)-amd64
+	chmod a+x local-dev/kind
+endif
+endif
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -140,7 +161,7 @@ endif
 
 .PHONY: install-metallb
 install-metallb:
-	LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_CLUSTER) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \
+	LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \
 	export LAGOON_KIND_NETWORK_RANGE=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.')/29 && \
 	$(HELM) upgrade \
 		--install \
@@ -152,7 +173,7 @@ install-metallb:
 		metallb \
 		metallb/metallb && \
 	$$(envsubst < test-resources/test-suite.metallb-pool.yaml.tpl > test-resources/test-suite.metallb-pool.yaml) && \
-	$(KUBECTL) apply -f test-resources/test-suite.metallb-pool.yaml \
+	$(KUBECTL) apply -f test-resources/test-suite.metallb-pool.yaml
 
 # cert-manager is used to allow self-signed certificates to be generated automatically by ingress in the same way lets-encrypt would
 .PHONY: install-certmanager
@@ -246,11 +267,12 @@ install-lagoon-remote: install-registry
 
 .PHONY: create-kind-cluster
 create-kind-cluster:
-	docker network inspect $(KIND_CLUSTER) >/dev/null || docker network create $(KIND_CLUSTER) \
-		&& LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_CLUSTER) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \
+	docker network inspect $(KIND_NETWORK) >/dev/null || docker network create $(KIND_NETWORK) \
+		&& LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \
 		&& export KIND_NODE_IP=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.') \
  		&& envsubst < test-resources/test-suite.kind-config.yaml.tpl > test-resources/test-suite.kind-config.yaml \
- 		&& kind create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml
+		&& export KIND_EXPERIMENTAL_DOCKER_NETWORK=$(KIND_NETWORK) \
+		&& $(KIND) create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml
 
 # Create a kind cluster locally and run the test e2e test suite against it
 .PHONY: kind/test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up locally
@@ -259,15 +281,15 @@ kind/test-e2e: create-kind-cluster install-lagoon-remote kind/re-test-e2e
 .PHONY: local-kind/test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up locally
 kind/re-test-e2e:
 	export KIND_CLUSTER=$(KIND_CLUSTER) && \
-	kind export kubeconfig --name=$(KIND_CLUSTER) && \
+	$(KIND) export kubeconfig --name=$(KIND_CLUSTER) && \
 	export HARBOR_VERSION=$(HARBOR_VERSION) && \
 	export OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=$(OVERRIDE_BUILD_DEPLOY_DIND_IMAGE) && \
 	$(MAKE) test-e2e
 
 .PHONY: clean
 kind/clean:
 	docker compose down && \
-	kind delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_CLUSTER)
+	$(KIND) delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_NETWORK)
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up inside github action.

config/default/manager_auth_proxy_patch.yaml

@@ -24,6 +24,7 @@ spec:
         - "--enable-deprecated-apis"
         - "--lagoon-feature-flag-support-k8upv2"
         - "--skip-tls-verify"
+        - "--cleanup-harbor-repository-on-delete" # enabled for tests
         ports:
         - containerPort: 8443
           name: https

test/e2e/e2e_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package e2e
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
@@ -54,6 +55,10 @@ var (
 		"lagoon_tasks_running_current",
 		"lagoon_tasks_started_total",
 	}
+
+	createPolicyWant        = `{"algorithm":"or","rules":[{"action":"retain","params":{"latestPulledN":3},"scope_selectors":{"repository":[{"decoration":"repoMatches","kind":"doublestar","pattern":"[^pr\\-]*/*"}]},"tag_selectors":[{"decoration":"matches","extras":"{\"untagged\":true}","kind":"doublestar","pattern":"**"}],"template":"latestPulledN"},{"action":"retain","params":{"latestPulledN":1},"scope_selectors":{"repository":[{"decoration":"repoMatches","kind":"doublestar","pattern":"pr-*"}]},"tag_selectors":[{"decoration":"matches","extras":"{\"untagged\":true}","kind":"doublestar","pattern":"**"}],"template":"latestPulledN"}],"scope":{"level":"project"},"trigger":{"kind":"Schedule","settings":{"cron":"0 3 3 * * 3"}}}`
+	deletePolicyWant        = `{"algorithm":"or","rules":[],"scope":{"level":"project"},"trigger":{"kind":"Schedule","settings":{"cron":""}}}`
+	projectRepositoriesWant = `[{"artifact_count":1,"name":"nginx-example/main/nginx","pull_count":1}]`
 )
 
 func init() {
@@ -62,19 +67,30 @@ func init() {
 }
 
 var _ = Describe("controller", Ordered, func() {
+	// get the ingress lb ip for use later
+	ip, err := utils.GetIngressLB()
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
 	BeforeAll(func() {
 		By("start local services")
 		Expect(utils.StartLocalServices()).To(Succeed())
 
+		By("removing manager namespace")
+		cmd := exec.Command("kubectl", "delete", "ns", namespace)
+		_, _ = utils.Run(cmd)
+
 		By("creating manager namespace")
-		cmd := exec.Command("kubectl", "create", "ns", namespace)
+		cmd = exec.Command("kubectl", "create", "ns", namespace)
 		_, _ = utils.Run(cmd)
 
 		// when running a re-test, it is best to make sure the old namespace doesn't exist
 		By("removing existing test resources")
 		// remove the old namespace
 		cmd = exec.Command("kubectl", "delete", "ns", "nginx-example-main")
 		_, _ = utils.Run(cmd)
+		By("delete harbor project")
+		_ = utils.DeleteHarborProject(ip, "nginx-example")
+
 		// clean up the k8up crds
 		utils.UninstallK8upCRDs()
 	})
@@ -84,10 +100,6 @@ var _ = Describe("controller", Ordered, func() {
 		By("stop metrics consumer")
 		utils.StopMetricsConsumer()
 
-		By("removing manager namespace")
-		cmd := exec.Command("kubectl", "delete", "ns", namespace)
-		_, _ = utils.Run(cmd)
-
 		By("stop local services")
 		utils.StopLocalServices()
 	})
@@ -164,22 +176,7 @@ var _ = Describe("controller", Ordered, func() {
 			for _, name := range []string{"7m5zypx", "8m5zypx", "9m5zypx", "1m5zypx"} {
 				if name == "9m5zypx" {
 					By("creating a LagoonBuild resource via rabbitmq")
-					cmd = exec.Command(
-						"curl",
-						"-s",
-						"-u",
-						"guest:guest",
-						"-H",
-						"'Accept: application/json'",
-						"-H",
-						"'Content-Type:application/json'",
-						"-X",
-						"POST",
-						"-d",
-						fmt.Sprintf("@test/e2e/testdata/lagoon-build-%s.json", name),
-						"http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish",
-					)
-					_, err = utils.Run(cmd)
+					err = utils.PublishMessage(fmt.Sprintf("@test/e2e/testdata/lagoon-build-%s.json", name))
 					ExpectWithOffset(1, err).NotTo(HaveOccurred())
 				} else {
 					By("creating a LagoonBuild resource")
@@ -334,22 +331,7 @@ var _ = Describe("controller", Ordered, func() {
 				time.Sleep(5 * time.Second)
 
 				By(fmt.Sprintf("creating a %s restore task via rabbitmq", name))
-				cmd = exec.Command(
-					"curl",
-					"-s",
-					"-u",
-					"guest:guest",
-					"-H",
-					"'Accept: application/json'",
-					"-H",
-					"'Content-Type:application/json'",
-					"-X",
-					"POST",
-					"-d",
-					fmt.Sprintf("@test/e2e/testdata/%s-restore.json", name),
-					"http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish",
-				)
-				_, err = utils.Run(cmd)
+				err = utils.PublishMessage(fmt.Sprintf("@test/e2e/testdata/%s-restore.json", name))
 				ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 				time.Sleep(10 * time.Second)
@@ -400,23 +382,40 @@ var _ = Describe("controller", Ordered, func() {
 			}
 			EventuallyWithOffset(1, verifyRobotCredentialsRotate, duration, interval).Should(Succeed())
 
+			By("check harbor project policies before creating policy")
+			_, err = utils.QueryHarborProjectPolicies(ip, "nginx-example")
+			if err != nil {
+				if !strings.Contains(err.Error(), "project metadata value is empty: retention_id") {
+					ExpectWithOffset(1, err).NotTo(HaveOccurred())
+				}
+			}
+			By("creating harbor policy update via rabbitmq")
+			err = utils.PublishMessage("@test/e2e/testdata/create-retention-policy.json")
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			time.Sleep(10 * time.Second)
+			By("check harbor project policies after creating policy")
+			after, err := utils.QueryHarborProjectPolicies(ip, "nginx-example")
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			err = comparePolicy(createPolicyWant, after)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("delete harbor policy via rabbitmq")
+			err = utils.PublishMessage("@test/e2e/testdata/remove-retention-policy.json")
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			time.Sleep(10 * time.Second)
+			By("check harbor project policies after deleting policy")
+			after, err = utils.QueryHarborProjectPolicies(ip, "nginx-example")
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			err = comparePolicy(deletePolicyWant, after)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("check harbor project repositories before deleting environment")
+			before, err := utils.QueryHarborRepositories(ip, "nginx-example")
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			err = compareRepositories(projectRepositoriesWant, before)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
 			By("delete environment via rabbitmq")
-			cmd = exec.Command(
-				"curl",
-				"-s",
-				"-u",
-				"guest:guest",
-				"-H",
-				"'Accept: application/json'",
-				"-H",
-				"'Content-Type:application/json'",
-				"-X",
-				"POST",
-				"-d",
-				"@test/e2e/testdata/remove-environment.json",
-				"http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish",
-			)
-			_, err = utils.Run(cmd)
+			err = utils.PublishMessage("@test/e2e/testdata/remove-environment.json")
 			ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 			By("validating that the namespace deletes")
@@ -435,6 +434,12 @@ var _ = Describe("controller", Ordered, func() {
 			}
 			EventuallyWithOffset(1, verifyNamespaceRemoved, duration, interval).Should(Succeed())
 
+			By("check harbor project repositories after deleting environment")
+			after, err = utils.QueryHarborRepositories(ip, "nginx-example")
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			err = compareRepositories("null", after)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
 			By("validating that unauthenticated metrics requests fail")
 			runCmd := `curl -s -k https://remote-controller-controller-manager-metrics-service.remote-controller-system.svc.cluster.local:8443/metrics | grep -v "#" | grep "lagoon_"`
 			_, err = utils.RunCommonsCommand(namespace, runCmd)
@@ -451,3 +456,45 @@ var _ = Describe("controller", Ordered, func() {
 		})
 	})
 })
+
+func comparePolicy(want, got string) error {
+	// this removes the next scheduled time from the payload as it can vary
+	var m map[string]interface{}
+	if err := json.Unmarshal([]byte(got), &m); err != nil {
+		return err
+	}
+	delete(m["scope"].(map[string]interface{}), "ref")
+	delete(m["trigger"].(map[string]interface{})["settings"].(map[string]interface{}), "next_scheduled_time")
+	delete(m, "id")
+	p, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	if want != string(p) {
+		return fmt.Errorf("resulting policies don't match:\nwant: %s\ngot: %s", want, string(p))
+	}
+	return nil
+}
+
+func compareRepositories(want, got string) error {
+	if got == "null" && want == "null" {
+		return nil
+	}
+	// this removes the next scheduled time from the payload as it can vary
+	var m []interface{}
+	if err := json.Unmarshal([]byte(got), &m); err != nil {
+		return err
+	}
+	delete(m[0].(map[string]interface{}), "id")
+	delete(m[0].(map[string]interface{}), "creation_time")
+	delete(m[0].(map[string]interface{}), "update_time")
+	delete(m[0].(map[string]interface{}), "project_id")
+	p, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	if want != string(p) {
+		return fmt.Errorf("resulting policies don't match:\nwant: %s\ngot: %s", want, string(p))
+	}
+	return nil
+}

test/e2e/testdata/create-retention-policy.json

@@ -0,0 +1,20 @@
+{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc",
+    "payload":"{
+        \"misc\":{
+            \"miscResource\":\"eyJ0eXBlIjoiaGFyYm9yUmV0ZW50aW9uUG9saWN5IiwiZXZlbnRUeXBlIjoidXBkYXRlUG9saWN5IiwiZGF0YSI6eyJwcm9qZWN0Ijp7ImlkIjoxLCJuYW1lIjoibmdpbngtZXhhbXBsZSJ9LCJwb2xpY3kiOnsicnVsZXMiOlt7Im5hbWUiOiJhbGwgYnJhbmNoZXMsIGV4Y2x1ZGluZyBwdWxscmVxdWVzdHMiLCJwYXR0ZXJuIjoiW15wclxcLV0qLyoiLCJsYXRlc3RQdWxsZWQiOjN9LHsibmFtZSI6InB1bGxyZXF1ZXN0cyIsInBhdHRlcm4iOiJwci0qIiwibGF0ZXN0UHVsbGVkIjoxfV0sInNjaGVkdWxlIjoiMyAzICogKiAzIn19fQ==\"
+        },
+        \"key\":\"deploytarget:harborpolicy:update\",
+        \"environment\":{
+            \"name\":\"main\",
+            \"openshiftProjectName\":\"nginx-example-main\"
+        },
+        \"project\":{
+            \"name\":\"nginx-example\"
+        },
+        \"advancedTask\":{}
+    }",
+"payload_encoding":"string"
+}
+
+
+

test/e2e/testdata/lagoon-build-1m5zypx.yaml

@@ -13,6 +13,7 @@ spec:
     project:
         name: nginx-example
         environment: main
+        id: 1
         organization:
             id: 123
             name: test-org

test/e2e/testdata/lagoon-build-7m5zypx.yaml

@@ -13,6 +13,7 @@ spec:
     gitReference: origin/main
     project:
         name: nginx-example
+        id: 1
         environment: main
         uiLink: https://dashboard.amazeeio.cloud/projects/project/project-environment/deployments/lagoon-build-7m5zypx
         routerPattern: 'main-nginx-example'