From e41eaf5119701bbd9c219b2c9ae39b8ba077d862 Mon Sep 17 00:00:00 2001 From: Denis Arsh Date: Mon, 1 Jul 2024 14:36:35 +0300 Subject: [PATCH 1/2] add honor lables to scrape_config --- pkg/app/dbs/tmpl/prom/prometheus/scrapeconfigs.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/app/dbs/tmpl/prom/prometheus/scrapeconfigs.tpl b/pkg/app/dbs/tmpl/prom/prometheus/scrapeconfigs.tpl index d2f0fc1b..c7a42e4f 100644 --- a/pkg/app/dbs/tmpl/prom/prometheus/scrapeconfigs.tpl +++ b/pkg/app/dbs/tmpl/prom/prometheus/scrapeconfigs.tpl @@ -14,6 +14,7 @@ data: evaluation_interval: 10s scrape_configs: - job_name: cnvrg-metrics + honor_labels: true relabel_configs: - source_labels: [__meta_kubernetes_pod_name] action: replace From 7c7106db95933be5e855af0b876495a3ab85934a Mon Sep 17 00:00:00 2001 From: Denis Arsh Date: Wed, 3 Jul 2024 15:19:15 +0300 Subject: [PATCH 2/2] add ginko tests --- Makefile | 12 +- controllers/app/controller_test.go | 2388 ----------------------- controllers/test/controller_old_test.go | 2380 ++++++++++++++++++++++ controllers/test/controller_test.go | 72 + controllers/test/gen.go | 36 + controllers/test/suite_test.go | 116 ++ controllers/test/suite_utils_test.go | 150 ++ go.mod | 5 + go.sum | 48 + scripts/fetch_ext_bins.sh | 108 + 10 files changed, 2926 insertions(+), 2389 deletions(-) delete mode 100644 controllers/app/controller_test.go create mode 100644 controllers/test/controller_old_test.go create mode 100644 controllers/test/controller_test.go create mode 100644 controllers/test/gen.go create mode 100644 controllers/test/suite_test.go create mode 100644 controllers/test/suite_utils_test.go create mode 100644 scripts/fetch_ext_bins.sh diff --git a/Makefile b/Makefile index e0becf97..a4bcde40 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +GOTEST ?= CGO_ENABLED=0 go test + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -18,7 +20,7 @@ bundle: kustomize build config/manifests | operator-sdk generate bundle -q --overwrite --version 4.3.16 # Run tests -test: generate fmt vet manifests +test: generate fmt vet manifests test-controller rm -f ./controllers/test-report.html ./controllers/junit.xml CNVRG_OPERATOR_MAX_CONCURRENT_RECONCILES=1 go test ./controllers/ -v -timeout 40m @@ -179,3 +181,11 @@ else CONTROLLER_GEN=$(shell which controller-gen) endif +.PHONY: fetch-ext-bin +fetch-ext-bin: ## fetch external kubebuilder bins required to run testenv + source ./scripts/fetch_ext_bins.sh; fetch_tools; setup_envs; + +.PHONY: test-controller +test-controller: fetch-ext-bin + $(GOTEST) ./controllers/test/... + diff --git a/controllers/app/controller_test.go b/controllers/app/controller_test.go deleted file mode 100644 index 5d9a2f72..00000000 --- a/controllers/app/controller_test.go +++ /dev/null @@ -1,2388 +0,0 @@ -package app - -// -//import ( -// "context" -// "fmt" -// mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" -// "github.com/AccessibleAI/cnvrg-operator/pkg/desired" -// "github.com/AccessibleAI/cnvrg-operator/pkg/networking" -// . "github.com/onsi/ginkgo" -// . "github.com/onsi/gomega" -// "github.com/teris-io/shortid" -// v1 "k8s.io/api/apps/v1" -// "k8s.io/api/autoscaling/v1" -// corev1 "k8s.io/api/core/v1" -// "k8s.io/apimachinery/pkg/api/resource" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -// "k8s.io/apimachinery/pkg/types" -// "reflect" -// "sort" -// "strings" -// "time" -//) -// -//// +kubebuilder:docs-gen:collapse=Imports -// -//var _ = Describe("CnvrgApp controller", func() { -// -// const ( -// timeout = time.Second * 60 -// interval = time.Millisecond * 250 -// ) -// -// Context("Test PG", func() { -// It("PG Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// testApp.Spec.Labels = labels -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("PG Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("PG HugePages - defaults", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// testApp.Spec.Dbs.Pg.HugePages.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// v := corev1.Volume{ -// Name: "hugepage", -// VolumeSource: corev1.VolumeSource{ -// EmptyDir: &corev1.EmptyDirVolumeSource{ -// Medium: "HugePages", -// SizeLimit: nil, -// }, -// }, -// } -// Expect(deployment.Spec.Template.Spec.Volumes).Should(ContainElement(v)) -// -// vm := corev1.VolumeMount{Name: "hugepage", MountPath: "/hugepages"} -// Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).Should(ContainElement(vm)) -// -// shouldLimits := map[corev1.ResourceName]resource.Quantity{ -// "hugepages-2Mi": resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), -// corev1.ResourceCPU: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Cpu), -// corev1.ResourceMemory: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), -// } -// -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["hugepages-2Mi"])) -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["cpu"])) -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["memory"])) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("PG HugePages - custom", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// testApp.Spec.Dbs.Pg.HugePages.Enabled = true -// testApp.Spec.Dbs.Pg.HugePages.Size = "1Gi" -// testApp.Spec.Dbs.Pg.HugePages.Memory = "2Gi" -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// v := corev1.Volume{ -// Name: "hugepage", -// VolumeSource: corev1.VolumeSource{ -// EmptyDir: &corev1.EmptyDirVolumeSource{ -// Medium: "HugePages", -// SizeLimit: nil, -// }, -// }, -// } -// Expect(deployment.Spec.Template.Spec.Volumes).Should(ContainElement(v)) -// -// vm := corev1.VolumeMount{Name: "hugepage", MountPath: "/hugepages"} -// Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).Should(ContainElement(vm)) -// -// shouldLimits := map[corev1.ResourceName]resource.Quantity{ -// "hugepages-2Mi": resource.MustParse("2Gi"), -// corev1.ResourceCPU: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Cpu), -// corev1.ResourceMemory: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), -// } -// -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["hugepages-2Mi"])) -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["cpu"])) -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["memory"])) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("PG HugePages - disabled", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// shouldLimits := map[corev1.ResourceName]resource.Quantity{ -// corev1.ResourceCPU: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Cpu), -// corev1.ResourceMemory: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), -// } -// -// v := corev1.Volume{ -// Name: "hugepage", -// VolumeSource: corev1.VolumeSource{ -// EmptyDir: &corev1.EmptyDirVolumeSource{ -// Medium: "HugePages", -// SizeLimit: nil, -// }, -// }, -// } -// Expect(deployment.Spec.Template.Spec.Volumes).ShouldNot(ContainElement(v)) -// vm := corev1.VolumeMount{Name: "hugepage", MountPath: "/hugepages"} -// Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).ShouldNot(ContainElement(vm)) -// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(BeEquivalentTo(shouldLimits)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("PG NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// testApp.Spec.Dbs.Pg.HugePages.Enabled = true -// testApp.Spec.Dbs.Pg.NodeSelector = map[string]string{"foo": "bar"} -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("PG Tenancy & NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// testApp.Spec.Dbs.Pg.HugePages.Enabled = true -// testApp.Spec.Dbs.Pg.NodeSelector = map[string]string{"foo": "bar"} -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("Pg creds secret generator", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// // create app -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// // get pg creds -// pgCreds := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.CredsRef, Namespace: ns}, &pgCreds) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // enforce reconcile loop - enable ES and make sure it was deployed -// appRes := mlopsv1.CnvrgApp{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrgapp", Namespace: ns}, &appRes) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// rvBeforeUpdate := appRes.ObjectMeta.ResourceVersion -// appRes.Spec.Dbs.Es.Enabled = true -// Expect(k8sClient.Update(ctx, &appRes)).Should(Succeed()) -// sts := v1.StatefulSet{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure resource version has been updated -// Expect(rvBeforeUpdate).Should(Not(Equal(appRes.ObjectMeta.ResourceVersion))) -// // get pg creds after reconcile -// pgCredsAfterReconcile := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.CredsRef, Namespace: ns}, &pgCredsAfterReconcile) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure es creds wasn't mutated between reconciliation loops -// Expect(pgCreds.Data["POSTGRESQL_PASSWORD"]).Should(Equal(pgCredsAfterReconcile.Data["POSTGRESQL_PASSWORD"])) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Redis", func() { -// It("Redis Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Redis.Enabled = true -// testApp.Spec.Labels = labels -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Redis Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Redis.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Redis NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Redis.Enabled = true -// testApp.Spec.Dbs.Redis.NodeSelector = map[string]string{"foo": "bar"} -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("Redis Tenancy & NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Redis.Enabled = true -// testApp.Spec.Dbs.Redis.NodeSelector = map[string]string{"foo": "bar"} -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("Redis creds secret generator", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Redis.Enabled = true -// // create app -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// // get pg creds -// redisCreds := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.CredsRef, Namespace: ns}, &redisCreds) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // enforce reconcile loop - enable ES and make sure it was deployed -// appRes := mlopsv1.CnvrgApp{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrgapp", Namespace: ns}, &appRes) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// rvBeforeUpdate := appRes.ObjectMeta.ResourceVersion -// appRes.Spec.Dbs.Es.Enabled = true -// Expect(k8sClient.Update(ctx, &appRes)).Should(Succeed()) -// sts := v1.StatefulSet{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure resource version has been updated -// Expect(rvBeforeUpdate).Should(Not(Equal(appRes.ObjectMeta.ResourceVersion))) -// // get pg creds after reconcile -// redisCredsAfterReconcile := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.CredsRef, Namespace: ns}, &redisCredsAfterReconcile) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure es creds wasn't mutated between reconciliation loops -// Expect(redisCreds.Data["redis.conf"]).Should(Equal(redisCredsAfterReconcile.Data["redis.conf"])) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Minio", func() { -// It("Minio Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Minio.Enabled = true -// testApp.Spec.Labels = labels -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Minio Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Minio.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Minio NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Minio.Enabled = true -// testApp.Spec.Dbs.Minio.NodeSelector = map[string]string{"foo": "bar"} -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Minio Tenancy & NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Minio.Enabled = true -// testApp.Spec.Dbs.Minio.NodeSelector = map[string]string{"foo": "bar"} -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Es", func() { -// It("Es Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Labels = labels -// -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(sts.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Es Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Annotations = annotations -// -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(sts.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Es NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Dbs.Es.NodeSelector = map[string]string{"foo": "bar"} -// -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(sts.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("Es Tenancy & NodeSelector", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Dbs.Es.NodeSelector = map[string]string{"foo": "bar"} -// testApp.Spec.Tenancy.Enabled = true -// -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(sts.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(sts.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) -// Expect(sts.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// -// }) -// It("Es default xms xmx", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Requests.Memory = "4Gi" -// testApp.Spec.Dbs.Es.Enabled = true -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: "-Xms2g -Xmx2g"} -// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Es 5Gi requests - xms xmx", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Requests.Memory = "5Gi" -// testApp.Spec.Dbs.Es.Enabled = true -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: "-Xms2g -Xmx2g"} -// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Es 6Gi requests - xms xmx", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Requests.Memory = "6Gi" -// testApp.Spec.Dbs.Es.Enabled = true -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: "-Xms3g -Xmx3g"} -// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Es 6000Mi requests - xms xmx", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Requests.Memory = "6000Mi" -// testApp.Spec.Dbs.Es.Enabled = true -// sts := v1.StatefulSet{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: ""} -// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Es creds secret generator", func() { -// ns := createNs() -// ctx := context.Background() -// infra := getDefaultTestInfraSpec(ns) -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Es.Enabled = true -// // create infra -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// // create app -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// -// // get es creds -// esCreds := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.CredsRef, Namespace: ns}, &esCreds) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // enforce reconcile loop - enable PG and make sure it was deployed -// appRes := mlopsv1.CnvrgApp{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrgapp", Namespace: ns}, &appRes) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// rvBeforeUpdate := appRes.ObjectMeta.ResourceVersion -// appRes.Spec.Dbs.Pg.Enabled = true -// Expect(k8sClient.Update(ctx, &appRes)).Should(Succeed()) -// deployment := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure resource version has been updated -// Expect(rvBeforeUpdate).Should(Not(Equal(appRes.ObjectMeta.ResourceVersion))) -// // get es creds after reconcile -// esCredsAfterReconcile := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.CredsRef, Namespace: ns}, &esCredsAfterReconcile) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure es creds wasn't mutated between reconciliation loops -// Expect(esCreds.Data["CNVRG_ES_PASS"]).Should(Equal(esCredsAfterReconcile.Data["CNVRG_ES_PASS"])) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Cnvrg Control Plane", func() { -// -// It("WebApp Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.Labels = labels -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("WebApp Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("WebApp Tenancy", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Sidekiq Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Sidekiq.Split = true -// testApp.Spec.ControlPlane.Sidekiq.Enabled = true -// testApp.Spec.Labels = labels -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Sidekiq Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Sidekiq.Split = true -// testApp.Spec.ControlPlane.Sidekiq.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Sidekiq Tenancy", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Sidekiq.Split = true -// testApp.Spec.ControlPlane.Sidekiq.Enabled = true -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Hyper Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Hyper.Enabled = true -// testApp.Spec.Labels = labels -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Hyper Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Hyper.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Hyper Tenancy", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Hyper.Enabled = true -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Scheduler Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.CnvrgScheduler.Enabled = true -// testApp.Spec.Labels = labels -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "scheduler", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Scheduler Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.CnvrgScheduler.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "scheduler", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Scheduler Tenancy", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.CnvrgScheduler.Enabled = true -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "scheduler", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Mpi Operator Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Hyper.Enabled = true -// testApp.Spec.Labels = labels -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Mpi Operator Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// annotations := map[string]string{"foo1": "bar1"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Hyper.Enabled = true -// testApp.Spec.Annotations = annotations -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// It("Mpi Operator Tenancy", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Hyper.Enabled = true -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("ImageHub for WebApp - default ImageHub", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.ControlPlane.Image = "app:1.2.3" -// -// dep := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// shouldBe := fmt.Sprintf("%s/%s", testApp.Spec.ImageHub, testApp.Spec.ControlPlane.Image) -// Expect(dep.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("ImageHub for WebApp - custom ImageHub", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.ImageHub = "foo/bar" -// testApp.Spec.ControlPlane.Image = "app:1.2.3" -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// shouldBe := fmt.Sprintf("%s/%s", testApp.Spec.ImageHub, testApp.Spec.ControlPlane.Image) -// Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("ImageHub for Sidekiq - custom ImageHub", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.Sidekiq.Enabled = true -// testApp.Spec.ControlPlane.Sidekiq.Split = true -// testApp.Spec.ImageHub = "foo/bar" -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// shouldBe := fmt.Sprintf("%s/%s", testApp.Spec.ImageHub, testApp.Spec.ControlPlane.Image) -// Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("image for WebApp - disable ImageHub", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.ImageHub = "foo/bar" -// testApp.Spec.ControlPlane.Image = "foo/app:1.2.3" -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// shouldBe := fmt.Sprintf("%s", testApp.Spec.ControlPlane.Image) -// Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Labels/Annotations CCP ConfigMap", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Labels = map[string]string{"foo": "bar", "foo1": "bar1"} -// testApp.Spec.Annotations = map[string]string{"foo1": "bar1"} -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// -// cm := corev1.ConfigMap{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-annotation-label", Namespace: ns}, &cm) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(cm.Data["labels"]).Should(ContainSubstring("foo")) -// Expect(cm.Data["labels"]).Should(ContainSubstring("bar")) -// Expect(cm.Data["annotations"]).Should(ContainSubstring("foo1")) -// Expect(cm.Data["annotations"]).Should(ContainSubstring("bar1")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Proxy enabled - webapp", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.ControlPlane.WebApp.Enabled = true -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// dep := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// env := corev1.EnvFromSource{ -// ConfigMapRef: &corev1.ConfigMapEnvSource{ -// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, -// }, -// } -// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Proxy enabled - sidekiq", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Split = true -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// dep := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// env := corev1.EnvFromSource{ -// ConfigMapRef: &corev1.ConfigMapEnvSource{ -// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, -// }, -// } -// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Proxy enabled - searchkiq", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.ControlPlane.Searchkiq.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Split = true -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// dep := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "searchkiq", Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// env := corev1.EnvFromSource{ -// ConfigMapRef: &corev1.ConfigMapEnvSource{ -// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, -// }, -// } -// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Proxy enabled - systemkiq", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.ControlPlane.Systemkiq.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Split = true -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// dep := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "systemkiq", Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// env := corev1.EnvFromSource{ -// ConfigMapRef: &corev1.ConfigMapEnvSource{ -// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, -// }, -// } -// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Proxy enabled - ccp networking configmap", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.ControlPlane.WebApp.Enabled = true -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// cm := corev1.ConfigMap{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-networking-config", Namespace: ns}, &cm) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(cm.Data["CNVRG_PROXY_CONFIG_REF"]).Should(BeEquivalentTo(app.Spec.Networking.Proxy.ConfigRef)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Proxy disabled - ccp networking configmap", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.WebApp.Enabled = true -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// cm := corev1.ConfigMap{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-networking-config", Namespace: ns}, &cm) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// _, found := cm.Data["CNVRG_PROXY_CONFIG_REF"] -// Expect(found).Should(BeFalse()) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("HPA disabled", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.WebApp.Enabled = true -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// hpa := v2beta1.HorizontalPodAutoscaler{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &hpa) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeFalse()) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("WebApp HPA Enabled", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.WebApp.Enabled = true -// app.Spec.ControlPlane.WebApp.Hpa.Enabled = true -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// hpa := v2beta1.HorizontalPodAutoscaler{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &hpa) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.WebApp.Replicas))) -// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.WebApp.Hpa.MaxReplicas))) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Sidekiq HPA Enabled", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.Sidekiq.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Split = true -// app.Spec.ControlPlane.Sidekiq.Hpa.Enabled = true -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// hpa := v2beta1.HorizontalPodAutoscaler{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &hpa) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Sidekiq.Replicas))) -// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Sidekiq.Hpa.MaxReplicas))) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Searchkiq HPA Enabled", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.Searchkiq.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Split = true -// app.Spec.ControlPlane.Searchkiq.Hpa.Enabled = true -// app.Spec.ControlPlane.Searchkiq.Hpa.MaxReplicas = 10 -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// hpa := v2beta1.HorizontalPodAutoscaler{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "searchkiq", Namespace: ns}, &hpa) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Searchkiq.Replicas))) -// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Searchkiq.Hpa.MaxReplicas))) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("Systemkiq HPA Enabled", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.Systemkiq.Enabled = true -// app.Spec.ControlPlane.Sidekiq.Split = true -// app.Spec.ControlPlane.Systemkiq.Hpa.Enabled = true -// app.Spec.ControlPlane.Systemkiq.Hpa.Utilization = 90 -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// hpa := v2beta1.HorizontalPodAutoscaler{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "systemkiq", Namespace: ns}, &hpa) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Systemkiq.Replicas))) -// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Systemkiq.Hpa.MaxReplicas))) -// Expect(*hpa.Spec.Metrics[0].Resource.TargetAverageUtilization).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Systemkiq.Hpa.Utilization))) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("SMTP OpensslVerifyMode", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.SMTP.OpensslVerifyMode = "foo" -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// secret := corev1.Secret{} -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-smtp", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(string(secret.Data["SMTP_OPENSSL_VERIFY_MODE"])).To(Equal("foo")) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// It("SMTP Sender", func() { -// ctx := context.Background() -// ns := createNs() -// app := getDefaultTestAppSpec(ns) -// app.Spec.ControlPlane.SMTP.Sender = "test@cnvrg.io" -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// secret := corev1.Secret{} -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-smtp", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(string(secret.Data["SMTP_SENDER"])).To(Equal("test@cnvrg.io")) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Object Storage Secret", func() { -// It("Object Storage Secret - Minio enabled - random keys", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Minio.Enabled = true -// testApp.Spec.ControlPlane.ObjectStorage.Type = "minio" -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// secret := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).ShouldNot(BeEmpty()) -// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).ShouldNot(BeEmpty()) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Object Storage Secret - Minio enabled - explicitly configured keys", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Minio.Enabled = true -// testApp.Spec.ControlPlane.ObjectStorage.Type = "minio" -// testApp.Spec.ControlPlane.ObjectStorage.AccessKey = "access-key" -// testApp.Spec.ControlPlane.ObjectStorage.SecretKey = "secret-key" -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// secret := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(Equal([]byte("access-key"))) -// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(Equal([]byte("secret-key"))) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Object Storage Secret - Minio external - explicitly configured keys", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.ObjectStorage.Type = "minio" -// testApp.Spec.ControlPlane.ObjectStorage.AccessKey = "access-key" -// testApp.Spec.ControlPlane.ObjectStorage.SecretKey = "secret-key" -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// secret := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(Equal([]byte("access-key"))) -// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(Equal([]byte("secret-key"))) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Object Storage Secret - AWS S3 - explicitly configured keys", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.ObjectStorage.Type = "aws" -// testApp.Spec.ControlPlane.ObjectStorage.AccessKey = "access-key" -// testApp.Spec.ControlPlane.ObjectStorage.SecretKey = "secret-key" -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// secret := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(Equal([]byte("access-key"))) -// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(Equal([]byte("secret-key"))) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Object Storage Secret - AWS S3 - IAM S3 access", func() { -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.ObjectStorage.Type = "aws" -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// secret := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(BeEmpty()) -// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(BeEmpty()) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// }) -// -// Context("CnvrgRouter", func() { -// It("Cnvrg Router Labels", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true -// testApp.Spec.Labels = labels -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Cnvrg Router Annotations", func() { -// ns := createNs() -// ctx := context.Background() -// labels := map[string]string{"foo": "bar"} -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true -// testApp.Spec.Annotations = labels -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo", "bar")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// It("Cnvrg Router Tenancy", func() { -// ns := createNs() -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true -// testApp.Spec.Tenancy.Enabled = true -// -// deployment := v1.Deployment{} -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// t := corev1.Toleration{ -// Key: testApp.Spec.Tenancy.Key, -// Operator: "Equal", -// Value: testApp.Spec.Tenancy.Value, -// Effect: "NoSchedule", -// } -// -// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) -// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// }) -// -// }) -// -// Context("Control Plane - Default Istio ingress", func() { -// -// It("Default Istio ingress", func() { -// ns := createNs() -// gwName := fmt.Sprintf(mlopsv1.IstioGwName, ns) -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Networking.Ingress.IstioGwEnabled = true -// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.Logging.Kibana.Enabled = true -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Monitoring.Prometheus.Enabled = true -// testApp.Spec.Monitoring.Grafana.Enabled = true -// -// infra := getDefaultTestInfraSpec(ns) -// infra.Spec.Monitoring.Prometheus.Enabled = true -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// gw := &unstructured.Unstructured{} -// gw.SetGroupVersionKind(desired.Kinds[desired.IstioGwGVK]) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: gwName, Namespace: ns}, gw) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// vs := &unstructured.Unstructured{} -// vs.SetGroupVersionKind(desired.Kinds[desired.IstioVsGVK]) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Logging.Kibana.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Prometheus.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Grafana.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// -// }) -// -// It("Custom Istio ingress Name", func() { -// ns := createNs() -// gwName := "foo-bar" -// ctx := context.Background() -// -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Networking.Ingress.IstioGwEnabled = true -// testApp.Spec.Networking.Ingress.IstioGwName = gwName -// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.Logging.Kibana.Enabled = true -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Monitoring.Prometheus.Enabled = true -// testApp.Spec.Monitoring.Grafana.Enabled = true -// -// infra := getDefaultTestInfraSpec(ns) -// infra.Spec.Monitoring.Prometheus.Enabled = true -// infra.Spec.Networking.Ingress.Type = mlopsv1.NginxIngress -// -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// -// gw := &unstructured.Unstructured{} -// gw.SetGroupVersionKind(desired.Kinds[desired.IstioGwGVK]) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: gwName, Namespace: ns}, gw) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// vs := &unstructured.Unstructured{} -// vs.SetGroupVersionKind(desired.Kinds[desired.IstioVsGVK]) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Logging.Kibana.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Prometheus.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Grafana.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// -// }) -// -// It("Istio Disabled - Custom Istio ingress Name", func() { -// ns := createNs() -// -// gwName := "foo-bar" -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Networking.Ingress.IstioGwName = gwName -// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true -// testApp.Spec.ControlPlane.WebApp.Enabled = true -// testApp.Spec.Logging.Kibana.Enabled = true -// testApp.Spec.Dbs.Es.Enabled = true -// testApp.Spec.Monitoring.Prometheus.Enabled = true -// testApp.Spec.Monitoring.Grafana.Enabled = true -// -// infra := getDefaultTestInfraSpec(ns) -// infra.Spec.Monitoring.Prometheus.Enabled = true -// infra.Spec.Networking.Ingress.Type = mlopsv1.NginxIngress -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// -// vs := &unstructured.Unstructured{} -// vs.SetGroupVersionKind(desired.Kinds[desired.IstioVsGVK]) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Logging.Kibana.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Prometheus.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Grafana.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, vs) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) -// -// gw := &unstructured.Unstructured{} -// gw.SetGroupVersionKind(desired.Kinds[desired.IstioGwGVK]) -// time.Sleep(time.Second * 3) -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: gwName, Namespace: ns}, gw) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeFalse()) -// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Proxy ConfigMap creation", func() { -// It("Proxy configmap test creation - default no_proxy", func() { -// -// ctx := context.Background() -// ns := createNs() -// expectedNoProxy := networking.DefaultNoProxy("cluster.local") -// app := getDefaultTestAppSpec(ns) -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.Networking.Proxy.HttpProxy = []string{ -// "http://proxy1.org.local", -// "http://proxy2.org.local", -// } -// app.Spec.Networking.Proxy.HttpsProxy = []string{ -// "https://proxy1.org.local", -// "https://proxy2.org.local", -// } -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// -// Eventually(func() bool { -// appRes := mlopsv1.CnvrgApp{} -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, &appRes) -// if err != nil { -// return false -// } -// sort.Strings(appRes.Spec.Networking.Proxy.NoProxy) -// sort.Strings(expectedNoProxy) -// return reflect.DeepEqual(appRes.Spec.Networking.Proxy.NoProxy, expectedNoProxy) -// }, timeout, interval).Should(BeTrue()) -// -// cm := corev1.ConfigMap{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Networking.Proxy.ConfigRef, Namespace: ns}, &cm) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(cm.Data).Should(HaveKeyWithValue("HTTP_PROXY", "http://proxy1.org.local,http://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("http_proxy", "http://proxy1.org.local,http://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("https_proxy", "https://proxy1.org.local,https://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("HTTPS_PROXY", "https://proxy1.org.local,https://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("NO_PROXY", strings.Join(expectedNoProxy, ","))) -// Expect(cm.Data).Should(HaveKeyWithValue("no_proxy", strings.Join(expectedNoProxy, ","))) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// It("Proxy configmap test creation - custom no_proxy", func() { -// -// ctx := context.Background() -// ns := createNs() -// noProxy := []string{".foo.bar"} -// expectedNoProxy := append(noProxy, networking.DefaultNoProxy("cluster.local")...) -// app := getDefaultTestAppSpec(ns) -// cm := corev1.ConfigMap{} -// -// app.Spec.Networking.Proxy.Enabled = true -// app.Spec.Networking.Proxy.HttpProxy = []string{ -// "http://proxy1.org.local", -// "http://proxy2.org.local", -// } -// app.Spec.Networking.Proxy.HttpsProxy = []string{ -// "https://proxy1.org.local", -// "https://proxy2.org.local", -// } -// -// app.Spec.Networking.Proxy.NoProxy = noProxy -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// -// Eventually(func() bool { -// appRes := mlopsv1.CnvrgApp{} -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, &appRes) -// if err != nil { -// return false -// } -// sort.Strings(appRes.Spec.Networking.Proxy.NoProxy) -// sort.Strings(expectedNoProxy) -// return reflect.DeepEqual(appRes.Spec.Networking.Proxy.NoProxy, expectedNoProxy) -// }, timeout, interval).Should(BeTrue()) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Networking.Proxy.ConfigRef, Namespace: ns}, &cm) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(cm.Data).Should(HaveKeyWithValue("HTTP_PROXY", "http://proxy1.org.local,http://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("http_proxy", "http://proxy1.org.local,http://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("https_proxy", "https://proxy1.org.local,https://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("HTTPS_PROXY", "https://proxy1.org.local,https://proxy2.org.local")) -// Expect(cm.Data).Should(HaveKeyWithValue("NO_PROXY", strings.Join(expectedNoProxy, ","))) -// Expect(cm.Data).Should(HaveKeyWithValue("no_proxy", strings.Join(expectedNoProxy, ","))) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// It("Proxy configmap test creation - proxy disabled", func() { -// -// ctx := context.Background() -// ns := createNs() -// app := getEmptyTestAppSpec(ns) -// -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// -// Eventually(func() bool { -// appRes := mlopsv1.CnvrgApp{} -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, &appRes) -// if err != nil { -// return false -// } -// return appRes.Spec.Networking.Proxy.Enabled == false -// }, timeout, interval).Should(BeTrue()) -// -// cm := corev1.ConfigMap{} -// -// time.Sleep(3) -// -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Networking.Proxy.ConfigRef, Namespace: ns}, &cm) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeFalse()) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// }) -// }) -// -// Context("Test Priority class", func() { -// It("CnvrgApp and CnvrgJob priority classes names", func() { -// ns := createNs() -// ctx := context.Background() -// infra := getDefaultTestInfraSpec(ns) -// app := getDefaultTestAppSpec(ns) -// app.Spec.CnvrgJobPriorityClass.Name = "" -// app.Spec.PriorityClass.AppClassRef = "" -// app.Spec.Dbs.Pg.Enabled = true -// // create infra -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// // create app -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// -// deployment := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// expectedApp := mlopsv1.CnvrgApp{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: ns}, &expectedApp) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// Expect(expectedApp.Spec.PriorityClass.AppClassRef).To(Equal(infra.Spec.PriorityClass.AppClassRef)) -// Expect(expectedApp.Spec.CnvrgJobPriorityClass.Name).To(Equal(infra.Spec.CnvrgJobPriorityClass.Name)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// }) -// }) -// Context("Test AppMonitoring", func() { -// It("Prom creds secret generator", func() { -// ns := createNs() -// ctx := context.Background() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Monitoring.Prometheus.Enabled = true -// infra := getDefaultTestInfraSpec(ns) -// infra.Spec.Monitoring.Prometheus.Enabled = true -// // create infra -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// // create app -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// // get prom creds -// promCreds := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.CredsRef, Namespace: ns}, &promCreds) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // enforce reconcile loop - enable Prometheus Operator and make sure it was deployed -// infraRes := mlopsv1.CnvrgInfra{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: ns}, &infraRes) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// rvBeforeUpdate := infraRes.ObjectMeta.ResourceVersion -// infraRes.Spec.Monitoring.PrometheusOperator.Enabled = true -// Expect(k8sClient.Update(ctx, &infraRes)).Should(Succeed()) -// dep := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrg-prometheus-operator", Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure resource version has been updated -// Expect(rvBeforeUpdate).Should(Not(Equal(infraRes.ObjectMeta.ResourceVersion))) -// // get prometheus creds after reconcile -// promCredsAfterReconcile := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.CredsRef, Namespace: ns}, &promCredsAfterReconcile) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure redis creds wasn't mutated between reconciliation loops -// Expect(promCreds.Data["CNVRG_PROMETHEUS_PASS"]).Should(Equal(promCredsAfterReconcile.Data["CNVRG_PROMETHEUS_PASS"])) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// }) -// -// It("Prom upstream creds secret generator", func() { -// ns := createNs() -// ctx := context.Background() -// app := getDefaultTestAppSpec(ns) -// app.Spec.Monitoring.Prometheus.Enabled = true -// infra := getDefaultTestInfraSpec(ns) -// infra.Spec.Monitoring.Prometheus.Enabled = true -// // create infra -// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) -// // create app -// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) -// // get prom creds -// promUpstreamCreds := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.UpstreamRef, Namespace: ns}, &promUpstreamCreds) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // enforce reconcile loop - enable Prometheus Operator and make sure it was deployed -// infraRes := mlopsv1.CnvrgInfra{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: ns}, &infraRes) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// rvBeforeUpdate := infraRes.ObjectMeta.ResourceVersion -// infraRes.Spec.Monitoring.PrometheusOperator.Enabled = true -// Expect(k8sClient.Update(ctx, &infraRes)).Should(Succeed()) -// dep := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrg-prometheus-operator", Namespace: ns}, &dep) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure resource version has been updated -// Expect(rvBeforeUpdate).Should(Not(Equal(infraRes.ObjectMeta.ResourceVersion))) -// // get prometheus creds after reconcile -// promUpstreamCredsAfterReconcile := corev1.Secret{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.UpstreamRef, Namespace: ns}, &promUpstreamCredsAfterReconcile) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// // Make sure redis creds wasn't mutated between reconciliation loops -// Expect(promUpstreamCreds.Data).Should(Equal(promUpstreamCredsAfterReconcile.Data)) -// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) -// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) -// }) -// -// }) -// -// Context("Test Events", func() { -// -// It("Simple event test", func() { -// -// ns := createNs() -// ctx := context.Background() -// testApp := getDefaultTestAppSpec(ns) -// testApp.Spec.Dbs.Pg.Enabled = true -// -// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) -// -// deployment := v1.Deployment{} -// Eventually(func() bool { -// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) -// if err != nil { -// return false -// } -// return true -// }, timeout, interval).Should(BeTrue()) -// -// events := corev1.EventList{} -// if err := k8sClient.List(ctx, &events); err != nil { -// fmt.Println(err) -// } -// es := corev1.EventSource{Component: "cnvrgapp"} -// for _, e := range events.Items { -// if e.Source == es { -// Succeed() -// return -// } -// } -// Fail("expected cnvrgapp event not found") -// }) -// -// }) -// -//}) -// -//func createNs() string { -// ns, _ := shortid.Generate() -// ns = strings.ReplaceAll(strings.ToLower(ns), "-", "z") -// ns = strings.ReplaceAll(ns, "_", "z") -// testNs := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} -// err := k8sClient.Create(context.Background(), testNs) -// if err != nil { -// panic(err) -// } -// return ns -//} -// -//func getEmptyTestAppSpec(ns string) *mlopsv1.CnvrgApp { -// -// return &mlopsv1.CnvrgApp{ -// TypeMeta: metav1.TypeMeta{ -// Kind: "CnvrgApp", -// APIVersion: "mlops.cnvrg.io/v1", -// }, -// ObjectMeta: metav1.ObjectMeta{ -// Name: "cnvrgapp", -// Namespace: ns, -// }, -// } -//} -// -//func getDefaultTestAppSpec(ns string) *mlopsv1.CnvrgApp { -// testSpec := mlopsv1.DefaultCnvrgAppSpec() -// testSpec.Cri = mlopsv1.CriTypeDocker -// testSpec.CnvrgAppPriorityClass.Name = "foo-bar" -// testSpec.CnvrgJobPriorityClass.Name = "foo-bar" -// testSpec.ClusterDomain = "test.local" -// return &mlopsv1.CnvrgApp{ -// TypeMeta: metav1.TypeMeta{ -// Kind: "CnvrgApp", -// APIVersion: "mlops.cnvrg.io/v1", -// }, -// ObjectMeta: metav1.ObjectMeta{ -// Name: "cnvrgapp", -// Namespace: ns, -// }, -// Spec: testSpec, -// } -//} diff --git a/controllers/test/controller_old_test.go b/controllers/test/controller_old_test.go new file mode 100644 index 00000000..14ba6767 --- /dev/null +++ b/controllers/test/controller_old_test.go @@ -0,0 +1,2380 @@ +package test + +// +//import ( +// "context" +// mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" +// "github.com/teris-io/shortid" +// v1 "k8s.io/api/apps/v1" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// "strings" +// +// . "github.com/onsi/ginkgo" +// . "github.com/onsi/gomega" +// +// corev1 "k8s.io/api/core/v1" +// "k8s.io/apimachinery/pkg/api/resource" +// +// "k8s.io/apimachinery/pkg/types" +// +// "time" +//) +// +//// // +kubebuilder:docs-gen:collapse=Imports +//var _ = Describe("CnvrgApp controller", func() { +// const ( +// timeout = time.Second * 60 +// interval = time.Millisecond * 250 +// ) +// +// Context("Test PG", func() { +// It("PG Labels", func() { +// ns := createNs() +// ctx := context.Background() +// labels := map[string]string{"foo": "bar"} +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// testApp.Spec.Labels = labels +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// }) +// It("PG Annotations", func() { +// ns := createNs() +// ctx := context.Background() +// annotations := map[string]string{"foo1": "bar1"} +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// testApp.Spec.Annotations = annotations +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// }) +// It("PG HugePages - defaults", func() { +// ns := createNs() +// ctx := context.Background() +// +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// testApp.Spec.Dbs.Pg.HugePages.Enabled = true +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// +// v := corev1.Volume{ +// Name: "hugepage", +// VolumeSource: corev1.VolumeSource{ +// EmptyDir: &corev1.EmptyDirVolumeSource{ +// Medium: "HugePages", +// SizeLimit: nil, +// }, +// }, +// } +// Expect(deployment.Spec.Template.Spec.Volumes).Should(ContainElement(v)) +// +// vm := corev1.VolumeMount{Name: "hugepage", MountPath: "/hugepages"} +// Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).Should(ContainElement(vm)) +// +// shouldLimits := map[corev1.ResourceName]resource.Quantity{ +// "hugepages-2Mi": resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), +// corev1.ResourceCPU: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Cpu), +// corev1.ResourceMemory: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), +// } +// +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["hugepages-2Mi"])) +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["cpu"])) +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["memory"])) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// +// }) +// It("PG HugePages - custom", func() { +// ns := createNs() +// ctx := context.Background() +// +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// testApp.Spec.Dbs.Pg.HugePages.Enabled = true +// testApp.Spec.Dbs.Pg.HugePages.Size = "1Gi" +// testApp.Spec.Dbs.Pg.HugePages.Memory = "2Gi" +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// +// v := corev1.Volume{ +// Name: "hugepage", +// VolumeSource: corev1.VolumeSource{ +// EmptyDir: &corev1.EmptyDirVolumeSource{ +// Medium: "HugePages", +// SizeLimit: nil, +// }, +// }, +// } +// Expect(deployment.Spec.Template.Spec.Volumes).Should(ContainElement(v)) +// +// vm := corev1.VolumeMount{Name: "hugepage", MountPath: "/hugepages"} +// Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).Should(ContainElement(vm)) +// +// shouldLimits := map[corev1.ResourceName]resource.Quantity{ +// "hugepages-2Mi": resource.MustParse("2Gi"), +// corev1.ResourceCPU: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Cpu), +// corev1.ResourceMemory: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), +// } +// +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["hugepages-2Mi"])) +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["cpu"])) +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(ContainElement(shouldLimits["memory"])) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// +// }) +// It("PG HugePages - disabled", func() { +// ns := createNs() +// ctx := context.Background() +// +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// +// shouldLimits := map[corev1.ResourceName]resource.Quantity{ +// corev1.ResourceCPU: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Cpu), +// corev1.ResourceMemory: resource.MustParse(testApp.Spec.Dbs.Pg.Limits.Memory), +// } +// +// v := corev1.Volume{ +// Name: "hugepage", +// VolumeSource: corev1.VolumeSource{ +// EmptyDir: &corev1.EmptyDirVolumeSource{ +// Medium: "HugePages", +// SizeLimit: nil, +// }, +// }, +// } +// Expect(deployment.Spec.Template.Spec.Volumes).ShouldNot(ContainElement(v)) +// vm := corev1.VolumeMount{Name: "hugepage", MountPath: "/hugepages"} +// Expect(deployment.Spec.Template.Spec.Containers[0].VolumeMounts).ShouldNot(ContainElement(vm)) +// Expect(deployment.Spec.Template.Spec.Containers[0].Resources.Limits).Should(BeEquivalentTo(shouldLimits)) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// +// }) +// It("PG NodeSelector", func() { +// ns := createNs() +// ctx := context.Background() +// +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// testApp.Spec.Dbs.Pg.HugePages.Enabled = true +// testApp.Spec.Dbs.Pg.NodeSelector = map[string]string{"foo": "bar"} +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// +// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// +// }) +// It("PG Tenancy & NodeSelector", func() { +// ns := createNs() +// ctx := context.Background() +// +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// testApp.Spec.Dbs.Pg.HugePages.Enabled = true +// testApp.Spec.Dbs.Pg.NodeSelector = map[string]string{"foo": "bar"} +// testApp.Spec.Tenancy.Enabled = true +// +// deployment := v1.Deployment{} +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// +// t := corev1.Toleration{ +// Key: testApp.Spec.Tenancy.Key, +// Operator: "Equal", +// Value: testApp.Spec.Tenancy.Value, +// Effect: "NoSchedule", +// } +// +// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// +// }) +// It("Pg creds secret generator", func() { +// ns := createNs() +// ctx := context.Background() +// testApp := getDefaultTestAppSpec(ns) +// testApp.Spec.Dbs.Pg.Enabled = true +// // create app +// Expect(tc.client.Create(ctx, testApp)).Should(Succeed()) +// // get pg creds +// pgCreds := corev1.Secret{} +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.CredsRef, Namespace: ns}, &pgCreds) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// // enforce reconcile loop - enable ES and make sure it was deployed +// appRes := mlopsv1.CnvrgApp{} +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: "cnvrgapp", Namespace: ns}, &appRes) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// rvBeforeUpdate := appRes.ObjectMeta.ResourceVersion +// appRes.Spec.Dbs.Es.Enabled = true +// Expect(tc.client.Update(ctx, &appRes)).Should(Succeed()) +// sts := v1.StatefulSet{} +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// // Make sure resource version has been updated +// Expect(rvBeforeUpdate).Should(Not(Equal(appRes.ObjectMeta.ResourceVersion))) +// // get pg creds after reconcile +// pgCredsAfterReconcile := corev1.Secret{} +// Eventually(func() bool { +// err := tc.client.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.CredsRef, Namespace: ns}, &pgCredsAfterReconcile) +// if err != nil { +// return false +// } +// return true +// }, timeout, interval).Should(BeTrue()) +// // Make sure es creds wasn't mutated between reconciliation loops +// Expect(pgCreds.Data["POSTGRESQL_PASSWORD"]).Should(Equal(pgCredsAfterReconcile.Data["POSTGRESQL_PASSWORD"])) +// Expect(tc.client.Delete(ctx, testApp)).Should(Succeed()) +// }) +// +// }) +//}) +// +//// Context("Test Redis", func() { +//// It("Redis Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Redis.Enabled = true +//// testApp.Spec.Labels = labels +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Redis Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Redis.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Redis NodeSelector", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Redis.Enabled = true +//// testApp.Spec.Dbs.Redis.NodeSelector = map[string]string{"foo": "bar"} +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// +//// }) +//// It("Redis Tenancy & NodeSelector", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Redis.Enabled = true +//// testApp.Spec.Dbs.Redis.NodeSelector = map[string]string{"foo": "bar"} +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// +//// }) +//// It("Redis creds secret generator", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Redis.Enabled = true +//// // create app +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// // get pg creds +//// redisCreds := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.CredsRef, Namespace: ns}, &redisCreds) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // enforce reconcile loop - enable ES and make sure it was deployed +//// appRes := mlopsv1.CnvrgApp{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrgapp", Namespace: ns}, &appRes) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// rvBeforeUpdate := appRes.ObjectMeta.ResourceVersion +//// appRes.Spec.Dbs.Es.Enabled = true +//// Expect(k8sClient.Update(ctx, &appRes)).Should(Succeed()) +//// sts := v1.StatefulSet{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure resource version has been updated +//// Expect(rvBeforeUpdate).Should(Not(Equal(appRes.ObjectMeta.ResourceVersion))) +//// // get pg creds after reconcile +//// redisCredsAfterReconcile := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Redis.CredsRef, Namespace: ns}, &redisCredsAfterReconcile) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure es creds wasn't mutated between reconciliation loops +//// Expect(redisCreds.Data["redis.conf"]).Should(Equal(redisCredsAfterReconcile.Data["redis.conf"])) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Test Minio", func() { +//// It("Minio Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Minio.Enabled = true +//// testApp.Spec.Labels = labels +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Minio Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Minio.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Minio NodeSelector", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Minio.Enabled = true +//// testApp.Spec.Dbs.Minio.NodeSelector = map[string]string{"foo": "bar"} +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Minio Tenancy & NodeSelector", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Minio.Enabled = true +//// testApp.Spec.Dbs.Minio.NodeSelector = map[string]string{"foo": "bar"} +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Minio.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Test Es", func() { +//// It("Es Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Labels = labels +//// +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(sts.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Es Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(sts.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Es NodeSelector", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Dbs.Es.NodeSelector = map[string]string{"foo": "bar"} +//// +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(sts.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// +//// }) +//// It("Es Tenancy & NodeSelector", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Dbs.Es.NodeSelector = map[string]string{"foo": "bar"} +//// testApp.Spec.Tenancy.Enabled = true +//// +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(sts.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(sts.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(sts.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// +//// }) +//// It("Es default xms xmx", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Requests.Memory = "4Gi" +//// testApp.Spec.Dbs.Es.Enabled = true +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: "-Xms2g -Xmx2g"} +//// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Es 5Gi requests - xms xmx", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Requests.Memory = "5Gi" +//// testApp.Spec.Dbs.Es.Enabled = true +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: "-Xms2g -Xmx2g"} +//// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Es 6Gi requests - xms xmx", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Requests.Memory = "6Gi" +//// testApp.Spec.Dbs.Es.Enabled = true +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: "-Xms3g -Xmx3g"} +//// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Es 6000Mi requests - xms xmx", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Requests.Memory = "6000Mi" +//// testApp.Spec.Dbs.Es.Enabled = true +//// sts := v1.StatefulSet{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, &sts) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// javaOpts := corev1.EnvVar{Name: "ES_JAVA_OPTS", Value: ""} +//// Expect(sts.Spec.Template.Spec.Containers[0].Env).Should(ContainElement(javaOpts)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Es creds secret generator", func() { +//// ns := createNs() +//// ctx := context.Background() +//// infra := getDefaultTestInfraSpec(ns) +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Es.Enabled = true +//// // create infra +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// // create app +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// +//// // get es creds +//// esCreds := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.CredsRef, Namespace: ns}, &esCreds) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // enforce reconcile loop - enable PG and make sure it was deployed +//// appRes := mlopsv1.CnvrgApp{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrgapp", Namespace: ns}, &appRes) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// rvBeforeUpdate := appRes.ObjectMeta.ResourceVersion +//// appRes.Spec.Dbs.Pg.Enabled = true +//// Expect(k8sClient.Update(ctx, &appRes)).Should(Succeed()) +//// deployment := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure resource version has been updated +//// Expect(rvBeforeUpdate).Should(Not(Equal(appRes.ObjectMeta.ResourceVersion))) +//// // get es creds after reconcile +//// esCredsAfterReconcile := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.CredsRef, Namespace: ns}, &esCredsAfterReconcile) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure es creds wasn't mutated between reconciliation loops +//// Expect(esCreds.Data["CNVRG_ES_PASS"]).Should(Equal(esCredsAfterReconcile.Data["CNVRG_ES_PASS"])) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Test Cnvrg Control Plane", func() { +//// +//// It("WebApp Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.Labels = labels +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("WebApp Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("WebApp Tenancy", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Sidekiq Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Sidekiq.Split = true +//// testApp.Spec.ControlPlane.Sidekiq.Enabled = true +//// testApp.Spec.Labels = labels +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Sidekiq Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Sidekiq.Split = true +//// testApp.Spec.ControlPlane.Sidekiq.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Sidekiq Tenancy", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Sidekiq.Split = true +//// testApp.Spec.ControlPlane.Sidekiq.Enabled = true +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Hyper Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Hyper.Enabled = true +//// testApp.Spec.Labels = labels +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Hyper Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Hyper.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Hyper Tenancy", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Hyper.Enabled = true +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Scheduler Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.CnvrgScheduler.Enabled = true +//// testApp.Spec.Labels = labels +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "scheduler", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Scheduler Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.CnvrgScheduler.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "scheduler", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Scheduler Tenancy", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.CnvrgScheduler.Enabled = true +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "scheduler", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Mpi Operator Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Hyper.Enabled = true +//// testApp.Spec.Labels = labels +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Mpi Operator Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// annotations := map[string]string{"foo1": "bar1"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Hyper.Enabled = true +//// testApp.Spec.Annotations = annotations +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo1", "bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// It("Mpi Operator Tenancy", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Hyper.Enabled = true +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.Hyper.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("ImageHub for WebApp - default ImageHub", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.ControlPlane.Image = "app:1.2.3" +//// +//// dep := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// shouldBe := fmt.Sprintf("%s/%s", testApp.Spec.ImageHub, testApp.Spec.ControlPlane.Image) +//// Expect(dep.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("ImageHub for WebApp - custom ImageHub", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.ImageHub = "foo/bar" +//// testApp.Spec.ControlPlane.Image = "app:1.2.3" +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// shouldBe := fmt.Sprintf("%s/%s", testApp.Spec.ImageHub, testApp.Spec.ControlPlane.Image) +//// Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("ImageHub for Sidekiq - custom ImageHub", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.Sidekiq.Enabled = true +//// testApp.Spec.ControlPlane.Sidekiq.Split = true +//// testApp.Spec.ImageHub = "foo/bar" +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// shouldBe := fmt.Sprintf("%s/%s", testApp.Spec.ImageHub, testApp.Spec.ControlPlane.Image) +//// Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("image for WebApp - disable ImageHub", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.ImageHub = "foo/bar" +//// testApp.Spec.ControlPlane.Image = "foo/app:1.2.3" +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// shouldBe := fmt.Sprintf("%s", testApp.Spec.ControlPlane.Image) +//// Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(shouldBe)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Labels/Annotations CCP ConfigMap", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Labels = map[string]string{"foo": "bar", "foo1": "bar1"} +//// testApp.Spec.Annotations = map[string]string{"foo1": "bar1"} +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// +//// cm := corev1.ConfigMap{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-annotation-label", Namespace: ns}, &cm) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(cm.Data["labels"]).Should(ContainSubstring("foo")) +//// Expect(cm.Data["labels"]).Should(ContainSubstring("bar")) +//// Expect(cm.Data["annotations"]).Should(ContainSubstring("foo1")) +//// Expect(cm.Data["annotations"]).Should(ContainSubstring("bar1")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Proxy enabled - webapp", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.ControlPlane.WebApp.Enabled = true +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// dep := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// env := corev1.EnvFromSource{ +//// ConfigMapRef: &corev1.ConfigMapEnvSource{ +//// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, +//// }, +//// } +//// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Proxy enabled - sidekiq", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Split = true +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// dep := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// env := corev1.EnvFromSource{ +//// ConfigMapRef: &corev1.ConfigMapEnvSource{ +//// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, +//// }, +//// } +//// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Proxy enabled - searchkiq", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.ControlPlane.Searchkiq.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Split = true +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// dep := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "searchkiq", Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// env := corev1.EnvFromSource{ +//// ConfigMapRef: &corev1.ConfigMapEnvSource{ +//// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, +//// }, +//// } +//// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Proxy enabled - systemkiq", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.ControlPlane.Systemkiq.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Split = true +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// dep := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "systemkiq", Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// env := corev1.EnvFromSource{ +//// ConfigMapRef: &corev1.ConfigMapEnvSource{ +//// LocalObjectReference: corev1.LocalObjectReference{Name: app.Spec.Networking.Proxy.ConfigRef}, +//// }, +//// } +//// Expect(dep.Spec.Template.Spec.Containers[0].EnvFrom).Should(ContainElement(env)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Proxy enabled - ccp networking configmap", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.ControlPlane.WebApp.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// cm := corev1.ConfigMap{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-networking-config", Namespace: ns}, &cm) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(cm.Data["CNVRG_PROXY_CONFIG_REF"]).Should(BeEquivalentTo(app.Spec.Networking.Proxy.ConfigRef)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Proxy disabled - ccp networking configmap", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.WebApp.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// cm := corev1.ConfigMap{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-networking-config", Namespace: ns}, &cm) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// _, found := cm.Data["CNVRG_PROXY_CONFIG_REF"] +//// Expect(found).Should(BeFalse()) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("HPA disabled", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.WebApp.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// hpa := v2beta1.HorizontalPodAutoscaler{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &hpa) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeFalse()) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("WebApp HPA Enabled", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.WebApp.Enabled = true +//// app.Spec.ControlPlane.WebApp.Hpa.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// hpa := v2beta1.HorizontalPodAutoscaler{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, &hpa) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.WebApp.Replicas))) +//// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.WebApp.Hpa.MaxReplicas))) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Sidekiq HPA Enabled", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.Sidekiq.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Split = true +//// app.Spec.ControlPlane.Sidekiq.Hpa.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// hpa := v2beta1.HorizontalPodAutoscaler{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "sidekiq", Namespace: ns}, &hpa) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Sidekiq.Replicas))) +//// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Sidekiq.Hpa.MaxReplicas))) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Searchkiq HPA Enabled", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.Searchkiq.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Split = true +//// app.Spec.ControlPlane.Searchkiq.Hpa.Enabled = true +//// app.Spec.ControlPlane.Searchkiq.Hpa.MaxReplicas = 10 +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// hpa := v2beta1.HorizontalPodAutoscaler{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "searchkiq", Namespace: ns}, &hpa) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Searchkiq.Replicas))) +//// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Searchkiq.Hpa.MaxReplicas))) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("Systemkiq HPA Enabled", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.Systemkiq.Enabled = true +//// app.Spec.ControlPlane.Sidekiq.Split = true +//// app.Spec.ControlPlane.Systemkiq.Hpa.Enabled = true +//// app.Spec.ControlPlane.Systemkiq.Hpa.Utilization = 90 +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// hpa := v2beta1.HorizontalPodAutoscaler{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "systemkiq", Namespace: ns}, &hpa) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(*hpa.Spec.MinReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Systemkiq.Replicas))) +//// Expect(hpa.Spec.MaxReplicas).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Systemkiq.Hpa.MaxReplicas))) +//// Expect(*hpa.Spec.Metrics[0].Resource.TargetAverageUtilization).Should(BeEquivalentTo(int32(app.Spec.ControlPlane.Systemkiq.Hpa.Utilization))) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("SMTP OpensslVerifyMode", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.SMTP.OpensslVerifyMode = "foo" +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// secret := corev1.Secret{} +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-smtp", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(string(secret.Data["SMTP_OPENSSL_VERIFY_MODE"])).To(Equal("foo")) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// It("SMTP Sender", func() { +//// ctx := context.Background() +//// ns := createNs() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.ControlPlane.SMTP.Sender = "test@cnvrg.io" +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// secret := corev1.Secret{} +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-smtp", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(string(secret.Data["SMTP_SENDER"])).To(Equal("test@cnvrg.io")) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Test Object Storage Secret", func() { +//// It("Object Storage Secret - Minio enabled - random keys", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Minio.Enabled = true +//// testApp.Spec.ControlPlane.ObjectStorage.Type = "minio" +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// secret := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).ShouldNot(BeEmpty()) +//// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).ShouldNot(BeEmpty()) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Object Storage Secret - Minio enabled - explicitly configured keys", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Minio.Enabled = true +//// testApp.Spec.ControlPlane.ObjectStorage.Type = "minio" +//// testApp.Spec.ControlPlane.ObjectStorage.AccessKey = "access-key" +//// testApp.Spec.ControlPlane.ObjectStorage.SecretKey = "secret-key" +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// secret := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(Equal([]byte("access-key"))) +//// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(Equal([]byte("secret-key"))) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Object Storage Secret - Minio external - explicitly configured keys", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.ObjectStorage.Type = "minio" +//// testApp.Spec.ControlPlane.ObjectStorage.AccessKey = "access-key" +//// testApp.Spec.ControlPlane.ObjectStorage.SecretKey = "secret-key" +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// secret := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(Equal([]byte("access-key"))) +//// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(Equal([]byte("secret-key"))) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Object Storage Secret - AWS S3 - explicitly configured keys", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.ObjectStorage.Type = "aws" +//// testApp.Spec.ControlPlane.ObjectStorage.AccessKey = "access-key" +//// testApp.Spec.ControlPlane.ObjectStorage.SecretKey = "secret-key" +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// secret := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(Equal([]byte("access-key"))) +//// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(Equal([]byte("secret-key"))) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Object Storage Secret - AWS S3 - IAM S3 access", func() { +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.ObjectStorage.Type = "aws" +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// secret := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cp-object-storage", Namespace: ns}, &secret) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(secret.Data["CNVRG_STORAGE_ACCESS_KEY"]).Should(BeEmpty()) +//// Expect(secret.Data["CNVRG_STORAGE_SECRET_KEY"]).Should(BeEmpty()) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("CnvrgRouter", func() { +//// It("Cnvrg Router Labels", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true +//// testApp.Spec.Labels = labels +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Labels).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Cnvrg Router Annotations", func() { +//// ns := createNs() +//// ctx := context.Background() +//// labels := map[string]string{"foo": "bar"} +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true +//// testApp.Spec.Annotations = labels +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(deployment.Annotations).Should(HaveKeyWithValue("foo", "bar")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// It("Cnvrg Router Tenancy", func() { +//// ns := createNs() +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true +//// testApp.Spec.Tenancy.Enabled = true +//// +//// deployment := v1.Deployment{} +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// t := corev1.Toleration{ +//// Key: testApp.Spec.Tenancy.Key, +//// Operator: "Equal", +//// Value: testApp.Spec.Tenancy.Value, +//// Effect: "NoSchedule", +//// } +//// +//// Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElement(t)) +//// Expect(deployment.Spec.Template.Spec.NodeSelector).Should(HaveKeyWithValue("purpose", "cnvrg-control-plane")) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Control Plane - Default Istio ingress", func() { +//// +//// It("Default Istio ingress", func() { +//// ns := createNs() +//// gwName := fmt.Sprintf(mlopsv1.IstioGwName, ns) +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Networking.Ingress.IstioGwEnabled = true +//// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.Logging.Kibana.Enabled = true +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Monitoring.Prometheus.Enabled = true +//// testApp.Spec.Monitoring.Grafana.Enabled = true +//// +//// infra := getDefaultTestInfraSpec(ns) +//// infra.Spec.Monitoring.Prometheus.Enabled = true +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// gw := &unstructured.Unstructured{} +//// gw.SetGroupVersionKind(desired.Kinds[desired.IstioGwGVK]) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: gwName, Namespace: ns}, gw) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// vs := &unstructured.Unstructured{} +//// vs.SetGroupVersionKind(desired.Kinds[desired.IstioVsGVK]) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Logging.Kibana.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Prometheus.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Grafana.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// +//// }) +//// +//// It("Custom Istio ingress Name", func() { +//// ns := createNs() +//// gwName := "foo-bar" +//// ctx := context.Background() +//// +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Networking.Ingress.IstioGwEnabled = true +//// testApp.Spec.Networking.Ingress.IstioGwName = gwName +//// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.Logging.Kibana.Enabled = true +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Monitoring.Prometheus.Enabled = true +//// testApp.Spec.Monitoring.Grafana.Enabled = true +//// +//// infra := getDefaultTestInfraSpec(ns) +//// infra.Spec.Monitoring.Prometheus.Enabled = true +//// infra.Spec.Networking.Ingress.Type = mlopsv1.NginxIngress +//// +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// +//// gw := &unstructured.Unstructured{} +//// gw.SetGroupVersionKind(desired.Kinds[desired.IstioGwGVK]) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: gwName, Namespace: ns}, gw) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// vs := &unstructured.Unstructured{} +//// vs.SetGroupVersionKind(desired.Kinds[desired.IstioVsGVK]) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Logging.Kibana.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Prometheus.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Grafana.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// +//// }) +//// +//// It("Istio Disabled - Custom Istio ingress Name", func() { +//// ns := createNs() +//// +//// gwName := "foo-bar" +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Networking.Ingress.IstioGwName = gwName +//// testApp.Spec.ControlPlane.CnvrgRouter.Enabled = true +//// testApp.Spec.ControlPlane.WebApp.Enabled = true +//// testApp.Spec.Logging.Kibana.Enabled = true +//// testApp.Spec.Dbs.Es.Enabled = true +//// testApp.Spec.Monitoring.Prometheus.Enabled = true +//// testApp.Spec.Monitoring.Grafana.Enabled = true +//// +//// infra := getDefaultTestInfraSpec(ns) +//// infra.Spec.Monitoring.Prometheus.Enabled = true +//// infra.Spec.Networking.Ingress.Type = mlopsv1.NginxIngress +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// +//// vs := &unstructured.Unstructured{} +//// vs.SetGroupVersionKind(desired.Kinds[desired.IstioVsGVK]) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.WebApp.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Es.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Logging.Kibana.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Prometheus.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Monitoring.Grafana.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.ControlPlane.CnvrgRouter.SvcName, Namespace: ns}, vs) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// Expect(vs.Object["spec"].(map[string]interface{})["gateways"]).Should(ContainElement(gwName)) +//// +//// gw := &unstructured.Unstructured{} +//// gw.SetGroupVersionKind(desired.Kinds[desired.IstioGwGVK]) +//// time.Sleep(time.Second * 3) +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: gwName, Namespace: ns}, gw) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeFalse()) +//// Expect(k8sClient.Delete(ctx, testApp)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Test Proxy ConfigMap creation", func() { +//// It("Proxy configmap test creation - default no_proxy", func() { +//// +//// ctx := context.Background() +//// ns := createNs() +//// expectedNoProxy := networking.DefaultNoProxy("cluster.local") +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.Networking.Proxy.HttpProxy = []string{ +//// "http://proxy1.org.local", +//// "http://proxy2.org.local", +//// } +//// app.Spec.Networking.Proxy.HttpsProxy = []string{ +//// "https://proxy1.org.local", +//// "https://proxy2.org.local", +//// } +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// +//// Eventually(func() bool { +//// appRes := mlopsv1.CnvrgApp{} +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, &appRes) +//// if err != nil { +//// return false +//// } +//// sort.Strings(appRes.Spec.Networking.Proxy.NoProxy) +//// sort.Strings(expectedNoProxy) +//// return reflect.DeepEqual(appRes.Spec.Networking.Proxy.NoProxy, expectedNoProxy) +//// }, timeout, interval).Should(BeTrue()) +//// +//// cm := corev1.ConfigMap{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Networking.Proxy.ConfigRef, Namespace: ns}, &cm) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(cm.Data).Should(HaveKeyWithValue("HTTP_PROXY", "http://proxy1.org.local,http://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("http_proxy", "http://proxy1.org.local,http://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("https_proxy", "https://proxy1.org.local,https://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("HTTPS_PROXY", "https://proxy1.org.local,https://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("NO_PROXY", strings.Join(expectedNoProxy, ","))) +//// Expect(cm.Data).Should(HaveKeyWithValue("no_proxy", strings.Join(expectedNoProxy, ","))) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// It("Proxy configmap test creation - custom no_proxy", func() { +//// +//// ctx := context.Background() +//// ns := createNs() +//// noProxy := []string{".foo.bar"} +//// expectedNoProxy := append(noProxy, networking.DefaultNoProxy("cluster.local")...) +//// app := getDefaultTestAppSpec(ns) +//// cm := corev1.ConfigMap{} +//// +//// app.Spec.Networking.Proxy.Enabled = true +//// app.Spec.Networking.Proxy.HttpProxy = []string{ +//// "http://proxy1.org.local", +//// "http://proxy2.org.local", +//// } +//// app.Spec.Networking.Proxy.HttpsProxy = []string{ +//// "https://proxy1.org.local", +//// "https://proxy2.org.local", +//// } +//// +//// app.Spec.Networking.Proxy.NoProxy = noProxy +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// +//// Eventually(func() bool { +//// appRes := mlopsv1.CnvrgApp{} +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, &appRes) +//// if err != nil { +//// return false +//// } +//// sort.Strings(appRes.Spec.Networking.Proxy.NoProxy) +//// sort.Strings(expectedNoProxy) +//// return reflect.DeepEqual(appRes.Spec.Networking.Proxy.NoProxy, expectedNoProxy) +//// }, timeout, interval).Should(BeTrue()) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Networking.Proxy.ConfigRef, Namespace: ns}, &cm) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(cm.Data).Should(HaveKeyWithValue("HTTP_PROXY", "http://proxy1.org.local,http://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("http_proxy", "http://proxy1.org.local,http://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("https_proxy", "https://proxy1.org.local,https://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("HTTPS_PROXY", "https://proxy1.org.local,https://proxy2.org.local")) +//// Expect(cm.Data).Should(HaveKeyWithValue("NO_PROXY", strings.Join(expectedNoProxy, ","))) +//// Expect(cm.Data).Should(HaveKeyWithValue("no_proxy", strings.Join(expectedNoProxy, ","))) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// It("Proxy configmap test creation - proxy disabled", func() { +//// +//// ctx := context.Background() +//// ns := createNs() +//// app := getEmptyTestAppSpec(ns) +//// +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// +//// Eventually(func() bool { +//// appRes := mlopsv1.CnvrgApp{} +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, &appRes) +//// if err != nil { +//// return false +//// } +//// return appRes.Spec.Networking.Proxy.Enabled == false +//// }, timeout, interval).Should(BeTrue()) +//// +//// cm := corev1.ConfigMap{} +//// +//// time.Sleep(3) +//// +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Networking.Proxy.ConfigRef, Namespace: ns}, &cm) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeFalse()) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// }) +//// }) +//// +//// Context("Test Priority class", func() { +//// It("CnvrgApp and CnvrgJob priority classes names", func() { +//// ns := createNs() +//// ctx := context.Background() +//// infra := getDefaultTestInfraSpec(ns) +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.CnvrgJobPriorityClass.Name = "" +//// app.Spec.PriorityClass.AppClassRef = "" +//// app.Spec.Dbs.Pg.Enabled = true +//// // create infra +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// // create app +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// +//// deployment := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// expectedApp := mlopsv1.CnvrgApp{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: ns}, &expectedApp) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// Expect(expectedApp.Spec.PriorityClass.AppClassRef).To(Equal(infra.Spec.PriorityClass.AppClassRef)) +//// Expect(expectedApp.Spec.CnvrgJobPriorityClass.Name).To(Equal(infra.Spec.CnvrgJobPriorityClass.Name)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// }) +//// }) +//// Context("Test AppMonitoring", func() { +//// It("Prom creds secret generator", func() { +//// ns := createNs() +//// ctx := context.Background() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Monitoring.Prometheus.Enabled = true +//// infra := getDefaultTestInfraSpec(ns) +//// infra.Spec.Monitoring.Prometheus.Enabled = true +//// // create infra +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// // create app +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// // get prom creds +//// promCreds := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.CredsRef, Namespace: ns}, &promCreds) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // enforce reconcile loop - enable Prometheus Operator and make sure it was deployed +//// infraRes := mlopsv1.CnvrgInfra{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: ns}, &infraRes) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// rvBeforeUpdate := infraRes.ObjectMeta.ResourceVersion +//// infraRes.Spec.Monitoring.PrometheusOperator.Enabled = true +//// Expect(k8sClient.Update(ctx, &infraRes)).Should(Succeed()) +//// dep := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrg-prometheus-operator", Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure resource version has been updated +//// Expect(rvBeforeUpdate).Should(Not(Equal(infraRes.ObjectMeta.ResourceVersion))) +//// // get prometheus creds after reconcile +//// promCredsAfterReconcile := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.CredsRef, Namespace: ns}, &promCredsAfterReconcile) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure redis creds wasn't mutated between reconciliation loops +//// Expect(promCreds.Data["CNVRG_PROMETHEUS_PASS"]).Should(Equal(promCredsAfterReconcile.Data["CNVRG_PROMETHEUS_PASS"])) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// }) +//// +//// It("Prom upstream creds secret generator", func() { +//// ns := createNs() +//// ctx := context.Background() +//// app := getDefaultTestAppSpec(ns) +//// app.Spec.Monitoring.Prometheus.Enabled = true +//// infra := getDefaultTestInfraSpec(ns) +//// infra.Spec.Monitoring.Prometheus.Enabled = true +//// // create infra +//// Expect(k8sClient.Create(ctx, infra)).Should(Succeed()) +//// // create app +//// Expect(k8sClient.Create(ctx, app)).Should(Succeed()) +//// // get prom creds +//// promUpstreamCreds := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.UpstreamRef, Namespace: ns}, &promUpstreamCreds) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // enforce reconcile loop - enable Prometheus Operator and make sure it was deployed +//// infraRes := mlopsv1.CnvrgInfra{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: ns}, &infraRes) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// rvBeforeUpdate := infraRes.ObjectMeta.ResourceVersion +//// infraRes.Spec.Monitoring.PrometheusOperator.Enabled = true +//// Expect(k8sClient.Update(ctx, &infraRes)).Should(Succeed()) +//// dep := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: "cnvrg-prometheus-operator", Namespace: ns}, &dep) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure resource version has been updated +//// Expect(rvBeforeUpdate).Should(Not(Equal(infraRes.ObjectMeta.ResourceVersion))) +//// // get prometheus creds after reconcile +//// promUpstreamCredsAfterReconcile := corev1.Secret{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: app.Spec.Monitoring.Prometheus.UpstreamRef, Namespace: ns}, &promUpstreamCredsAfterReconcile) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// // Make sure redis creds wasn't mutated between reconciliation loops +//// Expect(promUpstreamCreds.Data).Should(Equal(promUpstreamCredsAfterReconcile.Data)) +//// Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) +//// Expect(k8sClient.Delete(ctx, infra)).Should(Succeed()) +//// }) +//// +//// }) +//// +//// Context("Test Events", func() { +//// +//// It("Simple event test", func() { +//// +//// ns := createNs() +//// ctx := context.Background() +//// testApp := getDefaultTestAppSpec(ns) +//// testApp.Spec.Dbs.Pg.Enabled = true +//// +//// Expect(k8sClient.Create(ctx, testApp)).Should(Succeed()) +//// +//// deployment := v1.Deployment{} +//// Eventually(func() bool { +//// err := k8sClient.Get(ctx, types.NamespacedName{Name: testApp.Spec.Dbs.Pg.SvcName, Namespace: ns}, &deployment) +//// if err != nil { +//// return false +//// } +//// return true +//// }, timeout, interval).Should(BeTrue()) +//// +//// events := corev1.EventList{} +//// if err := k8sClient.List(ctx, &events); err != nil { +//// fmt.Println(err) +//// } +//// es := corev1.EventSource{Component: "cnvrgapp"} +//// for _, e := range events.Items { +//// if e.Source == es { +//// Succeed() +//// return +//// } +//// } +//// Fail("expected cnvrgapp event not found") +//// }) +//// +//// }) +//// +//// }) +//func createNs() string { +// ns, _ := shortid.Generate() +// ns = strings.ReplaceAll(strings.ToLower(ns), "-", "z") +// ns = strings.ReplaceAll(ns, "_", "z") +// testNs := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} +// err := tc.client.Create(context.Background(), testNs) +// if err != nil { +// panic(err) +// } +// return ns +//} +// +//func getEmptyTestAppSpec(ns string) *mlopsv1.CnvrgApp { +// +// return &mlopsv1.CnvrgApp{ +// TypeMeta: metav1.TypeMeta{ +// Kind: "CnvrgApp", +// APIVersion: "mlops.cnvrg.io/v1", +// }, +// ObjectMeta: metav1.ObjectMeta{ +// Name: "cnvrgapp", +// Namespace: ns, +// }, +// } +//} +// +//func getDefaultTestAppSpec(ns string) *mlopsv1.CnvrgApp { +// testSpec := mlopsv1.DefaultCnvrgAppSpec() +// testSpec.ClusterDomain = "test.local" +// return &mlopsv1.CnvrgApp{ +// TypeMeta: metav1.TypeMeta{ +// Kind: "CnvrgApp", +// APIVersion: "mlops.cnvrg.io/v1", +// }, +// ObjectMeta: metav1.ObjectMeta{ +// Name: "cnvrgapp", +// Namespace: ns, +// }, +// Spec: testSpec, +// } +//} diff --git a/controllers/test/controller_test.go b/controllers/test/controller_test.go new file mode 100644 index 00000000..c34800bb --- /dev/null +++ b/controllers/test/controller_test.go @@ -0,0 +1,72 @@ +package test + +import ( + cnvrgv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" +) + +var _ = Describe("Cnvrg App controller", func() { + labels := map[string]string{"foo": "bar"} + annotations := map[string]string{"foo1": "bar1"} + + It("should pass crd sanity", func() { + namespace := genNS(GenName("ns")) + ExpectCreate(&namespace) + + By("dry-running a basic cnvg app") + obj := genApp(namespace.Name, GenName("cnvrg")) + ExpectDryRunCreate(&obj) + }) + + It("should create and destroy managed resources", func() { + nsName := GenName("ns") + namespace := genNS(nsName) + ExpectCreate(&namespace) + + By("creating simple cnvrg app", func() { + appName := GenName("cnvrg") + obj := genApp(nsName, appName) + ExpectCreate(&obj) + EventuallyGet(Key(nsName, appName), &obj) + }) + + By("validating app with PG enabled", func() { + appName := GenName("cnvrg") + appKey := Key(nsName, appName) + var obj cnvrgv1.CnvrgApp + var managedDeployment v1.Deployment + + By("creating cnvrg app with PG", func() { + obj = genApp(nsName, appName) + obj.Spec.Dbs.Pg.Enabled = true + obj.Spec.Labels = labels + obj.Spec.Annotations = annotations + ExpectCreate(&obj) + EventuallyGet(appKey, &obj) + }) + deployKey := Key(nsName, obj.Spec.Dbs.Pg.SvcName) + By("validating managed deployment exists", func() { + EventuallyGet(deployKey, &managedDeployment) + + // expected labels exists on managed deployment + Expect(managedDeployment.Labels).To(SatisfyAll( + MatchSubMap(labels), + )) + + // expected annotations exists on managed deployment + Expect(managedDeployment.Annotations).To(SatisfyAll( + MatchSubMap(annotations), + )) + }) + + By("deleting app", func() { + ExpectDelete(appKey, &obj) + EventuallyGone(appKey, &obj) + }) + }) + + }) +}, +) diff --git a/controllers/test/gen.go b/controllers/test/gen.go new file mode 100644 index 00000000..c77c35d2 --- /dev/null +++ b/controllers/test/gen.go @@ -0,0 +1,36 @@ +package test + +import ( + mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func genApp(ns string, name string) mlopsv1.CnvrgApp { + testSpec := mlopsv1.DefaultCnvrgAppSpec() + testSpec.ClusterDomain = "test.local" + return mlopsv1.CnvrgApp{ + TypeMeta: metav1.TypeMeta{ + Kind: "CnvrgApp", + APIVersion: "mlops.cnvrg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: testSpec, + } +} + +func genNS(name string) corev1.Namespace { + testSpec := mlopsv1.DefaultCnvrgAppSpec() + testSpec.ClusterDomain = "test.local" + return corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } +} diff --git a/controllers/test/suite_test.go b/controllers/test/suite_test.go new file mode 100644 index 00000000..6648e951 --- /dev/null +++ b/controllers/test/suite_test.go @@ -0,0 +1,116 @@ +package test + +import ( + "context" + mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" + "github.com/AccessibleAI/cnvrg-operator/controllers/app" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "math/rand" + "os" + "path/filepath" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "testing" + "time" +) + +var tc TestCtx + +type TestCtx struct { + ctx context.Context + + env *envtest.Environment + mgr ctrl.Manager + client client.Client + scheme *runtime.Scheme + rnd *rand.Rand +} + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + setExtBinsEnv() + err := os.Setenv("ACK_GINKGO_DEPRECATIONS", "1.16.4") + Expect(err).To(BeNil()) + + SetDefaultEventuallyTimeout(10 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + SetDefaultConsistentlyDuration(2 * time.Second) + SetDefaultConsistentlyPollingInterval(250 * time.Millisecond) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + done := make(chan struct{}) + + go func() { + projectBaseDir := "../../" + crdPaths := []string{ + filepath.Join(projectBaseDir, "charts/mlops/crds/"), + } + + env := &envtest.Environment{ + CRDDirectoryPaths: crdPaths, + } + + cfg, err := env.Start() + Expect(err).ToNot(HaveOccurred()) + + scheme := runtime.NewScheme() + Expect(clientgoscheme.AddToScheme(scheme)) + Expect(mlopsv1.AddToScheme(scheme)).To(Succeed()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + LeaderElection: false, + }) + Expect(err).To(BeNil()) + + ctx := signals.SetupSignalHandler() + tc = TestCtx{ + ctx: ctx, + env: env, + mgr: mgr, + scheme: mgr.GetScheme(), + client: mgr.GetClient(), + rnd: rand.New(rand.NewSource(GinkgoRandomSeed())), + } + err = (&app.CnvrgAppReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr) + + Expect(err).ToNot(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err := mgr.Start(tc.ctx) + Expect(err).ToNot(HaveOccurred()) + + gexec.KillAndWait(4 * time.Second) + + // Teardown the test environment once controller is fnished. + // Otherwise from Kubernetes 1.21+, teardon timeouts waiting on + // kube-apiserver to return + err = env.Stop() + Expect(err).ToNot(HaveOccurred()) + }() + + close(done) + }() + + Eventually(done, 60).Should(BeClosed()) +}) + +func setExtBinsEnv() { + const extBinsLocation = "/tmp/kubebuilder/bin/" // scripts/fetch_ext_bins.sh should run before + Expect(os.Setenv("TEST_ASSET_KUBE_APISERVER", extBinsLocation+"kube-apiserver")).To(Succeed()) + Expect(os.Setenv("TEST_ASSET_ETCD", extBinsLocation+"etcd")).To(Succeed()) + Expect(os.Setenv("TEST_ASSET_KUBECTL", extBinsLocation+"kubectl")).To(Succeed()) +} diff --git a/controllers/test/suite_utils_test.go b/controllers/test/suite_utils_test.go new file mode 100644 index 00000000..8b97da6b --- /dev/null +++ b/controllers/test/suite_utils_test.go @@ -0,0 +1,150 @@ +package test + +import ( + "context" + "errors" + "github.com/onsi/gomega" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" + kerrors "k8s.io/apimachinery/pkg/api/errors" + ktypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "strings" +) + +// ExpectDryRunCreate runs a dry-run creation. +func ExpectDryRunCreate(obj client.Object) { + ExpectWithOffset(1, Create(obj, client.DryRunAll)).To(Succeed()) +} + +// ExpectCreate creates an object as part of an "It". +// It's just a helper to make setup actions easy to find. +func ExpectCreate(obj client.Object) { + ExpectWithOffset(1, Create(obj)).To(Succeed()) +} + +// Create wraps the k8s client create. +func Create(obj client.Object, opts ...client.CreateOption) error { + return tc.client.Create(context.Background(), obj, opts...) +} + +// GenName generates a random name from a given base. +func GenName(base string) string { + const ( + charset = "abcdefghijklmnopqrstuvwxyz" + suffixLen = 8 + ) + sb := strings.Builder{} + sb.WriteString(base) + for i := 0; i < suffixLen; i++ { + sb.WriteByte(charset[tc.rnd.Intn(len(charset))]) + } + return sb.String() +} + +// Key builds a namespace-scoped object key. +func Key(namespace string, name string) ktypes.NamespacedName { + return ktypes.NamespacedName{ + Namespace: namespace, + Name: name, + } +} + +// EventuallyGet gets an object with an eventually wrapper. +func EventuallyGet(key ktypes.NamespacedName, obj client.Object, intervals ...interface{}) { + // this matches the return value, so any non error result is fine + EventuallyWithOffset(1, GetF(key, obj), intervals...).ShouldNot(BeNil()) +} + +// GetF returns a function that wraps the k8s client get. +// It helps with the use of async matchers. +func GetF(key ktypes.NamespacedName, obj client.Object) func() (interface{}, error) { + return func() (interface{}, error) { + err := Get(key, obj) + return obj, err + } +} + +// Get wraps the k8s client get. +func Get(key ktypes.NamespacedName, obj client.Object) error { + return tc.client.Get(tc.ctx, key, obj) +} + +// MatchSubMap succeeds if the provided map are a subset of actual. +func MatchSubMap(kv map[string]string) types.GomegaMatcher { + kk := gstruct.Keys{} + for k, v := range kv { + kk[k] = gomega.Equal(v) + } + return gstruct.MatchKeys(gstruct.IgnoreExtras, kk) +} + +// Delete wraps the k8s client delete. +func Delete(obj client.Object, opts ...client.DeleteOption) error { + return tc.client.Delete(tc.ctx, obj, opts...) +} + +// DeleteF returns a function that wraps the k8s client delete. +// It helps with the use of async matchers. +func DeleteF(obj client.Object, opts ...client.DeleteOption) func() error { + return func() error { + return Delete(obj, opts...) + } +} + +// ExpectDelete deletes an object. +// For objects without finalizers, an easier option is ExceptDeleteGone. +func ExpectDelete(key ktypes.NamespacedName, obj client.Object) { + ExpectWithOffset(1, Get(key, obj)).To(Succeed()) + ExpectWithOffset(1, Delete(obj)).To(Succeed()) +} + +// EventuallyGone expects an object to eventually disappear. +func EventuallyGone(key ktypes.NamespacedName, obj client.Object, intervals ...interface{}) { + EventuallyWithOffset(1, func() bool { + err := Get(key, obj) + return kerrors.IsNotFound(err) + }, intervals...).Should(BeTrue()) +} + +// ExpectDeleteGone deletes an object and waits for it to disappear. +// It is like calling ExpectDelete followed by EventuallyGone. +// Do not use this method for objects with finalizers. +func ExpectDeleteGone(key ktypes.NamespacedName, obj client.Object, intervals ...interface{}) { + ExpectWithOffset(1, Get(key, obj)).To(Succeed()) + ExpectWithOffset(1, Delete(obj)).To(Succeed()) + Eventually(func() bool { + err := Get(key, obj) + return kerrors.IsNotFound(err) + }, intervals...).Should(BeTrue()) +} + +// EventuallyFinalize waits for an object to have a deletion timestamp and removes all finalizers from it. +// It then waits for the finalized object to disappear. +func EventuallyFinalize(key ktypes.NamespacedName, obj client.Object, intervals ...interface{}) { + ctx := tc.ctx + EventuallyWithOffset(1, func() error { + err := tc.client.Get(ctx, key, obj) + if err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return err + } + if !obj.GetDeletionTimestamp().IsZero() { + return errors.New("not deleted") + } + obj.SetFinalizers(nil) + return Update(obj) + }, intervals...).Should(BeNil()) + EventuallyWithOffset(1, func() bool { + err := tc.client.Get(ctx, key, obj) + return kerrors.IsNotFound(err) + }, intervals...).Should(BeTrue()) +} + +// Update wraps the k8s client update. +func Update(obj client.Object, opts ...client.UpdateOption) error { + return tc.client.Update(tc.ctx, obj, opts...) +} diff --git a/go.mod b/go.mod index 514012e8..e84df939 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,11 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.30.0 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 + github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.18.0 gopkg.in/d4l3k/messagediff.v1 v1.2.1 @@ -59,6 +62,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect @@ -85,6 +89,7 @@ require ( google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.0 // indirect diff --git a/go.sum b/go.sum index 1df0921f..74e79478 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1 github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -42,18 +44,29 @@ github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdX github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -67,6 +80,7 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -102,8 +116,17 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= @@ -144,12 +167,15 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 h1:xzABM9let0HLLqFypcxvLmlvEciCHL7+Lv+4vwZqecI= +github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569/go.mod h1:2Ly+NIftZN4de9zRmENdYbvPQeaVIYKWpLFStLFEBgI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -170,9 +196,11 @@ golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGb golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -180,14 +208,21 @@ golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -208,6 +243,7 @@ golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= @@ -220,6 +256,12 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= @@ -229,11 +271,17 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/d4l3k/messagediff.v1 v1.2.1 h1:70AthpjunwzUiarMHyED52mj9UwtAnE89l1Gmrt3EU0= gopkg.in/d4l3k/messagediff.v1 v1.2.1/go.mod h1:EUzikiKadqXWcD1AzJLagx0j/BeeWGtn++04Xniyg44= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scripts/fetch_ext_bins.sh b/scripts/fetch_ext_bins.sh new file mode 100644 index 00000000..95276f6a --- /dev/null +++ b/scripts/fetch_ext_bins.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Source - https://github.com/kubernetes-sigs/cluster-api/blob/95fe9e2c2c48cb7c765e40fe97861f22765441ff/scripts/fetch_ext_bins.sh +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Enable tracing in this script off by setting the TRACE variable in your +# environment to any value: +# +# $ TRACE=1 test.sh +TRACE=${TRACE:-""} +if [[ -n "${TRACE}" ]]; then + set -x +fi + +k8s_version=1.30.0 +goarch=amd64 +goos="unknown" + +if [[ "${OSTYPE}" == "linux"* ]]; then + goos="linux" +elif [[ "${OSTYPE}" == "darwin"* ]]; then + goos="darwin" +fi + +if [[ "$goos" == "unknown" ]]; then + echo "OS '$OSTYPE' not supported. Aborting." >&2 + exit 1 +fi + +# Turn colors in this script off by setting the NO_COLOR variable in your +# environment to any value: +# +# $ NO_COLOR=1 test.sh +NO_COLOR=${NO_COLOR:-""} +if [[ -z "${NO_COLOR}" ]]; then + header=$'\e[1;33m' + reset=$'\e[0m' +else + header='' + reset='' +fi + +function header_text { + echo "$header$*$reset" +} + +tmp_root=/tmp + +# Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable +# in your environment to any value: +# +# $ SKIP_FETCH_TOOLS=1 ./fetch_ext_bins.sh +# +# If you skip fetching tools, this script will use the tools already on your +# machine, but rebuild the kubebuilder and kubebuilder-bin binaries. +SKIP_FETCH_TOOLS=${SKIP_FETCH_TOOLS:-""} + +# fetch k8s API gen tools and make it available under kb_root_dir/bin. +function fetch_tools { + if [[ -n "$SKIP_FETCH_TOOLS" ]]; then + return 0 + fi + + mkdir -p ${tmp_root} + + # use the pre-existing version in the temporary folder if it matches our k8s version + if [[ -x "${tmp_root}/kubebuilder/bin/kube-apiserver" ]]; then + version=$(${tmp_root}/kubebuilder/bin/kube-apiserver --version) + if [[ $version == *"${k8s_version}"* ]]; then + return 0 + fi + fi + + header_text "fetching kubebuilder-tools@${k8s_version}" + kb_tools_archive_name="kubebuilder-tools-${k8s_version}-${goos}-${goarch}.tar.gz" + kb_tools_download_url="https://storage.googleapis.com/kubebuilder-tools/${kb_tools_archive_name}" + + kb_tools_archive_path="${tmp_root}/${kb_tools_archive_name}" + if [[ ! -f ${kb_tools_archive_path} ]]; then + curl -fsL ${kb_tools_download_url} -o "${kb_tools_archive_path}" + fi + tar -zvxf "${kb_tools_archive_path}" -C "${tmp_root}/" +} + +function setup_envs { + header_text "setting up kubebuilder-tools@${k8s_version} env vars" + + # Setup env vars + export PATH=${tmp_root}/kubebuilder/bin:$PATH + export TEST_ASSET_KUBECTL=${tmp_root}/kubebuilder/bin/kubectl + export TEST_ASSET_KUBE_APISERVER=${tmp_root}/kubebuilder/bin/kube-apiserver + export TEST_ASSET_ETCD=${tmp_root}/kubebuilder/bin/etcd +} \ No newline at end of file