diff --git a/Makefile b/Makefile index ac217b7a..bd7c5047 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,8 @@ integration-test: $(GINKGO) $(ETCD) $(GINKGO) -v controllers/machine -- -enable-integration-tests -enable-unit-tests=false $(GINKGO) -v controllers/cluster -- -enable-integration-tests -enable-unit-tests=false $(GINKGO) -v controllers/configmap -- -enable-integration-tests -enable-unit-tests=false + $(GINKGO) -v controllers/tests/cluster_for_akodeploymentconfig/default_adc -- -enable-integration-tests -enable-unit-tests=false -root-dir="../../../.." + $(GINKGO) -v controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors -- -enable-integration-tests -enable-unit-tests=false -root-dir="../../../.." .PHONY: kind-e2e-test kind-e2e-test: $(KUSTOMIZE) $(KIND) $(KUBECTL) $(JQ) $(YTT) diff --git a/controllers/akodeploymentconfig/akodeploymentconfig_controller_intg_test.go b/controllers/akodeploymentconfig/akodeploymentconfig_controller_intg_test.go index f9b19db9..d279d6f0 100644 --- a/controllers/akodeploymentconfig/akodeploymentconfig_controller_intg_test.go +++ b/controllers/akodeploymentconfig/akodeploymentconfig_controller_intg_test.go @@ -9,23 +9,24 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" - akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako" - ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" "github.com/vmware/alb-sdk/go/models" "github.com/vmware/alb-sdk/go/session" akov1alpha1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/cluster-api/util/conditions" - kcfg "sigs.k8s.io/cluster-api/util/kubeconfig" - ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + kcfg "sigs.k8s.io/cluster-api/util/kubeconfig" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako" + ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" ) func intgTestAkoDeploymentConfigController() { @@ -51,56 +52,9 @@ func intgTestAkoDeploymentConfigController() { userCreateCalled bool ) - staticCluster = &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "integration-test-8ed12g", - Namespace: "integration-test-8ed12g", - }, - Spec: clusterv1.ClusterSpec{}, - } - - defaultAkoDeploymentConfigCommonSpec := akoov1alpha1.AKODeploymentConfigSpec{ - DataNetwork: akoov1alpha1.DataNetwork{ - Name: "integration-test-8ed12g", - CIDR: "10.0.0.0/24", - IPPools: []akoov1alpha1.IPPool{ - { - Start: "10.0.0.1", - End: "10.0.0.10", - Type: "V4", - }, - }, - }, - ControlPlaneNetwork: akoov1alpha1.ControlPlaneNetwork{ - Name: "integration-test-8ed12g", - CIDR: "10.1.0.0/24", - }, - ServiceEngineGroup: "ha-test", - AdminCredentialRef: &akoov1alpha1.SecretRef{ - Name: "controller-credentials", - Namespace: "default", - }, - CertificateAuthorityRef: &akoov1alpha1.SecretRef{ - Name: "controller-ca", - Namespace: "default", - }, - } - akoDeploymentConfigCommonSpec := defaultAkoDeploymentConfigCommonSpec.DeepCopy() - akoDeploymentConfigCommonSpec.ClusterSelector = metav1.LabelSelector{ - MatchLabels: map[string]string{ - "test": "true", - }, - } - staticAkoDeploymentConfig = &akoov1alpha1.AKODeploymentConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ako-deployment-config", - }, - Spec: *akoDeploymentConfigCommonSpec, - } - staticDefaultAkoDeploymentConfig = &akoov1alpha1.AKODeploymentConfig{ - ObjectMeta: metav1.ObjectMeta{Name: akoov1alpha1.WorkloadClusterAkoDeploymentConfig}, - Spec: defaultAkoDeploymentConfigCommonSpec, - } + staticCluster = &testutil.DefaultCluster + staticAkoDeploymentConfig = testutil.GetCustomizedADC(testutil.CustomizedADCLabels) + staticDefaultAkoDeploymentConfig = testutil.GetDefaultADC() staticControllerCredentials = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controllers/akodeploymentconfig/phases/phases_unit_test.go b/controllers/akodeploymentconfig/phases/phases_unit_test.go index 48cb5b34..65f49cbd 100644 --- a/controllers/akodeploymentconfig/phases/phases_unit_test.go +++ b/controllers/akodeploymentconfig/phases/phases_unit_test.go @@ -4,16 +4,16 @@ package phases import ( - ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" - "github.com/go-logr/logr" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" ) func ReconcilePhaseUnitTest() { diff --git a/controllers/akodeploymentconfig/suite_test.go b/controllers/akodeploymentconfig/suite_test.go index b910713f..82a979b7 100644 --- a/controllers/akodeploymentconfig/suite_test.go +++ b/controllers/akodeploymentconfig/suite_test.go @@ -8,77 +8,16 @@ import ( "testing" . "github.com/onsi/ginkgo" - akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/akodeploymentconfig" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/funcs" testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" - akov1alpha1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - - networkv1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - - adccluster "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/akodeploymentconfig/cluster" - cluster "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/cluster" - ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" ) // suite is used for unit and integration testing this controller. var suite = builder.NewTestSuiteForController( - func(mgr ctrlmgr.Manager) error { - rec := &akodeploymentconfig.AKODeploymentConfigReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("AKODeploymentConfig"), - Scheme: mgr.GetScheme(), - } - builder.FakeAvi = aviclient.NewFakeAviClient() - rec.SetAviClient(builder.FakeAvi) - - adcClusterReconciler := adccluster.NewReconciler(rec.Client, rec.Log, rec.Scheme) - adcClusterReconciler.GetRemoteClient = adccluster.GetFakeRemoteClient - rec.ClusterReconciler = adcClusterReconciler - - if err := rec.SetupWithManager(mgr); err != nil { - return err - } - - // involve the cluster controller as well for the resetting skip-default-adc label test - if err := (&cluster.ClusterReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Cluster"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - return err - } - - return nil - }, - func(scheme *runtime.Scheme) (err error) { - err = networkv1alpha1.AddToScheme(scheme) - if err != nil { - return err - } - err = akoov1alpha1.AddToScheme(scheme) - if err != nil { - return err - } - err = akov1alpha1.AddToScheme(scheme) - if err != nil { - return err - } - err = corev1.AddToScheme(scheme) - if err != nil { - return err - } - err = clusterv1.AddToScheme(scheme) - if err != nil { - return err - } - return nil - }, + funcs.AddAKODeploymentConfigAndClusterControllerToMgrFunc, + funcs.AddAllToSchemeFunc, filepath.Join(testutil.FindModuleDir("sigs.k8s.io/cluster-api"), "config", "crd", "bases"), filepath.Join(testutil.FindModuleDir("github.com/vmware/load-balancer-and-ingress-services-for-kubernetes"), "helm", "ako", "crds"), ) diff --git a/controllers/configmap/suite_test.go b/controllers/configmap/suite_test.go index 5ba4d3dc..8c944466 100644 --- a/controllers/configmap/suite_test.go +++ b/controllers/configmap/suite_test.go @@ -5,21 +5,22 @@ package configmap_test import ( "context" + "testing" + . "github.com/onsi/ginkgo" - akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/configmap" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" - testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "path/filepath" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" - "testing" - corev1 "k8s.io/api/core/v1" + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/configmap" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" ) // suite is used for unit and integration testing this controller. diff --git a/controllers/tests/cluster_for_akodeploymentconfig/default_adc/cluster_for_akodeploymentconfig_intg_test.go b/controllers/tests/cluster_for_akodeploymentconfig/default_adc/cluster_for_akodeploymentconfig_intg_test.go new file mode 100644 index 00000000..3f480e89 --- /dev/null +++ b/controllers/tests/cluster_for_akodeploymentconfig/default_adc/cluster_for_akodeploymentconfig_intg_test.go @@ -0,0 +1,156 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package default_adc_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" +) + +func intgTestClusterDisableAVIWithoutAnyADC() { + var ( + ctx *builder.IntegrationTestContext + staticCluster *clusterv1.Cluster + ) + + BeforeEach(func() { + ctx = suite.NewIntegrationTestContext() + staticCluster = testutil.GetDefaultCluster() + }) + + When("there is no ADC and a cluster is created", func() { + BeforeEach(func() { + testutil.CreateObjects(ctx, staticCluster.DeepCopy()) + }) + AfterEach(func() { + testutil.DeleteObjects(ctx, staticCluster.DeepCopy()) + testutil.EnsureRuntimeObjectMatchExpectation(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, &clusterv1.Cluster{}, testutil.NOTFOUND) + }) + It("shouldn't have 'networking.tkg.tanzu.vmware.com/avi'", func() { + testutil.EnsureClusterAviLabelExists(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, akoov1alpha1.AviClusterLabel, false) + }) + }) +} + +func intgTestClusterCanBeSelectedByADC() { + var ( + ctx *builder.IntegrationTestContext + + staticCluster *clusterv1.Cluster + staticAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig + staticDefaultAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig + + staticManagementNamespace *v1.Namespace + staticManagementCluster *clusterv1.Cluster + staticManagementAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig + ) + + BeforeEach(func() { + ctx = suite.NewIntegrationTestContext() + staticCluster = testutil.GetDefaultCluster() + staticAkoDeploymentConfig = testutil.GetCustomizedADC(testutil.CustomizedADCLabels) + staticDefaultAkoDeploymentConfig = testutil.GetDefaultADC() + + staticManagementNamespace = &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tkg-system"}, + } + staticManagementCluster = testutil.GetManagementCluster() + staticManagementAkoDeploymentConfig = testutil.GetManagementADC() + }) + + When("both default and customized ADC exist", func() { + + BeforeEach(func() { + testutil.CreateObjects(ctx, staticAkoDeploymentConfig.DeepCopy()) + testutil.CreateObjects(ctx, staticDefaultAkoDeploymentConfig.DeepCopy()) + testutil.CreateObjects(ctx, staticCluster.DeepCopy()) + }) + + It("labels the cluster dynamically", func() { + By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-all'", func() { + testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, akoov1alpha1.AviClusterLabel, staticDefaultAkoDeploymentConfig.Name) + }) + + By("add cluster label to use customized adc") + testutil.UpdateObjectLabels(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, testutil.CustomizedADCLabels) + + By("labels with 'networking.tkg.tanzu.vmware.com/avi: ako-deployment-config'", func() { + testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, akoov1alpha1.AviClusterLabel, staticAkoDeploymentConfig.Name) + }) + + By("create another customized ako-deployment-config2") + anotherAkoDeploymentConfig := staticAkoDeploymentConfig.DeepCopy() + anotherAkoDeploymentConfig.Name = "ako-deployment-config-2" + testutil.CreateObjects(ctx, anotherAkoDeploymentConfig.DeepCopy()) + + By("cluster should keep its label, even through another custom ADC matches the name. a.k.a it won't override", func() { + Consistently(func() bool { + obj := &clusterv1.Cluster{} + err := ctx.Client.Get(ctx.Context, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, obj) + if err != nil { + return false + } + val, ok := obj.Labels[akoov1alpha1.AviClusterLabel] + return ok && val == staticAkoDeploymentConfig.Name + }) + }) + + By("unset cluster label to use default adc") + testutil.UpdateObjectLabels(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, map[string]string{}) + + By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-all'", func() { + testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, akoov1alpha1.AviClusterLabel, staticDefaultAkoDeploymentConfig.Name) + }) + }) + }) + + When("management ADC exists", func() { + BeforeEach(func() { + testutil.CreateObjects(ctx, staticManagementNamespace.DeepCopy()) + testutil.CreateObjects(ctx, staticManagementAkoDeploymentConfig.DeepCopy()) + testutil.CreateObjects(ctx, staticManagementCluster.DeepCopy()) + }) + + It("labels the management cluster", func() { + By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-management-cluster'", func() { + testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{ + Name: staticManagementCluster.Name, + Namespace: staticManagementCluster.Namespace, + }, akoov1alpha1.AviClusterLabel, staticManagementAkoDeploymentConfig.Name) + }) + }) + }) +} diff --git a/controllers/tests/cluster_for_akodeploymentconfig/default_adc/suite_test.go b/controllers/tests/cluster_for_akodeploymentconfig/default_adc/suite_test.go new file mode 100644 index 00000000..18d7d6ef --- /dev/null +++ b/controllers/tests/cluster_for_akodeploymentconfig/default_adc/suite_test.go @@ -0,0 +1,38 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package default_adc_test + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/funcs" + testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" +) + +// suite is used for testing the interactions between the controllers +var suite = builder.NewTestSuiteForController( + funcs.AddAKODeploymentConfigAndClusterControllerToMgrFunc, + funcs.AddAllToSchemeFunc, + filepath.Join(testutil.FindModuleDir("sigs.k8s.io/cluster-api"), "config", "crd", "bases"), + filepath.Join(testutil.FindModuleDir("github.com/vmware/load-balancer-and-ingress-services-for-kubernetes"), "helm", "ako", "crds"), +) + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) + +func TestController(t *testing.T) { + suite.Register(t, "AKO Operator Controllers", intgTests, unitTests) +} + +func intgTests() { + Describe("Cluster not selected", intgTestClusterDisableAVIWithoutAnyADC) + Describe("Cluster selected by ADC", intgTestClusterCanBeSelectedByADC) +} + +func unitTests() { +} diff --git a/controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors/cluster_for_akodeploymentconfig_intg_test.go b/controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors/cluster_for_akodeploymentconfig_intg_test.go new file mode 100644 index 00000000..dc894a16 --- /dev/null +++ b/controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors/cluster_for_akodeploymentconfig_intg_test.go @@ -0,0 +1,78 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package default_adc_non_empty_selectors + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" +) + +func intgTestCanSelectedByDefaultADCWithNonEmptySelectors() { + var ( + ctx *builder.IntegrationTestContext + + labels map[string]string + staticCluster *clusterv1.Cluster + staticAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig + staticDefaultAkoDeploymentConfig *akoov1alpha1.AKODeploymentConfig + ) + + BeforeEach(func() { + ctx = suite.NewIntegrationTestContext() + + labels = map[string]string{ + "foo": "bar", + } + staticCluster = testutil.GetDefaultCluster() + staticAkoDeploymentConfig = testutil.GetCustomizedADC(labels) + staticDefaultAkoDeploymentConfig = testutil.GetDefaultADC() + }) + + When("install-ako-for-all has non-empty cluster selector", func() { + BeforeEach(func() { + staticDefaultAkoDeploymentConfig.Spec.ClusterSelector.MatchLabels = labels + testutil.CreateObjects(ctx, staticDefaultAkoDeploymentConfig.DeepCopy()) + }) + + // default adc with non-empty selector -> + // create a cluster selected by default adc -> + // create a new adc also can select cluster -> expect cluster should not change the adc + It("labels the cluster", func() { + By("create a cluster selected by default ADC") + staticCluster.Labels = labels + testutil.CreateObjects(ctx, staticCluster.DeepCopy()) + + By("labels with 'networking.tkg.tanzu.vmware.com/avi: install-ako-for-all'", func() { + testutil.EnsureClusterAviLabelMatchExpectation(ctx, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, akoov1alpha1.AviClusterLabel, staticDefaultAkoDeploymentConfig.Name) + }) + + By("create another ADC with same selector") + testutil.CreateObjects(ctx, staticAkoDeploymentConfig.DeepCopy()) + + By("cluster should keep its labels, since the default ADC cannot be override if it has non-empty selector", func() { + Consistently(func() bool { + obj := &clusterv1.Cluster{} + err := ctx.Client.Get(ctx.Context, client.ObjectKey{ + Name: staticCluster.Name, + Namespace: staticCluster.Namespace, + }, obj) + if err != nil { + return false + } + val, ok := obj.Labels[akoov1alpha1.AviClusterLabel] + return ok && val == staticDefaultAkoDeploymentConfig.Name + }) + }) + }) + }) +} diff --git a/controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors/suite_test.go b/controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors/suite_test.go new file mode 100644 index 00000000..b5514fd3 --- /dev/null +++ b/controllers/tests/cluster_for_akodeploymentconfig/default_adc_non_empty_selectors/suite_test.go @@ -0,0 +1,37 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package default_adc_non_empty_selectors + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/funcs" + testutil "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/util" +) + +// suite is used for testing the interactions between the controllers +var suite = builder.NewTestSuiteForController( + funcs.AddAKODeploymentConfigAndClusterControllerToMgrFunc, + funcs.AddAllToSchemeFunc, + filepath.Join(testutil.FindModuleDir("sigs.k8s.io/cluster-api"), "config", "crd", "bases"), + filepath.Join(testutil.FindModuleDir("github.com/vmware/load-balancer-and-ingress-services-for-kubernetes"), "helm", "ako", "crds"), +) + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) + +func TestControllerWithNonEmptyDefaultADC(t *testing.T) { + suite.Register(t, "AKO Operator Controllers with non-empty selector install-ako-for-all", intgTests, unitTests) +} + +func intgTests() { + Describe("Cluster selected by default ADC with non-empty selectors", intgTestCanSelectedByDefaultADCWithNonEmptySelectors) +} + +func unitTests() { +} diff --git a/pkg/handlers/cluster_for_akodeploymentconfig_handler.go b/pkg/handlers/cluster_for_akodeploymentconfig_handler.go index da9a74a0..1650d4f0 100644 --- a/pkg/handlers/cluster_for_akodeploymentconfig_handler.go +++ b/pkg/handlers/cluster_for_akodeploymentconfig_handler.go @@ -9,13 +9,14 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" - ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" "k8s.io/apimachinery/pkg/types" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" + + ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" ) // AkoDeploymentConfigForCluster returns a handler map function for mapping Cluster diff --git a/pkg/haprovider/haprovider.go b/pkg/haprovider/haprovider.go index 853175f0..9bf96735 100644 --- a/pkg/haprovider/haprovider.go +++ b/pkg/haprovider/haprovider.go @@ -9,19 +9,19 @@ import ( "net" "sync" - ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + ako_operator "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/ako-operator" akov1alpha1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1alpha1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/util/intstr" ) type HAProvider struct { diff --git a/pkg/test/funcs/add_to.go b/pkg/test/funcs/add_to.go new file mode 100644 index 00000000..99d036db --- /dev/null +++ b/pkg/test/funcs/add_to.go @@ -0,0 +1,74 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package funcs + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + networkv1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/akodeploymentconfig" + adccluster "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/akodeploymentconfig/cluster" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/controllers/cluster" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/aviclient" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" + akov1alpha1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1alpha1" +) + +var AddAllToSchemeFunc builder.AddToSchemeFunc = func(scheme *runtime.Scheme) (err error) { + err = networkv1alpha1.AddToScheme(scheme) + if err != nil { + return err + } + err = akoov1alpha1.AddToScheme(scheme) + if err != nil { + return err + } + err = akov1alpha1.AddToScheme(scheme) + if err != nil { + return err + } + err = corev1.AddToScheme(scheme) + if err != nil { + return err + } + err = clusterv1.AddToScheme(scheme) + if err != nil { + return err + } + return nil +} + +var AddAKODeploymentConfigAndClusterControllerToMgrFunc builder.AddToManagerFunc = func(mgr ctrlmgr.Manager) error { + rec := &akodeploymentconfig.AKODeploymentConfigReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("AKODeploymentConfig"), + Scheme: mgr.GetScheme(), + } + builder.FakeAvi = aviclient.NewFakeAviClient() + rec.SetAviClient(builder.FakeAvi) + + adcClusterReconciler := adccluster.NewReconciler(rec.Client, rec.Log, rec.Scheme) + adcClusterReconciler.GetRemoteClient = adccluster.GetFakeRemoteClient + rec.ClusterReconciler = adcClusterReconciler + + if err := rec.SetupWithManager(mgr); err != nil { + return err + } + + // involve the cluster controller as well for the resetting skip-default-adc label test + if err := (&cluster.ClusterReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Cluster"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + return err + } + + return nil +} diff --git a/pkg/test/util/data.go b/pkg/test/util/data.go new file mode 100644 index 00000000..be614ec1 --- /dev/null +++ b/pkg/test/util/data.go @@ -0,0 +1,129 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "math/rand" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" +) + +const ( + // CharSet defines the alphanumeric set for random string generation. + CharSet = "0123456789abcdefghijklmnopqrstuvwxyz" +) + +var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec + +// RandomString returns a random alphanumeric string. +func RandomString(n int) string { + result := make([]byte, n) + for i := range result { + result[i] = CharSet[rnd.Intn(len(CharSet))] + } + return string(result) +} + +// AKODeploymentConfig test data + +var DefaultAkoDeploymentConfigCommonSpec = akoov1alpha1.AKODeploymentConfigSpec{ + DataNetwork: akoov1alpha1.DataNetwork{ + Name: "integration-test-8ed12g", + CIDR: "10.0.0.0/24", + IPPools: []akoov1alpha1.IPPool{ + { + Start: "10.0.0.1", + End: "10.0.0.10", + Type: "V4", + }, + }, + }, + ControlPlaneNetwork: akoov1alpha1.ControlPlaneNetwork{ + Name: "integration-test-8ed12g", + CIDR: "10.1.0.0/24", + }, + ServiceEngineGroup: "ha-test", + AdminCredentialRef: &akoov1alpha1.SecretRef{ + Name: "controller-credentials", + Namespace: "default", + }, + CertificateAuthorityRef: &akoov1alpha1.SecretRef{ + Name: "controller-ca", + Namespace: "default", + }, +} + +// GetManagementADC returns a pointer to install-ako-for-management-cluster ADC for testing +// it always selects cluster labelled as {"cluster-role.tkg.tanzu.vmware.com/management": ""} +func GetManagementADC() *akoov1alpha1.AKODeploymentConfig { + adc := &akoov1alpha1.AKODeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{Name: akoov1alpha1.ManagementClusterAkoDeploymentConfig}, + Spec: DefaultAkoDeploymentConfigCommonSpec, + } + adc.Spec.ClusterSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster-role.tkg.tanzu.vmware.com/management": "", + }, + } + return adc +} + +// GetDefaultADC returns a pointer to install-ako-for-all ADC for testing +// it has empty cluster selector to match all clusters +func GetDefaultADC() *akoov1alpha1.AKODeploymentConfig { + return &akoov1alpha1.AKODeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{Name: akoov1alpha1.WorkloadClusterAkoDeploymentConfig}, + Spec: DefaultAkoDeploymentConfigCommonSpec, + } +} + +var ( + // CustomizedADCLabels defines a common selector used for testing + CustomizedADCLabels = map[string]string{"test": "true"} +) + +// GetCustomizedADC returns a pointer to a customized ADC for testing +// with specified cluster selector labels +func GetCustomizedADC(labels map[string]string) *akoov1alpha1.AKODeploymentConfig { + spec := DefaultAkoDeploymentConfigCommonSpec.DeepCopy() + spec.ClusterSelector = metav1.LabelSelector{ + MatchLabels: labels, + } + return &akoov1alpha1.AKODeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ako-deployment-config", + }, + Spec: *spec, + } +} + +// Cluster test data + +var DefaultCluster = clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "integration-test", + Namespace: "default", + }, + Spec: clusterv1.ClusterSpec{}, +} + +func GetDefaultCluster() *clusterv1.Cluster { + cluster := DefaultCluster.DeepCopy() + cluster.Name = cluster.Name + "-" + RandomString(6) + return cluster +} + +func GetManagementCluster() *clusterv1.Cluster { + cluster := DefaultCluster.DeepCopy() + cluster.Name = cluster.Name + "-" + RandomString(6) + "-mgmt" + cluster.Namespace = "tkg-system" + cluster.Labels = map[string]string{ + "cluster-role.tkg.tanzu.vmware.com/management": "", + } + return cluster +} diff --git a/pkg/test/util/utils.go b/pkg/test/util/utils.go index 2385ede9..cb5dc18c 100644 --- a/pkg/test/util/utils.go +++ b/pkg/test/util/utils.go @@ -5,14 +5,19 @@ package util import ( "encoding/json" + "os/exec" + "time" + . "github.com/onsi/gomega" - "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog" - "os/exec" capi "sigs.k8s.io/cluster-api/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" + + akoov1alpha1 "github.com/vmware-samples/load-balancer-operator-for-kubernetes/api/v1alpha1" + "github.com/vmware-samples/load-balancer-operator-for-kubernetes/pkg/test/builder" ) type ExpectResult int @@ -75,6 +80,9 @@ func ensureRuntimeObjectCreated(ctx *builder.IntegrationTestContext, o client.Ob case *corev1.ConfigMap: obj = o.(*corev1.ConfigMap) EnsureRuntimeObjectMatchExpectation(ctx, client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}, obj, EXIST) + case *akoov1alpha1.AKODeploymentConfig: + obj = o.(*akoov1alpha1.AKODeploymentConfig) + EnsureRuntimeObjectMatchExpectation(ctx, client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}, obj, EXIST) default: klog.Fatal("Unknown type object") } @@ -91,5 +99,47 @@ func EnsureRuntimeObjectMatchExpectation(ctx *builder.IntegrationTestContext, ob } } return res == expectResult + }, 30*time.Second).Should(BeTrue()) +} + +func EnsureClusterAviLabelExists(ctx *builder.IntegrationTestContext, key client.ObjectKey, label string, exists bool) { + Eventually(func() bool { + obj := &clusterv1.Cluster{} + err := ctx.Client.Get(ctx.Context, key, obj) + if err != nil { + return false + } + _, ok := obj.Labels[label] + return ok == exists }).Should(BeTrue()) } + +func EnsureClusterAviLabelMatchExpectation(ctx *builder.IntegrationTestContext, key client.ObjectKey, label, expectVal string) { + Eventually(func() bool { + obj := &clusterv1.Cluster{} + err := ctx.Client.Get(ctx.Context, key, obj) + if err != nil { + return false + } + val, ok := obj.Labels[label] + return ok && val == expectVal + }).Should(BeTrue()) +} + +func UpdateObjectLabels(ctx *builder.IntegrationTestContext, key client.ObjectKey, labels map[string]string) { + Eventually(func() error { + var cluster = new(clusterv1.Cluster) + + if err := ctx.Client.Get(ctx, client.ObjectKey{ + Name: key.Name, + Namespace: key.Namespace, + }, cluster); err != nil { + return err + } + cluster.Labels = labels + if err := ctx.Client.Update(ctx, cluster, &client.UpdateOptions{}); err != nil { + return err + } + return nil + }).Should(Succeed()) +}