From e86c6c75cbdfe06350ac7499ccd8e1aa31d131fc Mon Sep 17 00:00:00 2001 From: mgianluc Date: Thu, 10 Oct 2024 08:15:01 +0200 Subject: [PATCH] Cluster's rest config cache In large cluster, number of Secrets and ConfigMaps might be high. Do not cache those. In order to avoid fetching the Secret with cluster Kubeconfig, create a in memory cluster cache with restConfig. This feature is currently disabled. To enable set `disable-secret-caching=true` --- cmd/main.go | 17 ++ controllers/clustercache/cluster_cache.go | 244 ++++++++++++++++++ .../clustercache/cluster_cache_test.go | 144 +++++++++++ .../clustercache/clustercache_suite_test.go | 155 +++++++++++ controllers/clustercache/export_test.go | 44 ++++ controllers/clusterprofile_controller_test.go | 2 +- controllers/clustersummary_controller.go | 1 - controllers/clustersummary_transformations.go | 12 +- controllers/controllers_suite_test.go | 1 + controllers/handlers_helm.go | 4 +- controllers/handlers_kustomize.go | 4 +- controllers/handlers_resources.go | 4 +- controllers/handlers_utils.go | 8 +- controllers/resourcesummary.go | 4 +- controllers/resourcesummary_test.go | 2 +- controllers/suite_helpers_test.go | 4 +- go.mod | 10 +- go.sum | 20 +- 18 files changed, 655 insertions(+), 25 deletions(-) create mode 100644 controllers/clustercache/cluster_cache.go create mode 100644 controllers/clustercache/cluster_cache_test.go create mode 100644 controllers/clustercache/clustercache_suite_test.go create mode 100644 controllers/clustercache/export_test.go diff --git a/cmd/main.go b/cmd/main.go index a4d198fe..a8cf5542 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -83,6 +83,7 @@ var ( healthAddr string profilerAddress string driftDetectionConfigMap string + disableCaching bool ) const ( @@ -113,6 +114,14 @@ func main() { reportMode = controllers.ReportMode(tmpReportMode) + disableFor := []client.Object{} + if disableCaching { + disableFor = []client.Object{ + &corev1.Secret{}, + &corev1.ConfigMap{}, + } + } + ctrl.SetLogger(klog.Background()) ctrlOptions := ctrl.Options{ Scheme: scheme, @@ -125,6 +134,11 @@ func main() { Cache: cache.Options{ SyncPeriod: &syncPeriod, }, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: disableFor, + }, + }, PprofBindAddress: profilerAddress, } @@ -171,6 +185,9 @@ func initFlags(fs *pflag.FlagSet) { fs.BoolVar(&agentInMgmtCluster, "agent-in-mgmt-cluster", false, "When set, indicates drift-detection-manager needs to be started in the management cluster") + fs.BoolVar(&disableCaching, "disable-secret-caching", false, + "When set, disable caching secrets and configmaps") + fs.StringVar(&diagnosticsAddress, "diagnostics-address", ":8443", "The address the diagnostics endpoint binds to. Per default metrics are served via https and with"+ "authentication/authorization. To serve via http and without authentication/authorization set --insecure-diagnostics."+ diff --git a/controllers/clustercache/cluster_cache.go b/controllers/clustercache/cluster_cache.go new file mode 100644 index 00000000..bb2636b3 --- /dev/null +++ b/controllers/clustercache/cluster_cache.go @@ -0,0 +1,244 @@ +/* +Copyright 2024. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustercache + +import ( + "context" + "fmt" + "sync" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" + "k8s.io/klog/v2/textlogger" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" + + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clusterproxy" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" + libsveltosset "github.com/projectsveltos/libsveltos/lib/set" +) + +var ( + managerInstance *clusterCache + lock = &sync.Mutex{} +) + +type clusterCache struct { + rwMux sync.RWMutex + // Keeps cache of rest.Config for existing clusters + configs map[corev1.ObjectReference]*rest.Config + + // key: cluster, value: Secret with kubeconfig + clusters map[corev1.ObjectReference]*corev1.ObjectReference + + // key: secret, value: set of clusters + // A secret can potentially contain kubeconfig for one or more clusters + secrets map[corev1.ObjectReference]*libsveltosset.Set +} + +// GetManager return manager instance +func GetManager() *clusterCache { + if managerInstance == nil { + lock.Lock() + defer lock.Unlock() + if managerInstance == nil { + managerInstance = &clusterCache{ + configs: make(map[corev1.ObjectReference]*rest.Config), + clusters: make(map[corev1.ObjectReference]*corev1.ObjectReference), + secrets: make(map[corev1.ObjectReference]*libsveltosset.Set), + rwMux: sync.RWMutex{}, + } + } + } + + return managerInstance +} + +// RemoveCluster removes restConfig cached data for the cluster +func (m *clusterCache) RemoveCluster(clusterNamespace, clusterName string, + clusterType libsveltosv1beta1.ClusterType) { + + cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) + + m.rwMux.Lock() + defer m.rwMux.Unlock() + + // Remove from cache the restConfig for this cluster + delete(m.configs, *cluster) + + if sec, ok := m.clusters[*cluster]; ok { + m.updateSecretMap(sec, cluster) + } + + // Do not track this cluster anymore + delete(m.clusters, *cluster) +} + +// RemoveSecret removes any in-memory data related to secret +func (m *clusterCache) RemoveSecret(sec *corev1.ObjectReference) { + m.rwMux.Lock() + defer m.rwMux.Unlock() + + v, ok := m.secrets[*sec] + if !ok { + return + } + + clusters := v.Items() + for i := range clusters { + delete(m.configs, clusters[i]) + delete(m.clusters, clusters[i]) + } +} + +// GetKubernetesRestConfig returns managed cluster restConfig. +// If result is cached, it will be returned immediately. Otherwise it will be built +// by fetching the Secret containing the cluster kubeconfig. +// Admins restConfig are never cached. +func (m *clusterCache) GetKubernetesRestConfig(ctx context.Context, mgmtClient client.Client, + clusterNamespace, clusterName, adminNamespace, adminName string, + clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) (*rest.Config, error) { + + if adminNamespace != "" || adminName != "" { + // cluster configs for admins are not cached + return clusterproxy.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, + adminNamespace, adminName, clusterType, logger) + } + + m.rwMux.Lock() + defer m.rwMux.Unlock() + + cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) + + config, ok := m.configs[*cluster] + if ok { + logger.V(logs.LogInfo).Info("remote restConfig cache hit") + return config, nil + } + + logger.V(logs.LogDebug).Info("remote restConfig cache miss") + remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, + adminNamespace, adminName, clusterType, logger) + if err != nil { + return nil, err + } + + secretInfo, err := getSecretObjectReference(ctx, mgmtClient, clusterNamespace, clusterName, clusterType) + if err == nil { + // Either all internal structures are updated or none is + m.configs[*cluster] = remoteRestConfig + m.clusters[*cluster] = secretInfo + v, ok := m.secrets[*secretInfo] + if !ok { + v = &libsveltosset.Set{} + } + v.Insert(cluster) + m.secrets[*secretInfo] = v + } + + return remoteRestConfig, nil +} + +func (m *clusterCache) GetKubernetesClient(ctx context.Context, mgmtClient client.Client, + clusterNamespace, clusterName, adminNamespace, adminName string, + clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) (client.Client, error) { + + if adminNamespace != "" || adminName != "" { + // cluster configs for admins are not cached + return clusterproxy.GetKubernetesClient(ctx, mgmtClient, clusterNamespace, clusterName, + adminNamespace, adminName, clusterType, logger) + } + + config, err := m.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, + adminNamespace, adminName, clusterType, logger) + if err != nil { + return nil, err + } + logger.V(logs.LogVerbose).Info("return new client") + return client.New(config, client.Options{Scheme: mgmtClient.Scheme()}) +} + +func (m *clusterCache) StoreRestConfig(clusterNamespace, clusterName string, + clusterType libsveltosv1beta1.ClusterType, config *rest.Config) { + + cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) + + m.rwMux.Lock() + defer m.rwMux.Unlock() + + m.configs[*cluster] = config +} + +func (m *clusterCache) updateSecretMap(sec, cluster *corev1.ObjectReference) { + set, ok := m.secrets[*sec] + if ok { + set.Erase(cluster) + if set.Len() == 0 { + delete(m.secrets, *sec) + } + } +} + +func getClusterObjectReference(clusterNamespace, clusterName string, + clusterType libsveltosv1beta1.ClusterType) *corev1.ObjectReference { + + cluster := &corev1.ObjectReference{ + Namespace: clusterNamespace, + Name: clusterName, + Kind: clusterv1.ClusterKind, + APIVersion: clusterv1.GroupVersion.String(), + } + if clusterType == libsveltosv1beta1.ClusterTypeSveltos { + cluster.Kind = libsveltosv1beta1.SveltosClusterKind + cluster.APIVersion = libsveltosv1beta1.GroupVersion.String() + } + + return cluster +} + +func getSecretObjectReference(ctx context.Context, mgmtClient client.Client, + clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType) (*corev1.ObjectReference, error) { + + secretKind := "Secret" + if clusterType == libsveltosv1beta1.ClusterTypeCapi { + return &corev1.ObjectReference{ + Namespace: clusterNamespace, + Name: getClusterAPISecretName(clusterName), + Kind: secretKind, + APIVersion: corev1.SchemeGroupVersion.String(), + }, nil + } + + logger := textlogger.NewLogger(textlogger.NewConfig()) + secretName, err := clusterproxy.GetSveltosSecretName(ctx, logger, mgmtClient, clusterNamespace, clusterName) + if err != nil { + return nil, err + } + return &corev1.ObjectReference{ + Namespace: clusterNamespace, + Name: secretName, + Kind: secretKind, + APIVersion: corev1.SchemeGroupVersion.String(), + }, nil +} + +func getClusterAPISecretName(clusterName string) string { + return fmt.Sprintf("%s-%s", clusterName, secret.Kubeconfig) +} diff --git a/controllers/clustercache/cluster_cache_test.go b/controllers/clustercache/cluster_cache_test.go new file mode 100644 index 00000000..9da354be --- /dev/null +++ b/controllers/clustercache/cluster_cache_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2024. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustercache_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2/textlogger" + + "github.com/projectsveltos/addon-controller/controllers/clustercache" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" +) + +const ( + sveltosKubeconfigPostfix = "-sveltos-kubeconfig" +) + +var _ = Describe("Clustercache", func() { + var logger logr.Logger + var cluster *libsveltosv1beta1.SveltosCluster + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig()) + cluster = &libsveltosv1beta1.SveltosCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cache" + randomString(), + Namespace: "cache" + randomString(), + }, + } + }) + + It("GetKubernetesRestConfig stores in memory first time. RemoveCluster removes any entry associated to cluster", + func() { + secret := createClusterResources(cluster) + + cacheMgr := clustercache.GetManager() + _, err := cacheMgr.GetKubernetesRestConfig(context.TODO(), testEnv.Client, cluster.Namespace, + cluster.Name, "", "", libsveltosv1beta1.ClusterTypeSveltos, logger) + Expect(err).To(BeNil()) + + clusterObj := &corev1.ObjectReference{ + Namespace: cluster.Namespace, + Name: cluster.Name, + Kind: libsveltosv1beta1.SveltosClusterKind, + APIVersion: libsveltosv1beta1.GroupVersion.String(), + } + Expect(cacheMgr.GetConfigFromMap(clusterObj)).ToNot(BeNil()) + + storedSecret := cacheMgr.GetSecretForCluster(clusterObj) + Expect(storedSecret).ToNot(BeNil()) + Expect(storedSecret.Namespace).To(Equal(secret.Namespace)) + Expect(storedSecret.Name).To(Equal(secret.Name)) + + secretObj := &corev1.ObjectReference{ + Namespace: secret.Namespace, + Name: secret.Name, + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + } + storedCluster := cacheMgr.GetClusterFromSecret(secretObj) + Expect(storedCluster).ToNot(BeNil()) + Expect(storedCluster.Namespace).To(Equal(cluster.Namespace)) + Expect(storedCluster.Name).To(Equal(cluster.Name)) + + cacheMgr.RemoveCluster(cluster.Namespace, clusterObj.Name, libsveltosv1beta1.ClusterTypeSveltos) + Expect(cacheMgr.GetConfigFromMap(clusterObj)).To(BeNil()) + Expect(cacheMgr.GetSecretForCluster(clusterObj)).To(BeNil()) + Expect(cacheMgr.GetClusterFromSecret(secretObj)).To(BeNil()) + + }) + + It("RemoveSecret removes entries for all clusters using the modified secret", func() { + secret := createClusterResources(cluster) + + cacheMgr := clustercache.GetManager() + _, err := cacheMgr.GetKubernetesRestConfig(context.TODO(), testEnv.Client, cluster.Namespace, + cluster.Name, "", "", libsveltosv1beta1.ClusterTypeSveltos, logger) + Expect(err).To(BeNil()) + + clusterObj := &corev1.ObjectReference{ + Namespace: cluster.Namespace, + Name: cluster.Name, + Kind: libsveltosv1beta1.SveltosClusterKind, + APIVersion: libsveltosv1beta1.GroupVersion.String(), + } + Expect(cacheMgr.GetConfigFromMap(clusterObj)).ToNot(BeNil()) + + secretObj := &corev1.ObjectReference{ + Namespace: secret.Namespace, + Name: secret.Name, + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + } + cacheMgr.RemoveSecret(secretObj) + Expect(cacheMgr.GetConfigFromMap(clusterObj)).To(BeNil()) + }) +}) + +func createClusterResources(cluster *libsveltosv1beta1.SveltosCluster) *corev1.Secret { + By("Create the cluster's namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.Namespace, + }, + } + + By("Create the secret with cluster kubeconfig") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cluster.Name + sveltosKubeconfigPostfix, + }, + Data: map[string][]byte{ + "value": testEnv.Kubeconfig, + }, + } + + Expect(testEnv.Create(context.TODO(), ns)).To(Succeed()) + Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) + Expect(testEnv.Create(context.TODO(), secret)).To(Succeed()) + + Expect(waitForObject(context.TODO(), testEnv.Client, secret)).To(Succeed()) + return secret +} diff --git a/controllers/clustercache/clustercache_suite_test.go b/controllers/clustercache/clustercache_suite_test.go new file mode 100644 index 00000000..252fb90f --- /dev/null +++ b/controllers/clustercache/clustercache_suite_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2024. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustercache_test + +import ( + "context" + "fmt" + "path" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "sigs.k8s.io/cluster-api/util" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/api/v1beta1/index" + "github.com/projectsveltos/addon-controller/internal/test/helpers" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + libsveltoscrd "github.com/projectsveltos/libsveltos/lib/crd" + "github.com/projectsveltos/libsveltos/lib/utils" +) + +var ( + testEnv *helpers.TestEnvironment + cancel context.CancelFunc + ctx context.Context + scheme *runtime.Scheme +) + +var ( + cacheSyncBackoff = wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 1.5, + Steps: 8, + Jitter: 0.4, + } +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controllers Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctrl.SetLogger(klog.Background()) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + scheme, err = setupScheme() + Expect(err).To(BeNil()) + + testEnvConfig := helpers.NewTestEnvironmentConfiguration([]string{ + path.Join("config", "crd", "bases"), + }, scheme) + testEnv, err = testEnvConfig.Build(scheme) + if err != nil { + panic(err) + } + + Expect(index.AddDefaultIndexes(ctx, testEnv.Manager)).To(Succeed()) + + go func() { + By("Starting the manager") + err = testEnv.StartManager(ctx) + if err != nil { + panic(fmt.Sprintf("Failed to start the envtest manager: %v", err)) + } + }() + + var sveltosCRD *unstructured.Unstructured + sveltosCRD, err = utils.GetUnstructured(libsveltoscrd.GetSveltosClusterCRDYAML()) + Expect(err).To(BeNil()) + Expect(testEnv.Create(context.TODO(), sveltosCRD)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv, sveltosCRD)).To(Succeed()) + + if synced := testEnv.GetCache().WaitForCacheSync(ctx); !synced { + time.Sleep(time.Second) + } +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).ToNot(HaveOccurred()) +}) + +func setupScheme() (*runtime.Scheme, error) { + s := runtime.NewScheme() + if err := configv1beta1.AddToScheme(s); err != nil { + return nil, err + } + if err := clientgoscheme.AddToScheme(s); err != nil { + return nil, err + } + if err := libsveltosv1beta1.AddToScheme(s); err != nil { + return nil, err + } + + return s, nil +} + +func randomString() string { + const length = 10 + return "a-" + util.RandomString(length) +} + +// waitForObject waits for the cache to be updated helps in preventing test flakes due to the cache sync delays. +func waitForObject(ctx context.Context, c client.Client, obj client.Object) error { + // Makes sure the cache is updated with the new object + objCopy := obj.DeepCopyObject().(client.Object) + key := client.ObjectKeyFromObject(obj) + if err := wait.ExponentialBackoff( + cacheSyncBackoff, + func() (done bool, err error) { + if err := c.Get(ctx, key, objCopy); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil + }); err != nil { + return errors.Wrapf(err, "object %s, %s is not being added to the testenv client cache", obj.GetObjectKind().GroupVersionKind().String(), key) + } + return nil +} diff --git a/controllers/clustercache/export_test.go b/controllers/clustercache/export_test.go new file mode 100644 index 00000000..a88863a7 --- /dev/null +++ b/controllers/clustercache/export_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2024. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustercache + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" +) + +func (m *clusterCache) GetConfigFromMap(cluster *corev1.ObjectReference) *rest.Config { + return m.configs[*cluster] +} + +func (m *clusterCache) GetSecretForCluster(cluster *corev1.ObjectReference) *corev1.ObjectReference { + return m.clusters[*cluster] +} + +func (m *clusterCache) GetClusterFromSecret(secret *corev1.ObjectReference) *corev1.ObjectReference { + set := m.secrets[*secret] + if set == nil { + return nil + } + + if set.Len() == 0 { + return nil + } + + items := set.Items() + return &items[0] +} diff --git a/controllers/clusterprofile_controller_test.go b/controllers/clusterprofile_controller_test.go index 23ae456b..eb2106f5 100644 --- a/controllers/clusterprofile_controller_test.go +++ b/controllers/clusterprofile_controller_test.go @@ -21,10 +21,10 @@ import ( "sync" "time" - "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/controllers/clustersummary_controller.go b/controllers/clustersummary_controller.go index f914be06..763a81e6 100644 --- a/controllers/clustersummary_controller.go +++ b/controllers/clustersummary_controller.go @@ -1006,7 +1006,6 @@ func (r *ClusterSummaryReconciler) isReady(ctx context.Context, } isClusterReady, err := clusterproxy.IsClusterReadyToBeConfigured(ctx, r.Client, clusterRef, logger) - if err != nil { if apierrors.IsNotFound(err) { return false, nil diff --git a/controllers/clustersummary_transformations.go b/controllers/clustersummary_transformations.go index 172e93a5..9b121115 100644 --- a/controllers/clustersummary_transformations.go +++ b/controllers/clustersummary_transformations.go @@ -29,7 +29,9 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1" sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2" + "github.com/projectsveltos/addon-controller/controllers/clustercache" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clusterproxy" logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) @@ -154,6 +156,8 @@ func (r *ClusterSummaryReconciler) requeueClusterSummaryForReference( Namespace: o.GetNamespace(), Name: o.GetName(), } + cacheMgr := clustercache.GetManager() + cacheMgr.RemoveSecret(&key) default: key = corev1.ObjectReference{ APIVersion: o.GetObjectKind().GroupVersionKind().GroupVersion().String(), @@ -214,10 +218,16 @@ func (r *ClusterSummaryReconciler) requeueClusterSummaryForACluster( logger.V(logs.LogDebug).Info("reacting to Cluster change") + clusterInfo := getKeyFromObject(r.Scheme, cluster) + if !cluster.GetDeletionTimestamp().IsZero() { + cacheMgr := clustercache.GetManager() + cacheMgr.RemoveCluster(cluster.GetNamespace(), cluster.GetName(), + clusterproxy.GetClusterType(clusterInfo)) + } + r.PolicyMux.Lock() defer r.PolicyMux.Unlock() - clusterInfo := getKeyFromObject(r.Scheme, cluster) // Get all ClusterSummaries for this cluster and reconcile those requests := make([]ctrl.Request, r.getClusterMapForEntry(clusterInfo).Len()) consumers := r.getClusterMapForEntry(clusterInfo).Items() diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index f86d606d..a6b109fa 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index 94017b23..a49781ec 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -62,6 +62,7 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers/chartmanager" + "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/pkg/scope" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/clusterproxy" @@ -195,7 +196,8 @@ func deployHelmCharts(ctx context.Context, c client.Client, return err } - remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, + cacheMgr := clustercache.GetManager() + remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, adminNamespace, adminName, clusterSummary.Spec.ClusterType, logger) if err != nil { return err diff --git a/controllers/handlers_kustomize.go b/controllers/handlers_kustomize.go index 6095678b..95b85042 100644 --- a/controllers/handlers_kustomize.go +++ b/controllers/handlers_kustomize.go @@ -47,6 +47,7 @@ import ( "sigs.k8s.io/kustomize/kyaml/filesys" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/pkg/scope" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/clusterproxy" @@ -230,7 +231,8 @@ func undeployKustomizeRefs(ctx context.Context, c client.Client, return err } - remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, + cacheMgr := clustercache.GetManager() + remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, adminNamespace, adminName, clusterSummary.Spec.ClusterType, logger) if err != nil { return err diff --git a/controllers/handlers_resources.go b/controllers/handlers_resources.go index db4c7c8a..43ec9441 100644 --- a/controllers/handlers_resources.go +++ b/controllers/handlers_resources.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/pkg/scope" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/clusterproxy" @@ -284,7 +285,8 @@ func undeployResources(ctx context.Context, c client.Client, return err } - remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, + cacheMgr := clustercache.GetManager() + remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, adminNamespace, adminName, clusterSummary.Spec.ClusterType, logger) if err != nil { return err diff --git a/controllers/handlers_utils.go b/controllers/handlers_utils.go index 9eb53133..a1b40151 100644 --- a/controllers/handlers_utils.go +++ b/controllers/handlers_utils.go @@ -51,6 +51,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers/clustercache" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" @@ -1426,6 +1427,8 @@ func getConfigMap(ctx context.Context, c client.Client, configmapName types.Name return nil, err } + addTypeInformationToObject(c.Scheme(), configMap) + return configMap, nil } @@ -1444,6 +1447,8 @@ func getSecret(ctx context.Context, c client.Client, secretName types.Namespaced return nil, libsveltosv1beta1.ErrSecretTypeNotSupported } + addTypeInformationToObject(c.Scheme(), secret) + return secret, nil } @@ -1561,7 +1566,8 @@ func getRestConfig(ctx context.Context, c client.Client, clusterSummary *configv WithValues("clusterSummary", clusterSummary.Name).WithValues("admin", fmt.Sprintf("%s/%s", adminNamespace, adminName)) logger.V(logs.LogDebug).Info("get remote restConfig") - remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, + cacheMgr := clustercache.GetManager() + remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, adminNamespace, adminName, clusterSummary.Spec.ClusterType, logger) if err != nil { return nil, logger, err diff --git a/controllers/resourcesummary.go b/controllers/resourcesummary.go index 21af4000..b4a26d96 100644 --- a/controllers/resourcesummary.go +++ b/controllers/resourcesummary.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers/clustercache" driftdetection "github.com/projectsveltos/addon-controller/pkg/drift-detection" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/clusterproxy" @@ -72,7 +73,8 @@ func deployDriftDetectionManagerInCluster(ctx context.Context, c client.Client, } // Sveltos resources are deployed using cluster-admin role. - remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, c, clusterNamespace, + cacheMgr := clustercache.GetManager() + remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, c, clusterNamespace, clusterName, "", "", clusterType, logger) if err != nil { logger.V(logs.LogInfo).Error(err, "failed to get cluster rest config") diff --git a/controllers/resourcesummary_test.go b/controllers/resourcesummary_test.go index 477db5b9..4fccc630 100644 --- a/controllers/resourcesummary_test.go +++ b/controllers/resourcesummary_test.go @@ -323,7 +323,7 @@ func prepareCluster() *clusterv1.Cluster { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, - Name: cluster.Name + sveltosKubeconfigPostfix, + Name: cluster.Name + kubeconfigPostfix, }, Data: map[string][]byte{ "value": testEnv.Kubeconfig, diff --git a/controllers/suite_helpers_test.go b/controllers/suite_helpers_test.go index a54b0946..f98d8288 100644 --- a/controllers/suite_helpers_test.go +++ b/controllers/suite_helpers_test.go @@ -42,7 +42,7 @@ import ( ) const ( - sveltosKubeconfigPostfix = "-kubeconfig" + kubeconfigPostfix = "-kubeconfig" ) // addOwnerReference adds owner as OwnerReference of obj @@ -242,7 +242,7 @@ func prepareForDeployment(clusterProfile *configv1beta1.ClusterProfile, secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Namespace: clusterSummary.Spec.ClusterNamespace, - Name: clusterSummary.Spec.ClusterName + sveltosKubeconfigPostfix, + Name: clusterSummary.Spec.ClusterName + kubeconfigPostfix, }, Data: map[string][]byte{ "value": testEnv.Kubeconfig, diff --git a/go.mod b/go.mod index daa1fb06..df7b2fe6 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,8 @@ require ( sigs.k8s.io/kustomize/kyaml v0.18.1 ) +replace github.com/projectsveltos/libsveltos => github.com/gianlucam76/libsveltos v0.0.0-20241009142654-095e04b3cb80 + require ( dario.cat/mergo v1.0.1 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect @@ -92,7 +94,7 @@ require ( github.com/google/cel-go v0.21.0 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d // indirect + github.com/google/pprof v0.0.0-20241008150032-332c0e1a4a34 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect @@ -169,19 +171,19 @@ require ( golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect - golang.org/x/time v0.6.0 // indirect + golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.26.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 // indirect google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.31.1 // indirect k8s.io/cluster-bootstrap v0.31.1 // indirect - k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect + k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094 // indirect k8s.io/kubectl v0.31.1 // indirect oras.land/oras-go v1.2.6 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect diff --git a/go.sum b/go.sum index 10c7d2dc..195301da 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gdexlab/go-render v1.0.1 h1:rxqB3vo5s4n1kF0ySmoNeSPRYkEsyHgln4jFIQY7v0U= github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4VgFTmJX5JzM= +github.com/gianlucam76/libsveltos v0.0.0-20241009142654-095e04b3cb80 h1:SyVd8cvrnMKUEV5HQ/bEyKVkeoJzettDjpc0m2avtyg= +github.com/gianlucam76/libsveltos v0.0.0-20241009142654-095e04b3cb80/go.mod h1:xnuJwvKzkyOu8MW1a+0IL+diAcjqRMrWG5A40cRC/hE= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= @@ -187,8 +189,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d h1:Jaz2JzpQaQXyET0AjLBXShrthbpqMkhGiEfkcQAiAUs= -github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241008150032-332c0e1a4a34 h1:4iExbL0TFWhkSCZx6nfKwjM+CbnBySx18KssYmdL1fc= +github.com/google/pprof v0.0.0-20241008150032-332c0e1a4a34/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -324,8 +326,6 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v0.39.0 h1:wSxFKHx1L9gA9g7auKdpCdyCug5vDd4pdkKt/bDZBWQ= -github.com/projectsveltos/libsveltos v0.39.0/go.mod h1:e/E3vkU4Ph1ZcyJ+FaIwTJQOVn2XH0Y/7pLDHhMnWPY= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= @@ -498,8 +498,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -518,8 +518,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -557,8 +557,8 @@ k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= -k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094 h1:MErs8YA0abvOqJ8gIupA1Tz6PKXYUw34XsGlA7uSL1k= +k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094/go.mod h1:7ioBJr1A6igWjsR2fxq2EZ0mlMwYLejazSIc2bzMp2U= k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI=