From 11bb964631e1043c7f64c5013d47fe9c69bc8fe5 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 15 Jun 2023 18:27:26 +0000 Subject: [PATCH 01/58] use cm Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller.go | 57 +++++++--------------- pkg/syncutil/cachemanager/cachemanager.go | 12 +++++ 2 files changed, 29 insertions(+), 40 deletions(-) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 406ab6f5b02..d2dbaee93fc 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -26,7 +26,6 @@ import ( syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" - "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" syncutil "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" @@ -114,7 +113,7 @@ func (a *Adder) InjectWatchSet(watchSet *watch.Set) { func newReconciler(mgr manager.Manager, opa syncutil.OpaDataClient, wm *watch.Manager, cs *watch.ControllerSwitch, tracker *readiness.Tracker, processExcluder *process.Excluder, events <-chan event.GenericEvent, watchSet *watch.Set, regEvents chan<- event.GenericEvent) (*ReconcileConfig, error) { filteredOpa := syncutil.NewFilteredOpaDataClient(opa, watchSet) syncMetricsCache := syncutil.NewMetricsCache() - cm := cm.NewCacheManager(opa, syncMetricsCache, tracker, processExcluder) + cm := cm.NewCacheManager(filteredOpa, syncMetricsCache, tracker, processExcluder) syncAdder := syncc.Adder{ Events: events, @@ -136,17 +135,16 @@ func newReconciler(mgr manager.Manager, opa syncutil.OpaDataClient, wm *watch.Ma return nil, err } return &ReconcileConfig{ - reader: mgr.GetCache(), - writer: mgr.GetClient(), - statusClient: mgr.GetClient(), - scheme: mgr.GetScheme(), - opa: filteredOpa, - cs: cs, - watcher: w, - watched: watchSet, - syncMetricsCache: syncMetricsCache, - tracker: tracker, - processExcluder: processExcluder, + reader: mgr.GetCache(), + writer: mgr.GetClient(), + statusClient: mgr.GetClient(), + scheme: mgr.GetScheme(), + cs: cs, + watcher: w, + watched: watchSet, + cacheManager: cm, + tracker: tracker, + processExcluder: processExcluder, }, nil } @@ -175,11 +173,10 @@ type ReconcileConfig struct { writer client.Writer statusClient client.StatusClient - scheme *runtime.Scheme - opa syncutil.OpaDataClient - syncMetricsCache *syncutil.MetricsCache - cs *watch.ControllerSwitch - watcher *watch.Registrar + scheme *runtime.Scheme + cacheManager *cm.CacheManager + cs *watch.ControllerSwitch + watcher *watch.Registrar watched *watch.Set @@ -352,32 +349,12 @@ func (r *ReconcileConfig) replayData(ctx context.Context) error { return fmt.Errorf("replaying data for %+v: %w", gvk, err) } - defer r.syncMetricsCache.ReportSync() + defer r.cacheManager.ReportSyncMetrics() for i := range u.Items { - syncKey := syncutil.GetKeyForSyncMetrics(u.Items[i].GetNamespace(), u.Items[i].GetName()) - - isExcludedNamespace, err := r.skipExcludedNamespace(&u.Items[i]) - if err != nil { - log.Error(err, "error while excluding namespaces") - } - - if isExcludedNamespace { - continue - } - - if _, err := r.opa.AddData(ctx, &u.Items[i]); err != nil { - r.syncMetricsCache.AddObject(syncKey, syncutil.Tags{ - Kind: u.Items[i].GetKind(), - Status: metrics.ErrorStatus, - }) + if err := r.cacheManager.AddObject(ctx, &u.Items[i]); err != nil { return fmt.Errorf("adding data for %+v: %w", gvk, err) } - - r.syncMetricsCache.AddObject(syncKey, syncutil.Tags{ - Kind: u.Items[i].GetKind(), - Status: metrics.ActiveStatus, - }) } r.needsReplay.Remove(gvk) } diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index 73e723860b5..7780443167a 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -77,6 +77,18 @@ func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured. return nil } +func (c *CacheManager) WipeData(ctx context.Context, target interface{}) error { + if _, err := c.opa.RemoveData(ctx, target); err != nil { + return err + } + + // reset sync cache before sending the metric + c.syncMetricsCache.ResetCache() + c.syncMetricsCache.ReportSync() + + return nil +} + func (c *CacheManager) ReportSyncMetrics() { c.syncMetricsCache.ReportSync() } From 5314889b86bdfbbf1609627aeb0fa11e036c2f25 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 15 Jun 2023 18:37:08 +0000 Subject: [PATCH 02/58] replace process excluder in cm Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller.go | 2 +- pkg/syncutil/cachemanager/cachemanager.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index d2dbaee93fc..efcd8195948 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -292,7 +292,7 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.watched.Replace(newSyncOnly, func() { // swapping with the new excluder - r.processExcluder.Replace(newExcluder) + r.cacheManager.ReplaceExcluder(newExcluder) // *Note the following steps are not transactional with respect to admission control* diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index 7780443167a..1ee5f6fe639 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -92,3 +92,7 @@ func (c *CacheManager) WipeData(ctx context.Context, target interface{}) error { func (c *CacheManager) ReportSyncMetrics() { c.syncMetricsCache.ReportSync() } + +func (c *CacheManager) ReplaceExcluder(p *process.Excluder) { + c.processExcluder.Replace(p) +} From c38296106af6ff8ee5b44534e797045278f0a533 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 15 Jun 2023 21:14:52 +0000 Subject: [PATCH 03/58] nit: dont pass target Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/cachemanager/cachemanager.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index 1ee5f6fe639..0de3307f3c7 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -8,6 +8,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -77,8 +78,8 @@ func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured. return nil } -func (c *CacheManager) WipeData(ctx context.Context, target interface{}) error { - if _, err := c.opa.RemoveData(ctx, target); err != nil { +func (c *CacheManager) WipeData(ctx context.Context) error { + if _, err := c.opa.RemoveData(ctx, target.WipeData()); err != nil { return err } From ba564518ff5e8cb5e4e1f2688ea1d6ffb45a5b99 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:15:46 +0000 Subject: [PATCH 04/58] use cm.WipeData Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index efcd8195948..e25bb4b3ea8 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -30,7 +30,6 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" syncutil "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" - "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -318,15 +317,9 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque func (r *ReconcileConfig) wipeCacheIfNeeded(ctx context.Context) error { if r.needsWipe { - if _, err := r.opa.RemoveData(ctx, target.WipeData()); err != nil { + if err := r.cacheManager.WipeData(ctx); err != nil { return err } - - // reset sync cache before sending the metric - r.syncMetricsCache.ResetCache() - r.syncMetricsCache.ReportSync() - - r.needsWipe = false } return nil } From b74641141b0c41f49b2a109e7f2ba218b84f5960 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:07:12 +0000 Subject: [PATCH 05/58] inject cm Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller.go | 14 +++++------ .../config/config_controller_test.go | 23 ++++++++++++++----- pkg/controller/controller.go | 16 +++++++++++++ 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index e25bb4b3ea8..440f20b7032 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -28,7 +28,6 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" - syncutil "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" @@ -59,6 +58,7 @@ type Adder struct { Tracker *readiness.Tracker ProcessExcluder *process.Excluder WatchSet *watch.Set + CacheManager *cm.CacheManager } // Add creates a new ConfigController and adds it to the Manager with default RBAC. The Manager will set fields on the Controller @@ -67,7 +67,7 @@ func (a *Adder) Add(mgr manager.Manager) error { // Events will be used to receive events from dynamic watches registered // via the registrar below. events := make(chan event.GenericEvent, 1024) - r, err := newReconciler(mgr, a.Opa, a.WatchManager, a.ControllerSwitch, a.Tracker, a.ProcessExcluder, events, a.WatchSet, events) + r, err := newReconciler(mgr, a.CacheManager, a.WatchManager, a.ControllerSwitch, a.Tracker, a.ProcessExcluder, events, a.WatchSet, events) if err != nil { return err } @@ -105,15 +105,15 @@ func (a *Adder) InjectWatchSet(watchSet *watch.Set) { a.WatchSet = watchSet } +func (a *Adder) InjectCacheManager(cm *cm.CacheManager) { + a.CacheManager = cm +} + // newReconciler returns a new reconcile.Reconciler // events is the channel from which sync controller will receive the events // regEvents is the channel registered by Registrar to put the events in // events and regEvents point to same event channel except for testing. -func newReconciler(mgr manager.Manager, opa syncutil.OpaDataClient, wm *watch.Manager, cs *watch.ControllerSwitch, tracker *readiness.Tracker, processExcluder *process.Excluder, events <-chan event.GenericEvent, watchSet *watch.Set, regEvents chan<- event.GenericEvent) (*ReconcileConfig, error) { - filteredOpa := syncutil.NewFilteredOpaDataClient(opa, watchSet) - syncMetricsCache := syncutil.NewMetricsCache() - cm := cm.NewCacheManager(filteredOpa, syncMetricsCache, tracker, processExcluder) - +func newReconciler(mgr manager.Manager, cm *cm.CacheManager, wm *watch.Manager, cs *watch.ControllerSwitch, tracker *readiness.Tracker, processExcluder *process.Excluder, events <-chan event.GenericEvent, watchSet *watch.Set, regEvents chan<- event.GenericEvent) (*ReconcileConfig, error) { syncAdder := syncc.Adder{ Events: events, CacheManager: cm, diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index c9274362e1b..36ace958065 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -29,6 +29,8 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" @@ -140,7 +142,9 @@ func TestReconcile(t *testing.T) { processExcluder.Add(instance.Spec.Match) events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() - rec, _ := newReconciler(mgr, opaClient, wm, cs, tracker, processExcluder, events, watchSet, events) + syncMetricsCache := syncutil.NewMetricsCache() + cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) recFn, requests := SetupTestReconcile(rec) err = add(mgr, recFn) @@ -389,11 +393,12 @@ func setupController(mgr manager.Manager, wm *watch.Manager, tracker *readiness. // ControllerSwitch will be used to disable controllers during our teardown process, // avoiding conflicts in finalizer cleanup. cs := watch.NewSwitch() - processExcluder := process.Get() - watchSet := watch.NewSet() - rec, _ := newReconciler(mgr, opaClient, wm, cs, tracker, processExcluder, events, watchSet, nil) + syncMetricsCache := syncutil.NewMetricsCache() + cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + + rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, nil) err = add(mgr, rec) if err != nil { return fmt.Errorf("adding reconciler to manager: %w", err) @@ -434,7 +439,10 @@ func TestConfig_CacheContents(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() - rec, _ := newReconciler(mgr, opaClient, wm, cs, tracker, processExcluder, events, watchSet, events) + syncMetricsCache := syncutil.NewMetricsCache() + cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + + rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) err = add(mgr, rec) if err != nil { t.Fatal(err) @@ -595,7 +603,10 @@ func TestConfig_Retries(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() - rec, _ := newReconciler(mgr, opaClient, wm, cs, tracker, processExcluder, events, watchSet, events) + syncMetricsCache := syncutil.NewMetricsCache() + cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + + rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) err = add(mgr, rec) if err != nil { t.Fatal(err) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2fec2fcda8e..fb0a4fcc3c4 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -29,6 +29,8 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" corev1 "k8s.io/api/core/v1" @@ -69,6 +71,10 @@ type PubsubInjector interface { InjectPubsubSystem(pubsubSystem *pubsub.System) } +type CacheManagerInjector interface { + InjectCacheManager(cm *cm.CacheManager) +} + // Injectors is a list of adder structs that need injection. We can convert this // to an interface once we create controllers for things like data sync. var Injectors []Injector @@ -160,6 +166,11 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { } deps.GetPod = fakePodGetter } + + filteredOpa := syncutil.NewFilteredOpaDataClient(deps.Opa, deps.WatchSet) + syncMetricsCache := syncutil.NewMetricsCache() + cm := cm.NewCacheManager(filteredOpa, syncMetricsCache, deps.Tracker, deps.ProcessExcluder) + for _, a := range Injectors { a.InjectOpa(deps.Opa) a.InjectWatchManager(deps.WatchManger) @@ -180,6 +191,11 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { if a2, ok := a.(PubsubInjector); ok { a2.InjectPubsubSystem(deps.PubsubSystem) } + if a2, ok := a.(CacheManagerInjector); ok { + // this is used by the config controller to sync + a2.InjectCacheManager(cm) + } + if err := a.Add(m); err != nil { return err } From a45577288edcd2f86b34404401b99f083a9619c4 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:29:38 +0000 Subject: [PATCH 06/58] move cm, make syncc submodule Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/{syncutil => controller}/cachemanager/cachemanager.go | 0 .../cachemanager/cachemanager_test.go | 0 pkg/controller/{ => cachemanager}/sync/sync_controller.go | 2 +- pkg/controller/config/config_controller.go | 4 ++-- pkg/controller/config/config_controller_test.go | 2 +- pkg/controller/controller.go | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename pkg/{syncutil => controller}/cachemanager/cachemanager.go (100%) rename pkg/{syncutil => controller}/cachemanager/cachemanager_test.go (100%) rename pkg/controller/{ => cachemanager}/sync/sync_controller.go (98%) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go similarity index 100% rename from pkg/syncutil/cachemanager/cachemanager.go rename to pkg/controller/cachemanager/cachemanager.go diff --git a/pkg/syncutil/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go similarity index 100% rename from pkg/syncutil/cachemanager/cachemanager_test.go rename to pkg/controller/cachemanager/cachemanager_test.go diff --git a/pkg/controller/sync/sync_controller.go b/pkg/controller/cachemanager/sync/sync_controller.go similarity index 98% rename from pkg/controller/sync/sync_controller.go rename to pkg/controller/cachemanager/sync/sync_controller.go index 7dd5630bead..e67059c63ee 100644 --- a/pkg/controller/sync/sync_controller.go +++ b/pkg/controller/cachemanager/sync/sync_controller.go @@ -20,10 +20,10 @@ import ( "time" "github.com/go-logr/logr" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 440f20b7032..24df9506537 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -22,13 +22,13 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" - syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 36ace958065..01fc6fb9a0f 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -26,11 +26,11 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index fb0a4fcc3c4..226f7e692c3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -23,6 +23,7 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" @@ -30,7 +31,6 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" corev1 "k8s.io/api/core/v1" From 4750a94cd304a2223afc6af0787f0fee3c4557d3 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 21 Jun 2023 21:13:59 +0000 Subject: [PATCH 07/58] introduce cm config Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 17 ++++++++++++----- .../cachemanager/cachemanager_test.go | 6 +++--- pkg/controller/config/config_controller_test.go | 8 ++++---- pkg/controller/controller.go | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index 0de3307f3c7..69cbbce0df9 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -12,6 +12,13 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +type CacheManagerConfig struct { + Opa syncutil.OpaDataClient + SyncMetricsCache *syncutil.MetricsCache + Tracker *readiness.Tracker + ProcessExcluder *process.Excluder +} + type CacheManager struct { opa syncutil.OpaDataClient syncMetricsCache *syncutil.MetricsCache @@ -19,12 +26,12 @@ type CacheManager struct { processExcluder *process.Excluder } -func NewCacheManager(opa syncutil.OpaDataClient, syncMetricsCache *syncutil.MetricsCache, tracker *readiness.Tracker, processExcluder *process.Excluder) *CacheManager { +func NewCacheManager(config *CacheManagerConfig) *CacheManager { return &CacheManager{ - opa: opa, - syncMetricsCache: syncMetricsCache, - tracker: tracker, - processExcluder: processExcluder, + opa: config.Opa, + syncMetricsCache: config.SyncMetricsCache, + tracker: config.Tracker, + processExcluder: config.ProcessExcluder, } } diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index 2a0038ccf3d..f8838b15198 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -33,7 +33,7 @@ func TestCacheManager_AddObject_RemoveObject(t *testing.T) { assert.NoError(t, err) processExcluder := process.Get() - cm := NewCacheManager(opaClient, syncutil.NewMetricsCache(), tracker, processExcluder) + cm := NewCacheManager(&CacheManagerConfig{opaClient, syncutil.NewMetricsCache(), tracker, processExcluder}) ctx := context.Background() pod := fakes.Pod( @@ -70,7 +70,7 @@ func TestCacheManager_processExclusion(t *testing.T) { }, }) - cm := NewCacheManager(opaClient, syncutil.NewMetricsCache(), tracker, processExcluder) + cm := NewCacheManager(&CacheManagerConfig{opaClient, syncutil.NewMetricsCache(), tracker, processExcluder}) ctx := context.Background() pod := fakes.Pod( @@ -95,7 +95,7 @@ func TestCacheManager_errors(t *testing.T) { assert.NoError(t, err) processExcluder := process.Get() - cm := NewCacheManager(opaClient, syncutil.NewMetricsCache(), tracker, processExcluder) + cm := NewCacheManager(&CacheManagerConfig{opaClient, syncutil.NewMetricsCache(), tracker, processExcluder}) ctx := context.Background() pod := fakes.Pod( diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 01fc6fb9a0f..222e4944ba8 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -143,7 +143,7 @@ func TestReconcile(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) recFn, requests := SetupTestReconcile(rec) @@ -396,7 +396,7 @@ func setupController(mgr manager.Manager, wm *watch.Manager, tracker *readiness. processExcluder := process.Get() watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, nil) err = add(mgr, rec) @@ -440,7 +440,7 @@ func TestConfig_CacheContents(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) err = add(mgr, rec) @@ -604,7 +604,7 @@ func TestConfig_Retries(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(opaClient, syncMetricsCache, tracker, processExcluder) + cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) err = add(mgr, rec) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 226f7e692c3..2b16e332c48 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -169,7 +169,7 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { filteredOpa := syncutil.NewFilteredOpaDataClient(deps.Opa, deps.WatchSet) syncMetricsCache := syncutil.NewMetricsCache() - cm := cm.NewCacheManager(filteredOpa, syncMetricsCache, deps.Tracker, deps.ProcessExcluder) + cm := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: filteredOpa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, ProcessExcluder: deps.ProcessExcluder}) for _, a := range Injectors { a.InjectOpa(deps.Opa) From c3745a9e79067823691866242890ef41542d3b3d Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 23 Jun 2023 17:44:55 +0000 Subject: [PATCH 08/58] refactor: make cm watch aware dirty, refactor: make cm watch aware Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty: frontload with gvk aggregator Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty: use gvk aggregator for config Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty, refactor: replayData is async Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty, refactor: wipe and replay are async Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty, refactor: dual controllers Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty, refactor: nil check props Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty, refactor: cm.Start Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> dirty, fix: mark gvks to sync Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> tests Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 238 +++++++++++- .../cachemanager/cachemanager_test.go | 362 ++++++++++++++++-- .../cachemanager/sync/sync_controller.go | 5 +- pkg/controller/config/config_controller.go | 175 +-------- .../config/config_controller_test.go | 279 +++++++------- pkg/controller/controller.go | 30 +- pkg/syncutil/aggregator/aggregator.go | 31 +- 7 files changed, 787 insertions(+), 333 deletions(-) diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index 69cbbce0df9..e0313f1800f 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -3,20 +3,32 @@ package cachemanager import ( "context" "fmt" + "time" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) +var log = logf.Log.WithName("cache-manager") + type CacheManagerConfig struct { Opa syncutil.OpaDataClient SyncMetricsCache *syncutil.MetricsCache Tracker *readiness.Tracker ProcessExcluder *process.Excluder + Registrar *watch.Registrar + WatchedSet *watch.Set + GVKAggregator *aggregator.GVKAgreggator + Reader client.Reader } type CacheManager struct { @@ -24,15 +36,159 @@ type CacheManager struct { syncMetricsCache *syncutil.MetricsCache tracker *readiness.Tracker processExcluder *process.Excluder + registrar *watch.Registrar + watchedSet *watch.Set + gvkAggregator *aggregator.GVKAgreggator + gvksToRemove *watch.Set + gvksToSync *watch.Set + replayErrChan chan error + replayTicker time.Ticker + reader client.Reader } -func NewCacheManager(config *CacheManagerConfig) *CacheManager { - return &CacheManager{ +func NewCacheManager(config *CacheManagerConfig) (*CacheManager, error) { + if config.WatchedSet == nil { + return nil, fmt.Errorf("watchedSet must be non-nil") + } + if config.Registrar == nil { + return nil, fmt.Errorf("registrar must be non-nil") + } + if config.ProcessExcluder == nil { + return nil, fmt.Errorf("processExcluder must be non-nil") + } + if config.Tracker == nil { + return nil, fmt.Errorf("tracker must be non-nil") + } + if config.Reader == nil { + return nil, fmt.Errorf("reader must be non-nil") + } + + cm := &CacheManager{ opa: config.Opa, syncMetricsCache: config.SyncMetricsCache, tracker: config.Tracker, processExcluder: config.ProcessExcluder, + registrar: config.Registrar, + watchedSet: config.WatchedSet, + reader: config.Reader, + } + + cm.gvkAggregator = aggregator.NewGVKAggregator() + cm.gvksToRemove = watch.NewSet() + cm.gvksToSync = watch.NewSet() + + cm.replayTicker = *time.NewTicker(3 * time.Second) + + return cm, nil +} + +func (c *CacheManager) Start(ctx context.Context) error { + go c.updateDatastore(ctx) + + <-ctx.Done() + return nil +} + +// WatchGVKsToSync adjusts the watched set of gvks according to the newGVKs passed in +// for a given {syncSourceType, syncSourceName}. +func (c *CacheManager) WatchGVKsToSync(ctx context.Context, newGVKs []schema.GroupVersionKind, newExcluder *process.Excluder, syncSourceType, syncSourceName string) error { + netNewGVKs := []schema.GroupVersionKind{} + for _, gvk := range newGVKs { + if !c.gvkAggregator.IsPresent(gvk) { + netNewGVKs = append(netNewGVKs, gvk) + } + } + // mark these gvks for the background goroutine to sync + c.gvksToSync.Add(netNewGVKs...) + + opKey := aggregator.Key{Source: syncSourceType, ID: syncSourceName} + currentGVKsForKey := c.gvkAggregator.List(opKey) + + if len(newGVKs) == 0 { + // we are not syncing anything for this key anymore + if err := c.gvkAggregator.Remove(opKey); err != nil { + return fmt.Errorf("internal error removing gvks for aggregation: %w", err) + } + } else { + if err := c.gvkAggregator.Upsert(opKey, newGVKs); err != nil { + return fmt.Errorf("internal error upserting gvks for aggregation: %w", err) + } + } + + // stage the new watch set for events for the sync_controller to be + // the current watch set ... [1/3] + newGvkWatchSet := watch.NewSet() + newGvkWatchSet.AddSet(c.watchedSet) + // ... plus the net new gvks we are adding ... [2/3] + newGvkWatchSet.Add(netNewGVKs...) + + gvksToDeleteCandidates := getGVKsToDeleteCandidates(newGVKs, currentGVKsForKey) + gvksToDeleteSet := watch.NewSet() + for _, gvk := range gvksToDeleteCandidates { + // if this gvk is no longer required by any source, schedule it to be deleted + if !c.gvkAggregator.IsPresent(gvk) { + // Remove expectations for resources we no longer watch. + c.tracker.CancelData(gvk) + // mark these gvks for the background goroutine to un-sync + gvksToDeleteSet.Add(gvk) + } + } + c.gvksToRemove.AddSet(gvksToDeleteSet) + + // ... less the gvks to delete. [3/3] + newGvkWatchSet.RemoveSet(gvksToDeleteSet) + + // If the watch set has not changed AND the process excluder is the same we're done here. + if c.watchedSet.Equals(newGvkWatchSet) && newExcluder != nil { + if c.processExcluder.Equals(newExcluder) { + return nil + } else { + // there is a new excluder which means we need to schedule a wipe for any + // previously watched GVKs to be re-added to get a chance to be evaluated + // for this new process excluder. + + c.gvksToRemove.AddSet(newGvkWatchSet) + } + } + + // Start watching the newly added gvks set + var innerError error + c.watchedSet.Replace(newGvkWatchSet, func() { + // swapping with the new excluder + if newExcluder != nil { + c.processExcluder.Replace(newExcluder) + } + + // *Note the following steps are not transactional with respect to admission control* + + // Important: dynamic watches update must happen *after* updating our watchSet. + // Otherwise, the sync controller will drop events for the newly watched kinds. + // Defer error handling so object re-sync happens even if the watch is hard + // errored due to a missing GVK in the watch set. + innerError = c.registrar.ReplaceWatch(ctx, newGvkWatchSet.Items()) + }) + if innerError != nil { + return innerError } + + return nil +} + +// returns GVKs that are in currentGVKsForKey but not in newGVKs. +func getGVKsToDeleteCandidates(newGVKs []schema.GroupVersionKind, currentGVKsForKey map[schema.GroupVersionKind]struct{}) []schema.GroupVersionKind { + newGVKSet := make(map[schema.GroupVersionKind]struct{}) + for _, gvk := range newGVKs { + newGVKSet[gvk] = struct{}{} + } + + var toDelete []schema.GroupVersionKind + for gvk := range currentGVKsForKey { + if _, found := newGVKSet[gvk]; !found { + toDelete = append(toDelete, gvk) + } + } + + return toDelete } func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Unstructured) error { @@ -101,6 +257,80 @@ func (c *CacheManager) ReportSyncMetrics() { c.syncMetricsCache.ReportSync() } -func (c *CacheManager) ReplaceExcluder(p *process.Excluder) { - c.processExcluder.Replace(p) +func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.GroupVersionKind, reader client.Reader) error { + u := &unstructured.UnstructuredList{} + u.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind + "List", + }) + + err := reader.List(ctx, u) + if err != nil { + return fmt.Errorf("replaying data for %+v: %w", gvk, err) + } + + defer c.ReportSyncMetrics() + + for i := range u.Items { + if err := c.AddObject(ctx, &u.Items[i]); err != nil { + return fmt.Errorf("adding data for %+v: %w", gvk, err) + } + } + + return nil +} + +func (c *CacheManager) updateDatastore(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-c.replayTicker.C: + c.makeUpdates(ctx) + } + } +} + +// listAndSyncData returns a set of gvks that were successfully listed and synced. +func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupVersionKind, reader client.Reader) *watch.Set { + gvksSuccessfullySynced := watch.NewSet() + for _, gvk := range gvks { + err := c.listAndSyncDataForGVK(ctx, gvk, c.reader) + if err != nil { + log.Error(err, "internal: error syncing gvks cache data") + // we don't remove this gvk as we will try to re-add it later + // we also don't return on this error to be able to list and sync + // other gvks in order to protect against a bad gvk. + } else { + gvksSuccessfullySynced.Add(gvk) + } + } + return gvksSuccessfullySynced +} + +// makeUpdates performs a conditional wipe followed by a replay if necessary. +func (c *CacheManager) makeUpdates(ctx context.Context) { + // first, wipe the cache if needed + gvksToDelete := c.gvksToRemove.Items() + if len(gvksToDelete) > 0 { + // "checkpoint save" what needs to be replayed + gvksToReplay := c.gvkAggregator.ListAllGVKs() + // and add it to be synced below + c.gvksToSync.Add(gvksToReplay...) + + if err := c.WipeData(ctx); err != nil { + log.Error(err, "internal: error wiping cache") + // don't alter the toRemove set, we will try again + } else { + c.gvksToRemove.Remove(gvksToDelete...) + // any gvks that were just removed shouldn't be synced + c.gvksToSync.Remove(gvksToDelete...) + } + } + + // sync net new gvks + gvksToSyncList := c.gvksToSync.Items() + gvksSynced := c.listAndSyncData(ctx, gvksToSyncList, c.reader) + c.gvksToSync.RemoveSet(gvksSynced) } diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index f8838b15198..c0999766f09 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -10,12 +10,18 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" + "github.com/open-policy-agent/gatekeeper/v3/pkg/util" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" ) var cfg *rest.Config @@ -26,15 +32,7 @@ func TestMain(m *testing.M) { // TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. func TestCacheManager_AddObject_RemoveObject(t *testing.T) { - mgr, _ := testutils.SetupManager(t, cfg) - opaClient := &fakes.FakeOpa{} - - tracker, err := readiness.SetupTracker(mgr, false, false, false) - assert.NoError(t, err) - - processExcluder := process.Get() - cm := NewCacheManager(&CacheManagerConfig{opaClient, syncutil.NewMetricsCache(), tracker, processExcluder}) - ctx := context.Background() + cm, _, ctx := makeCacheManagerForTest(t, false, false) pod := fakes.Pod( fakes.WithNamespace("test-ns"), @@ -46,6 +44,8 @@ func TestCacheManager_AddObject_RemoveObject(t *testing.T) { require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) // test that pod is cache managed + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) require.True(t, opaClient.HasGVK(pod.GroupVersionKind())) // now remove the object and verify it's removed @@ -55,13 +55,7 @@ func TestCacheManager_AddObject_RemoveObject(t *testing.T) { // TestCacheManager_processExclusion makes sure that we don't add objects that are process excluded. func TestCacheManager_processExclusion(t *testing.T) { - mgr, _ := testutils.SetupManager(t, cfg) - opaClient := &fakes.FakeOpa{} - - tracker, err := readiness.SetupTracker(mgr, false, false, false) - assert.NoError(t, err) - - // exclude "test-ns-excluded" namespace + cm, _, ctx := makeCacheManagerForTest(t, false, false) processExcluder := process.Get() processExcluder.Add([]configv1alpha1.MatchEntry{ { @@ -69,9 +63,7 @@ func TestCacheManager_processExclusion(t *testing.T) { Processes: []string{"sync"}, }, }) - - cm := NewCacheManager(&CacheManagerConfig{opaClient, syncutil.NewMetricsCache(), tracker, processExcluder}) - ctx := context.Background() + cm.processExcluder.Replace(processExcluder) pod := fakes.Pod( fakes.WithNamespace("test-ns-excluded"), @@ -82,21 +74,17 @@ func TestCacheManager_processExclusion(t *testing.T) { require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) // test that pod from excluded namespace is not cache managed + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) } -// TestCacheManager_errors tests that we cache manager responds to errors from the opa client. +// TestCacheManager_errors tests that the cache manager responds to errors from the opa client. func TestCacheManager_errors(t *testing.T) { - mgr, _ := testutils.SetupManager(t, cfg) - opaClient := &fakes.FakeOpa{} - opaClient.SetErroring(true) // AddObject, RemoveObject will error out now. - - tracker, err := readiness.SetupTracker(mgr, false, false, false) - assert.NoError(t, err) - - processExcluder := process.Get() - cm := NewCacheManager(&CacheManagerConfig{opaClient, syncutil.NewMetricsCache(), tracker, processExcluder}) - ctx := context.Background() + cm, _, ctx := makeCacheManagerForTest(t, false, false) + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) + opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err pod := fakes.Pod( fakes.WithNamespace("test-ns"), @@ -109,3 +97,315 @@ func TestCacheManager_errors(t *testing.T) { require.ErrorContains(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") require.ErrorContains(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") } + +// TestCacheManager_listAndSyncData tests that the cache manager can add gvks to the data store. +func TestCacheManager_listAndSyncData(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) + + configMapGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "ConfigMap", + } + // Create configMaps to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + cm2 := unstructuredFor(configMapGVK, "config-test-2") + require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + + require.NoError(t, cacheManager.listAndSyncDataForGVK(ctx, configMapGVK, c)) + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + } + + require.Equal(t, 2, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // wipe cache + require.NoError(t, cacheManager.WipeData(ctx)) + require.False(t, opaClient.Contains(expected)) + + // create a second GVK + podGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + } + // Create pods to test for + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + pod2 := unstructuredFor(podGVK, "pod-2") + require.NoError(t, c.Create(ctx, pod2), "creating Pod pod-2") + + pod3 := unstructuredFor(podGVK, "pod-3") + require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") + + syncedSet := cacheManager.listAndSyncData(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}, c) + require.ElementsMatch(t, syncedSet.Items(), []schema.GroupVersionKind{configMapGVK, podGVK}) + + expected = map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + {Gvk: podGVK, Key: "default/pod-2"}: nil, + {Gvk: podGVK, Key: "default/pod-3"}: nil, + } + + require.Equal(t, 5, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") + require.NoError(t, c.Delete(ctx, pod3), "deleting Pod pod-3") + require.NoError(t, c.Delete(ctx, pod2), "deleting Pod pod-2") +} + +// TestCacheManager_makeUpdates tests that we can remove and add gvks to the data store. +func TestCacheManager_makeUpdates(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) + + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + fooGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Foo"} + barGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Bar"} + + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + // seed gvks + cacheManager.gvksToRemove.Add(fooGVK, barGVK) + cacheManager.gvksToSync.Add(configMapGVK) + cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK}) + + // after the updates we should not see any gvks that + // were removed but should see what was in the aggregator + // and the new gvks to sync. + cacheManager.makeUpdates(ctx) + + // check that the "work queues" were drained + require.Len(t, cacheManager.gvksToSync.Items(), 0) + require.Len(t, cacheManager.gvksToRemove.Items(), 0) + + // expect the following instances to be in the data store + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + require.Equal(t, 2, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") +} + +// TestCacheManager_WatchGVKsToSync also tests replay. +func TestCacheManager_WatchGVKsToSync(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + + // Create configMaps to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + cm2 := unstructuredFor(configMapGVK, "config-test-2") + require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + + syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} + require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}, nil, syncSourceOne.Source, syncSourceOne.ID)) + + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + + reqCondition := func(expected map[fakes.OpaKey]interface{}) func() bool { + return func() bool { + if opaClient.Len() != len(expected) { + return false + } + if opaClient.Contains(expected) { + return true + } + return false + } + } + require.Eventually(t, reqCondition(expected), testutils.TenSecondWaitFor, testutils.OneSecondTick) + + // now assert that the gvkAggregator looks as expected + cacheManager.gvkAggregator.IsPresent(configMapGVK) + gvks := cacheManager.gvkAggregator.List(syncSourceOne) + require.Len(t, gvks, 2) + _, foundConfigMap := gvks[configMapGVK] + require.True(t, foundConfigMap) + _, foundPod := gvks[podGVK] + require.True(t, foundPod) + + // now remove the podgvk and make sure we don't have pods in the cache anymore + require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{configMapGVK}, nil, syncSourceOne.Source, syncSourceOne.ID)) + + expected = map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + } + require.Eventually(t, reqCondition(expected), testutils.TenSecondWaitFor, testutils.OneSecondTick) + // now assert that the gvkAggregator looks as expected + cacheManager.gvkAggregator.IsPresent(configMapGVK) + gvks = cacheManager.gvkAggregator.List(syncSourceOne) + require.Len(t, gvks, 1) + _, foundConfigMap = gvks[configMapGVK] + require.True(t, foundConfigMap) + _, foundPod = gvks[podGVK] + require.False(t, foundPod) + + // now make sure that adding another sync source with the same gvk has no side effects + syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} + require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{configMapGVK}, nil, syncSourceTwo.Source, syncSourceTwo.ID)) + + reqConditionForAgg := func() bool { + cacheManager.gvkAggregator.IsPresent(configMapGVK) + gvks := cacheManager.gvkAggregator.List(syncSourceOne) + if len(gvks) != 1 { + return false + } + _, found := gvks[configMapGVK] + if !found { + return false + } + + gvks2 := cacheManager.gvkAggregator.List(syncSourceTwo) + if len(gvks2) != 1 { + return false + } + _, found2 := gvks2[configMapGVK] + if !found2 { + return false + } + + return true + } + require.Eventually(t, reqConditionForAgg, testutils.TenSecondWaitFor, testutils.OneSecondTick) + + // now add pod2 + require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{podGVK}, nil, syncSourceOne.Source, syncSourceOne.ID)) + expected2 := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + require.Eventually(t, reqCondition(expected2), testutils.TenSecondWaitFor, testutils.OneSecondTick) + + // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed + require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{}, nil, syncSourceTwo.Source, syncSourceTwo.ID)) + expected3 := map[fakes.OpaKey]interface{}{ + // config maps no longer required by any sync source + //{Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + //{Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + require.Eventually(t, reqCondition(expected3), testutils.TenSecondWaitFor, testutils.OneSecondTick) + + // now process exclude the remaing gvk, it should get removed now + blankExcluder := process.New() + blankExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []util.Wildcard{"default"}, + Processes: []string{"sync"}, + }, + }) + require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{podGVK}, blankExcluder, syncSourceOne.Source, syncSourceOne.ID)) + expected4 := map[fakes.OpaKey]interface{}{} + require.Eventually(t, reqCondition(expected4), testutils.TenSecondWaitFor, testutils.OneSecondTick) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") +} + +func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetName(name) + u.SetNamespace("default") + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + return u +} + +func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*CacheManager, client.Client, context.Context) { + ctx, cancelFunc := context.WithCancel(context.Background()) + mgr, wm := testutils.SetupManager(t, cfg) + + c := testclient.NewRetryClient(mgr.GetClient()) + opaClient := &fakes.FakeOpa{} + tracker, err := readiness.SetupTracker(mgr, false, false, false) + require.NoError(t, err) + processExcluder := process.Get() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []util.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + events := make(chan event.GenericEvent, 1024) + w, err := wm.NewRegistrar( + "test-cache-manager", + events) + cacheManager, err := NewCacheManager(&CacheManagerConfig{ + Opa: opaClient, + SyncMetricsCache: syncutil.NewMetricsCache(), + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watch.NewSet(), + Registrar: w, + Reader: c, + }) + require.NoError(t, err) + + if startCache { + go cacheManager.Start(ctx) + t.Cleanup(func() { + ctx.Done() + }) + } + + if startManager { + testutils.StartManager(ctx, t, mgr) + } + + t.Cleanup(func() { + cancelFunc() + }) + return cacheManager, c, ctx +} diff --git a/pkg/controller/cachemanager/sync/sync_controller.go b/pkg/controller/cachemanager/sync/sync_controller.go index e67059c63ee..9c518c481d0 100644 --- a/pkg/controller/cachemanager/sync/sync_controller.go +++ b/pkg/controller/cachemanager/sync/sync_controller.go @@ -172,13 +172,10 @@ func (r *ReconcileSync) Reconcile(ctx context.Context, request reconcile.Request logging.ResourceName, instance.GetName(), ) + reportMetrics = true if err := r.cm.AddObject(ctx, instance); err != nil { - reportMetrics = true - return reconcile.Result{}, err } - reportMetrics = true - return reconcile.Result{}, nil } diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 24df9506537..f3f59c12e08 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -23,7 +23,6 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" - syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" @@ -31,12 +30,10 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -45,7 +42,8 @@ import ( ) const ( - ctrlName = "config-controller" + CtrlName = "config-controller" + finalizerName = "finalizers.gatekeeper.sh/config" ) @@ -64,10 +62,7 @@ type Adder struct { // Add creates a new ConfigController and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func (a *Adder) Add(mgr manager.Manager) error { - // Events will be used to receive events from dynamic watches registered - // via the registrar below. - events := make(chan event.GenericEvent, 1024) - r, err := newReconciler(mgr, a.CacheManager, a.WatchManager, a.ControllerSwitch, a.Tracker, a.ProcessExcluder, events, a.WatchSet, events) + r, err := newReconciler(mgr, a.CacheManager, a.WatchManager, a.ControllerSwitch, a.Tracker, a.ProcessExcluder, a.WatchSet) if err != nil { return err } @@ -109,48 +104,27 @@ func (a *Adder) InjectCacheManager(cm *cm.CacheManager) { a.CacheManager = cm } -// newReconciler returns a new reconcile.Reconciler -// events is the channel from which sync controller will receive the events -// regEvents is the channel registered by Registrar to put the events in -// events and regEvents point to same event channel except for testing. -func newReconciler(mgr manager.Manager, cm *cm.CacheManager, wm *watch.Manager, cs *watch.ControllerSwitch, tracker *readiness.Tracker, processExcluder *process.Excluder, events <-chan event.GenericEvent, watchSet *watch.Set, regEvents chan<- event.GenericEvent) (*ReconcileConfig, error) { - syncAdder := syncc.Adder{ - Events: events, - CacheManager: cm, - } - // Create subordinate controller - we will feed it events dynamically via watch - if err := syncAdder.Add(mgr); err != nil { - return nil, fmt.Errorf("registering sync controller: %w", err) - } - +// newReconciler returns a new reconcile.Reconciler. +func newReconciler(mgr manager.Manager, cm *cm.CacheManager, wm *watch.Manager, cs *watch.ControllerSwitch, tracker *readiness.Tracker, processExcluder *process.Excluder, watchSet *watch.Set) (*ReconcileConfig, error) { if watchSet == nil { return nil, fmt.Errorf("watchSet must be non-nil") } - w, err := wm.NewRegistrar( - ctrlName, - regEvents) - if err != nil { - return nil, err - } return &ReconcileConfig{ - reader: mgr.GetCache(), - writer: mgr.GetClient(), - statusClient: mgr.GetClient(), - scheme: mgr.GetScheme(), - cs: cs, - watcher: w, - watched: watchSet, - cacheManager: cm, - tracker: tracker, - processExcluder: processExcluder, + reader: mgr.GetCache(), + writer: mgr.GetClient(), + statusClient: mgr.GetClient(), + scheme: mgr.GetScheme(), + cs: cs, + cacheManager: cm, + tracker: tracker, }, nil } // add adds a new Controller to mgr with r as the reconcile.Reconciler. func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ctrlName, mgr, controller.Options{Reconciler: r}) + c, err := controller.New(CtrlName, mgr, controller.Options{Reconciler: r}) if err != nil { return err } @@ -175,14 +149,8 @@ type ReconcileConfig struct { scheme *runtime.Scheme cacheManager *cm.CacheManager cs *watch.ControllerSwitch - watcher *watch.Registrar - watched *watch.Set - - needsReplay *watch.Set - needsWipe bool - tracker *readiness.Tracker - processExcluder *process.Excluder + tracker *readiness.Tracker } // +kubebuilder:rbac:groups=*,resources=*,verbs=get;list;watch @@ -233,15 +201,15 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque } } - newSyncOnly := watch.NewSet() newExcluder := process.New() var statsEnabled bool // If the config is being deleted the user is saying they don't want to // sync anything + gvksToSync := []schema.GroupVersionKind{} if exists && instance.GetDeletionTimestamp().IsZero() { for _, entry := range instance.Spec.Sync.SyncOnly { gvk := schema.GroupVersionKind{Group: entry.Group, Version: entry.Version, Kind: entry.Kind} - newSyncOnly.Add(gvk) + gvksToSync = append(gvksToSync, gvk) } newExcluder.Add(instance.Spec.Match) @@ -257,120 +225,13 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.tracker.DisableStats() } - // Remove expectations for resources we no longer watch. - diff := r.watched.Difference(newSyncOnly) - r.removeStaleExpectations(diff) - - // If the watch set has not changed, we're done here. - if r.watched.Equals(newSyncOnly) && r.processExcluder.Equals(newExcluder) { - // ...unless we have pending wipe / replay operations from a previous reconcile. - if !(r.needsWipe || r.needsReplay != nil) { - return reconcile.Result{}, nil - } - - // If we reach here, the watch set hasn't changed since last reconcile, but we - // have unfinished wipe/replay business from the last change. - } else { - // The watch set _has_ changed, so recalculate the replay set. - r.needsReplay = nil - r.needsWipe = true - } - - // --- Start watching the new set --- - - // This must happen first - signals to the opa client in the sync controller - // to drop events from no-longer-watched resources that may be in its queue. - if r.needsReplay == nil { - r.needsReplay = r.watched.Intersection(newSyncOnly) - } - - // Wipe all data to avoid stale state if needed. Happens once per watch-set-change. - if err := r.wipeCacheIfNeeded(ctx); err != nil { - return reconcile.Result{}, fmt.Errorf("wiping opa data cache: %w", err) - } - - r.watched.Replace(newSyncOnly, func() { - // swapping with the new excluder - r.cacheManager.ReplaceExcluder(newExcluder) - - // *Note the following steps are not transactional with respect to admission control* - - // Important: dynamic watches update must happen *after* updating our watchSet. - // Otherwise, the sync controller will drop events for the newly watched kinds. - // Defer error handling so object re-sync happens even if the watch is hard - // errored due to a missing GVK in the watch set. - err = r.watcher.ReplaceWatch(ctx, newSyncOnly.Items()) - }) - if err != nil { - return reconcile.Result{}, err - } - - // Replay cached data for any resources that were previously watched and still in the watch set. - // This is necessary because we wipe their data from Opa above. - // TODO(OREN): Improve later by selectively removing subtrees of data instead of a full wipe. - if err := r.replayData(ctx); err != nil { - return reconcile.Result{}, fmt.Errorf("replaying data: %w", err) + if err := r.cacheManager.WatchGVKsToSync(ctx, gvksToSync, newExcluder, "config", request.NamespacedName.Name); err != nil { + return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) } return reconcile.Result{}, nil } -func (r *ReconcileConfig) wipeCacheIfNeeded(ctx context.Context) error { - if r.needsWipe { - if err := r.cacheManager.WipeData(ctx); err != nil { - return err - } - } - return nil -} - -// replayData replays all watched and cached data into Opa following a config set change. -// In the future we can rework this to avoid the full opa data cache wipe. -func (r *ReconcileConfig) replayData(ctx context.Context) error { - if r.needsReplay == nil { - return nil - } - for _, gvk := range r.needsReplay.Items() { - u := &unstructured.UnstructuredList{} - u.SetGroupVersionKind(schema.GroupVersionKind{ - Group: gvk.Group, - Version: gvk.Version, - Kind: gvk.Kind + "List", - }) - err := r.reader.List(ctx, u) - if err != nil { - return fmt.Errorf("replaying data for %+v: %w", gvk, err) - } - - defer r.cacheManager.ReportSyncMetrics() - - for i := range u.Items { - if err := r.cacheManager.AddObject(ctx, &u.Items[i]); err != nil { - return fmt.Errorf("adding data for %+v: %w", gvk, err) - } - } - r.needsReplay.Remove(gvk) - } - r.needsReplay = nil - return nil -} - -// removeStaleExpectations stops tracking data for any resources that are no longer watched. -func (r *ReconcileConfig) removeStaleExpectations(stale *watch.Set) { - for _, gvk := range stale.Items() { - r.tracker.CancelData(gvk) - } -} - -func (r *ReconcileConfig) skipExcludedNamespace(obj *unstructured.Unstructured) (bool, error) { - isNamespaceExcluded, err := r.processExcluder.IsNamespaceExcluded(process.Sync, obj) - if err != nil { - return false, err - } - - return isNamespaceExcluded, err -} - func containsString(s string, items []string) bool { for _, item := range items { if item == s { diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 222e4944ba8..0f15d1aeff8 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -27,6 +27,7 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" @@ -37,8 +38,10 @@ import ( testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" "golang.org/x/net/context" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -122,16 +125,7 @@ func TestReconcile(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - // initialize OPA - driver, err := rego.New(rego.Tracing(true)) - if err != nil { - t.Fatalf("unable to set up Driver: %v", err) - } - - opaClient, err := constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) - if err != nil { - t.Fatalf("unable to set up OPA client: %s", err) - } + opaClient := &fakes.FakeOpa{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) @@ -143,8 +137,21 @@ func TestReconcile(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) - rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) + w, err := wm.NewRegistrar( + CtrlName, + events) + require.NoError(t, err) + cacheManager, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + Opa: opaClient, + SyncMetricsCache: syncMetricsCache, + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watchSet, + Registrar: w, + Reader: c, + }) + require.NoError(t, err) + rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) recFn, requests := SetupTestReconcile(rec) err = add(mgr, recFn) @@ -255,6 +262,25 @@ func TestReconcile(t *testing.T) { t.Fatal(err) } + fooNs := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + require.NoError(t, c.Create(ctx, fooNs)) + fooPod.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + require.NoError(t, c.Create(ctx, fooPod)) + // fooPod should be namespace excluded, hence not synced + g.Eventually(opaClient.Contains(map[fakes.OpaKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}}), timeout).ShouldNot(gomega.BeTrue()) + + require.NoError(t, c.Delete(ctx, fooPod)) testMgrStopped() cs.Stop() } @@ -285,7 +311,7 @@ func TestConfig_DeleteSyncResources(t *testing.T) { }, }, } - ctx := context.Background() + ctx, cancelFunc := context.WithCancel(context.Background()) err := c.Create(ctx, instance) if err != nil { t.Fatal(err) @@ -328,13 +354,10 @@ func TestConfig_DeleteSyncResources(t *testing.T) { events := make(chan event.GenericEvent, 1024) // set up controller and add it to the manager - err = setupController(mgr, wm, tracker, events) - if err != nil { - t.Fatal(err) - } + _, err = setupController(ctx, mgr, wm, tracker, events, c, false) + require.NoError(t, err, "failed to set up controller") // start manager that will start tracker and controller - ctx, cancelFunc := context.WithCancel(context.Background()) testutils.StartManager(ctx, t, mgr) once := gosync.Once{} defer func() { @@ -378,16 +401,21 @@ func TestConfig_DeleteSyncResources(t *testing.T) { }, timeout).Should(gomega.BeTrue()) } -func setupController(mgr manager.Manager, wm *watch.Manager, tracker *readiness.Tracker, events <-chan event.GenericEvent) error { +func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager, tracker *readiness.Tracker, events chan event.GenericEvent, reader client.Reader, useFakeOpa bool) (syncutil.OpaDataClient, error) { // initialize OPA - driver, err := rego.New(rego.Tracing(true)) - if err != nil { - return fmt.Errorf("unable to set up Driver: %w", err) - } + var opaClient syncutil.OpaDataClient + if useFakeOpa { + opaClient = &fakes.FakeOpa{} + } else { + driver, err := rego.New(rego.Tracing(true)) + if err != nil { + return nil, fmt.Errorf("unable to set up Driver: %w", err) + } - opaClient, err := constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) - if err != nil { - return fmt.Errorf("unable to set up OPA backend client: %w", err) + opaClient, err = constraintclient.NewClient(constraintclient.Targets(&target.K8sValidationTarget{}), constraintclient.Driver(driver)) + if err != nil { + return nil, fmt.Errorf("unable to set up OPA backend client: %w", err) + } } // ControllerSwitch will be used to disable controllers during our teardown process, @@ -396,18 +424,51 @@ func setupController(mgr manager.Manager, wm *watch.Manager, tracker *readiness. processExcluder := process.Get() watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) - - rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, nil) + w, err := wm.NewRegistrar( + CtrlName, + events) + if err != nil { + return nil, fmt.Errorf("cannot create registrar: %w", err) + } + cacheManager, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + Opa: opaClient, + SyncMetricsCache: syncMetricsCache, + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watchSet, + Registrar: w, + Reader: reader, + }) + if err != nil { + return nil, fmt.Errorf("error creating cache manager: %w", err) + } + go cacheManager.Start(ctx) + rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) + if err != nil { + return nil, fmt.Errorf("creating reconciler: %w", err) + } err = add(mgr, rec) if err != nil { - return fmt.Errorf("adding reconciler to manager: %w", err) + return nil, fmt.Errorf("adding reconciler to manager: %w", err) + } + + syncAdder := syncc.Adder{ + Events: events, + CacheManager: cacheManager, } - return nil + err = syncAdder.Add(mgr) + if err != nil { + return nil, fmt.Errorf("registering sync controller: %w", err) + } + return opaClient, nil } // Verify the Opa cache is populated based on the config resource. func TestConfig_CacheContents(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) + // Setup the Manager and Controller. + mgr, wm := setupManager(t) + c := testclient.NewRetryClient(mgr.GetClient()) g := gomega.NewGomegaWithT(t) nsGVK := schema.GroupVersionKind{ Group: "", @@ -419,36 +480,25 @@ func TestConfig_CacheContents(t *testing.T) { Version: "v1", Kind: "ConfigMap", } - instance := configFor([]schema.GroupVersionKind{ - nsGVK, - configMapGVK, - }) + // Create a configMap to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + cm.SetNamespace("default") + require.NoError(t, c.Create(ctx, cm), "creating configMap config-test-1") - // Setup the Manager and Controller. - mgr, wm := setupManager(t) - c := testclient.NewRetryClient(mgr.GetClient()) + cm2 := unstructuredFor(configMapGVK, "config-test-2") + cm2.SetNamespace("kube-system") + require.NoError(t, c.Create(ctx, cm2), "creating configMap config-test21") - opaClient := &fakes.FakeOpa{} - cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) - if err != nil { - t.Fatal(err) - } - processExcluder := process.Get() - processExcluder.Add(instance.Spec.Match) + require.NoError(t, err) events := make(chan event.GenericEvent, 1024) - watchSet := watch.NewSet() - syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) + opa, err := setupController(ctx, mgr, wm, tracker, events, c, true) + require.NoError(t, err, "failed to set up controller") - rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) - err = add(mgr, rec) - if err != nil { - t.Fatal(err) - } + opaClient, ok := opa.(*fakes.FakeOpa) + require.True(t, ok) - ctx, cancelFunc := context.WithCancel(context.Background()) testutils.StartManager(ctx, t, mgr) once := gosync.Once{} testMgrStopped := func() { @@ -456,54 +506,11 @@ func TestConfig_CacheContents(t *testing.T) { cancelFunc() }) } - defer testMgrStopped() // Create the Config object and expect the Reconcile to be created - ctx = context.Background() - - instance = configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) - - // Since we're reusing instance between tests, we must wait for it to be fully - // deleted. We also can't reuse the same instance without introducing - // flakiness as client.Client methods modify their input. - g.Eventually(ensureDeleted(ctx, c, instance), timeout). - ShouldNot(gomega.HaveOccurred()) - g.Eventually(ensureCreated(ctx, c, instance), timeout). - ShouldNot(gomega.HaveOccurred()) - - t.Cleanup(func() { - err = c.Delete(ctx, instance) - if !apierrors.IsNotFound(err) { - t.Errorf("got Delete(instance) error %v, want IsNotFound", err) - } - }) - - // Create a configMap to test for - cm := unstructuredFor(configMapGVK, "config-test-1") - cm.SetNamespace("default") - err = c.Create(ctx, cm) - if err != nil { - t.Fatalf("creating configMap config-test-1: %v", err) - } - - cm2 := unstructuredFor(configMapGVK, "config-test-2") - cm2.SetNamespace("kube-system") - err = c.Create(ctx, cm2) - if err != nil { - t.Fatalf("creating configMap config-test-2: %v", err) - } - - defer func() { - err = c.Delete(ctx, cm) - if err != nil { - t.Fatal(err) - } - err = c.Delete(ctx, cm2) - if err != nil { - t.Fatal(err) - } - }() + config := configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) + require.NoError(t, c.Create(ctx, config), "creating Config config") expected := map[fakes.OpaKey]interface{}{ {Gvk: nsGVK, Key: "default"}: nil, @@ -513,22 +520,16 @@ func TestConfig_CacheContents(t *testing.T) { g.Eventually(func() bool { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "checking initial opa cache contents") - - // Sanity - if !opaClient.HasGVK(nsGVK) { - t.Fatal("want opaClient.HasGVK(nsGVK) to be true but got false") - } + require.True(t, opaClient.HasGVK(nsGVK), "want opaClient.HasGVK(nsGVK) to be true but got false") // Reconfigure to drop the namespace watches - instance = configFor([]schema.GroupVersionKind{configMapGVK}) - forUpdate := instance.DeepCopy() - _, err = controllerutil.CreateOrUpdate(ctx, c, forUpdate, func() error { - forUpdate.Spec = instance.Spec - return nil - }) - if err != nil { - t.Fatalf("updating Config resource: %v", err) - } + config = configFor([]schema.GroupVersionKind{configMapGVK}) + configUpdate := config.DeepCopy() + // configUpdate.SetResourceVersion() + + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(configUpdate), configUpdate)) + configUpdate.Spec = config.Spec + require.NoError(t, c.Update(ctx, configUpdate), "updating Config config") // Expect namespaces to go away from cache g.Eventually(func() bool { @@ -561,18 +562,20 @@ func TestConfig_CacheContents(t *testing.T) { if opaClient.Len() == 0 { t.Fatal("sanity") } - err = c.Delete(ctx, instance) - if err != nil { - t.Fatalf("deleting Config resource: %v", err) - } + require.NoError(t, c.Delete(ctx, config), "deleting Config resource") // The cache will be cleared out. g.Eventually(func() int { return opaClient.Len() }, 10*time.Second).Should(gomega.BeZero(), "waiting for cache to empty") + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting configMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting configMap config-test-2") } func TestConfig_Retries(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) g := gomega.NewGomegaWithT(t) nsGVK := schema.GroupVersionKind{ Group: "", @@ -584,9 +587,7 @@ func TestConfig_Retries(t *testing.T) { Version: "v1", Kind: "ConfigMap", } - instance := configFor([]schema.GroupVersionKind{ - configMapGVK, - }) + instance := configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) // Setup the Manager and Controller. mgr, wm := setupManager(t) @@ -604,13 +605,32 @@ func TestConfig_Retries(t *testing.T) { events := make(chan event.GenericEvent, 1024) watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() - cacheManager := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder}) + w, err := wm.NewRegistrar( + CtrlName, + events) + require.NoError(t, err) + cacheManager, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + Opa: opaClient, + SyncMetricsCache: syncMetricsCache, + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watchSet, + Registrar: w, + Reader: c, + }) + require.NoError(t, err) + go cacheManager.Start(ctx) - rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, events, watchSet, events) + rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) err = add(mgr, rec) if err != nil { t.Fatal(err) } + syncAdder := syncc.Adder{ + Events: events, + CacheManager: cacheManager, + } + require.NoError(t, syncAdder.Add(mgr), "registering sync controller") // Use our special hookReader to inject controlled failures failPlease := make(chan string, 1) @@ -630,7 +650,6 @@ func TestConfig_Retries(t *testing.T) { }, } - ctx, cancelFunc := context.WithCancel(context.Background()) testutils.StartManager(ctx, t, mgr) once := gosync.Once{} testMgrStopped := func() { @@ -642,7 +661,6 @@ func TestConfig_Retries(t *testing.T) { defer testMgrStopped() // Create the Config object and expect the Reconcile to be created - ctx = context.Background() g.Eventually(func() error { return c.Create(ctx, instance.DeepCopy()) }, timeout).Should(gomega.BeNil()) @@ -677,20 +695,11 @@ func TestConfig_Retries(t *testing.T) { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "checking initial opa cache contents") - // Wipe the opa cache, we want to see it repopulate despite transient replay errors below. - _, err = opaClient.RemoveData(ctx, target.WipeData()) - if err != nil { - t.Fatalf("wiping opa cache: %v", err) - } - if opaClient.Contains(expected) { - t.Fatal("wipe failed") - } - // Make List fail once for ConfigMaps as the replay occurs following the reconfig below. failPlease <- "ConfigMapList" - // Reconfigure to add a namespace watch. - instance = configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) + // Reconfigure to force an internal replay. + instance = configFor([]schema.GroupVersionKind{configMapGVK}) forUpdate := instance.DeepCopy() _, err = controllerutil.CreateOrUpdate(ctx, c, forUpdate, func() error { forUpdate.Spec = instance.Spec diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2b16e332c48..af9568ee5b9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -18,12 +18,15 @@ package controller import ( "context" "flag" + "fmt" "os" "sync" constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" @@ -39,6 +42,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -167,9 +171,33 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { deps.GetPod = fakePodGetter } + // Events will be used to receive events from dynamic watches registered + // via the registrar below. + events := make(chan event.GenericEvent, 1024) filteredOpa := syncutil.NewFilteredOpaDataClient(deps.Opa, deps.WatchSet) syncMetricsCache := syncutil.NewMetricsCache() - cm := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: filteredOpa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, ProcessExcluder: deps.ProcessExcluder}) + w, err := deps.WatchManger.NewRegistrar( + config.CtrlName, + events) + if err != nil { + return err + } + cm, err := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: filteredOpa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, ProcessExcluder: deps.ProcessExcluder, Registrar: w, WatchedSet: deps.WatchSet}) + if err != nil { + return err + } + if err := cm.Start(context.TODO()); err != nil { + return fmt.Errorf("error starting cache manager: %w", err) + } + + syncAdder := syncc.Adder{ + Events: events, + CacheManager: cm, + } + // Create subordinate controller - we will feed it events dynamically via watch + if err := syncAdder.Add(m); err != nil { + return fmt.Errorf("registering sync controller: %w", err) + } for _, a := range Injectors { a.InjectOpa(deps.Opa) diff --git a/pkg/syncutil/aggregator/aggregator.go b/pkg/syncutil/aggregator/aggregator.go index 07eb91c9dcd..9925991316d 100644 --- a/pkg/syncutil/aggregator/aggregator.go +++ b/pkg/syncutil/aggregator/aggregator.go @@ -46,7 +46,7 @@ func (b *GVKAgreggator) IsPresent(gvk schema.GroupVersionKind) bool { return found } -// Remove deletes the any associations that Key k has in the GVKAggregator. +// Remove deletes any associations that Key k has in the GVKAggregator. // For any GVK in the association k --> [GVKs], we also delete any associations // between the GVK and the Key k stored in the reverse map. func (b *GVKAgreggator) Remove(k Key) error { @@ -98,6 +98,35 @@ func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error { return nil } +// List returnes the gvk set for a given Key. +func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { + b.mu.Lock() + defer b.mu.Unlock() + + gvks, _ := b.store[k] + return gvks +} + +func (b *GVKAgreggator) ListAllGVKs() []schema.GroupVersionKind { + b.mu.Lock() + defer b.mu.Unlock() + + allGVKs := []schema.GroupVersionKind{} + for gvk := range b.reverseStore { + allGVKs = append(allGVKs, gvk) + } + return allGVKs +} + +func (b *GVKAgreggator) Clear() { + b.mu.Lock() + defer b.mu.Unlock() + + b.store = make(map[Key]map[schema.GroupVersionKind]struct{}) + b.reverseStore = make(map[schema.GroupVersionKind]map[Key]struct{}) +} + + func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struct{}, k Key) error { for gvk := range gvks { keySet, found := b.reverseStore[gvk] From 8e70e79386b1029282a45d9bd8554b5115836af7 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 5 Jul 2023 19:04:41 +0000 Subject: [PATCH 09/58] review: push all cache mgmt in bckgr dirty, squash: config_c changes Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 207 ++++++++---------- .../cachemanager/cachemanager_test.go | 151 ++++++++----- pkg/controller/config/config_controller.go | 11 +- .../config/config_controller_test.go | 19 +- 4 files changed, 213 insertions(+), 175 deletions(-) diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index e0313f1800f..68a1dccb391 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -3,6 +3,7 @@ package cachemanager import ( "context" "fmt" + "sync" "time" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" @@ -32,18 +33,22 @@ type CacheManagerConfig struct { } type CacheManager struct { + // the processExcluder and the gvkAggregator define what the underlying + // cache should look like. we refer to those two as "the spec" + processExcluder *process.Excluder + gvkAggregator *aggregator.GVKAgreggator + // mu guards access to any part of the spec above + mu sync.RWMutex + opa syncutil.OpaDataClient syncMetricsCache *syncutil.MetricsCache tracker *readiness.Tracker - processExcluder *process.Excluder registrar *watch.Registrar watchedSet *watch.Set - gvkAggregator *aggregator.GVKAgreggator - gvksToRemove *watch.Set - gvksToSync *watch.Set replayErrChan chan error replayTicker time.Ticker reader client.Reader + excluderChanged bool } func NewCacheManager(config *CacheManagerConfig) (*CacheManager, error) { @@ -74,8 +79,6 @@ func NewCacheManager(config *CacheManagerConfig) (*CacheManager, error) { } cm.gvkAggregator = aggregator.NewGVKAggregator() - cm.gvksToRemove = watch.NewSet() - cm.gvksToSync = watch.NewSet() cm.replayTicker = *time.NewTicker(3 * time.Second) @@ -89,106 +92,43 @@ func (c *CacheManager) Start(ctx context.Context) error { return nil } -// WatchGVKsToSync adjusts the watched set of gvks according to the newGVKs passed in +// AddSource adjusts the watched set of gvks according to the newGVKs passed in // for a given {syncSourceType, syncSourceName}. -func (c *CacheManager) WatchGVKsToSync(ctx context.Context, newGVKs []schema.GroupVersionKind, newExcluder *process.Excluder, syncSourceType, syncSourceName string) error { - netNewGVKs := []schema.GroupVersionKind{} - for _, gvk := range newGVKs { - if !c.gvkAggregator.IsPresent(gvk) { - netNewGVKs = append(netNewGVKs, gvk) - } - } - // mark these gvks for the background goroutine to sync - c.gvksToSync.Add(netNewGVKs...) +func (c *CacheManager) AddSource(ctx context.Context, newGVKs []schema.GroupVersionKind, syncSourceType, syncSourceName string) error { + c.mu.Lock() + defer c.mu.Unlock() opKey := aggregator.Key{Source: syncSourceType, ID: syncSourceName} - currentGVKsForKey := c.gvkAggregator.List(opKey) - - if len(newGVKs) == 0 { - // we are not syncing anything for this key anymore - if err := c.gvkAggregator.Remove(opKey); err != nil { - return fmt.Errorf("internal error removing gvks for aggregation: %w", err) - } - } else { - if err := c.gvkAggregator.Upsert(opKey, newGVKs); err != nil { - return fmt.Errorf("internal error upserting gvks for aggregation: %w", err) - } - } - // stage the new watch set for events for the sync_controller to be - // the current watch set ... [1/3] - newGvkWatchSet := watch.NewSet() - newGvkWatchSet.AddSet(c.watchedSet) - // ... plus the net new gvks we are adding ... [2/3] - newGvkWatchSet.Add(netNewGVKs...) - - gvksToDeleteCandidates := getGVKsToDeleteCandidates(newGVKs, currentGVKsForKey) - gvksToDeleteSet := watch.NewSet() - for _, gvk := range gvksToDeleteCandidates { - // if this gvk is no longer required by any source, schedule it to be deleted - if !c.gvkAggregator.IsPresent(gvk) { - // Remove expectations for resources we no longer watch. - c.tracker.CancelData(gvk) - // mark these gvks for the background goroutine to un-sync - gvksToDeleteSet.Add(gvk) - } - } - c.gvksToRemove.AddSet(gvksToDeleteSet) - - // ... less the gvks to delete. [3/3] - newGvkWatchSet.RemoveSet(gvksToDeleteSet) - - // If the watch set has not changed AND the process excluder is the same we're done here. - if c.watchedSet.Equals(newGvkWatchSet) && newExcluder != nil { - if c.processExcluder.Equals(newExcluder) { - return nil - } else { - // there is a new excluder which means we need to schedule a wipe for any - // previously watched GVKs to be re-added to get a chance to be evaluated - // for this new process excluder. - - c.gvksToRemove.AddSet(newGvkWatchSet) - } + if err := c.gvkAggregator.Upsert(opKey, newGVKs); err != nil { + return fmt.Errorf("internal error adding source: %w", err) } - // Start watching the newly added gvks set - var innerError error - c.watchedSet.Replace(newGvkWatchSet, func() { - // swapping with the new excluder - if newExcluder != nil { - c.processExcluder.Replace(newExcluder) - } - - // *Note the following steps are not transactional with respect to admission control* + return nil +} - // Important: dynamic watches update must happen *after* updating our watchSet. - // Otherwise, the sync controller will drop events for the newly watched kinds. - // Defer error handling so object re-sync happens even if the watch is hard - // errored due to a missing GVK in the watch set. - innerError = c.registrar.ReplaceWatch(ctx, newGvkWatchSet.Items()) - }) - if innerError != nil { - return innerError +func (c *CacheManager) RemoveSource(ctx context.Context, syncSourceType, syncSourceName string) error { + c.mu.Lock() + defer c.mu.Unlock() + if err := c.gvkAggregator.Remove(aggregator.Key{Source: syncSourceType, ID: syncSourceName}); err != nil { + return fmt.Errorf("internal error removing source: %w", err) } return nil } -// returns GVKs that are in currentGVKsForKey but not in newGVKs. -func getGVKsToDeleteCandidates(newGVKs []schema.GroupVersionKind, currentGVKsForKey map[schema.GroupVersionKind]struct{}) []schema.GroupVersionKind { - newGVKSet := make(map[schema.GroupVersionKind]struct{}) - for _, gvk := range newGVKs { - newGVKSet[gvk] = struct{}{} - } - - var toDelete []schema.GroupVersionKind - for gvk := range currentGVKsForKey { - if _, found := newGVKSet[gvk]; !found { - toDelete = append(toDelete, gvk) - } +func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { + if c.processExcluder.Equals(newExcluder) { + return } - return toDelete + c.mu.Lock() + c.processExcluder.Replace(newExcluder) + // there is a new excluder which means we need to schedule a wipe for any + // previously watched GVKs to be re-added to get a chance to be evaluated + // for this new process excluder. + c.excluderChanged = true + c.mu.Unlock() } func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Unstructured) error { @@ -241,7 +181,7 @@ func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured. return nil } -func (c *CacheManager) WipeData(ctx context.Context) error { +func (c *CacheManager) wipeData(ctx context.Context) error { if _, err := c.opa.RemoveData(ctx, target.WipeData()); err != nil { return err } @@ -287,7 +227,13 @@ func (c *CacheManager) updateDatastore(ctx context.Context) { case <-ctx.Done(): return case <-c.replayTicker.C: - c.makeUpdates(ctx) + c.mu.RLock() + currentGVKsInAgg := watch.NewSet() + currentGVKsInAgg.Add(c.gvkAggregator.ListAllGVKs()...) + excluderChanged := c.excluderChanged + c.mu.RUnlock() + + c.makeUpdates(ctx, currentGVKsInAgg, excluderChanged) } } } @@ -310,27 +256,62 @@ func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupV } // makeUpdates performs a conditional wipe followed by a replay if necessary. -func (c *CacheManager) makeUpdates(ctx context.Context) { - // first, wipe the cache if needed - gvksToDelete := c.gvksToRemove.Items() - if len(gvksToDelete) > 0 { - // "checkpoint save" what needs to be replayed - gvksToReplay := c.gvkAggregator.ListAllGVKs() - // and add it to be synced below - c.gvksToSync.Add(gvksToReplay...) - - if err := c.WipeData(ctx); err != nil { +func (c *CacheManager) makeUpdates(ctx context.Context, currentGVKsInAgg *watch.Set, excluderChanged bool) { + if c.watchedSet.Equals(currentGVKsInAgg) && !excluderChanged { + return // nothing to do if both sets are the same and the excluder didn't change + } + + // replace the current watch set for the sync_controller to pick up + // any updates on said GVKs. + // also save the current watch set to make cache changes later + oldWatchSet := watch.NewSet() + oldWatchSet.AddSet(c.watchedSet) + + var innerError error + c.watchedSet.Replace(currentGVKsInAgg, func() { + // *Note the following steps are not transactional with respect to admission control* + + // Important: dynamic watches update must happen *after* updating our watchSet. + // Otherwise, the sync controller will drop events for the newly watched kinds. + // Defer error handling so object re-sync happens even if the watch is hard + // errored due to a missing GVK in the watch set. + innerError = c.registrar.ReplaceWatch(ctx, currentGVKsInAgg.Items()) + }) + if innerError != nil { + log.Error(innerError, "internal: error replacing watch set") + } + + gvksToDelete := oldWatchSet.Difference(currentGVKsInAgg).Items() + newGVKsToSync := currentGVKsInAgg.Difference(oldWatchSet) + + // remove any gvks not needing to be synced anymore + // or re evaluate all if the excluder changed. + if len(gvksToDelete) > 0 || excluderChanged { + if err := c.wipeData(ctx); err != nil { log.Error(err, "internal: error wiping cache") - // don't alter the toRemove set, we will try again - } else { - c.gvksToRemove.Remove(gvksToDelete...) - // any gvks that were just removed shouldn't be synced - c.gvksToSync.Remove(gvksToDelete...) } + + if excluderChanged { + c.unsetExcluderChanged() + } + + // everything that gets wiped needs to be readded + newGVKsToSync.AddSet(currentGVKsInAgg) + } + + // sync net new gvks and potentially replayed gvks from the cache wipe above + gvksSynced := c.listAndSyncData(ctx, newGVKsToSync.Items(), c.reader) + + gvksNotSynced := gvksSynced.Difference(newGVKsToSync) + for _, gvk := range gvksNotSynced.Items() { + log.Info(fmt.Sprintf("failed to sync gvk: %s; will retry", gvk)) } +} + +func (c *CacheManager) unsetExcluderChanged() { + c.mu.Lock() + defer c.mu.Unlock() - // sync net new gvks - gvksToSyncList := c.gvksToSync.Items() - gvksSynced := c.listAndSyncData(ctx, gvksToSyncList, c.reader) - c.gvksToSync.RemoveSet(gvksSynced) + // unset the excluderChanged bool now + c.excluderChanged = false } diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index c0999766f09..1929f8e2baa 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -126,7 +126,7 @@ func TestCacheManager_listAndSyncData(t *testing.T) { require.True(t, opaClient.Contains(expected)) // wipe cache - require.NoError(t, cacheManager.WipeData(ctx)) + require.NoError(t, cacheManager.wipeData(ctx)) require.False(t, opaClient.Contains(expected)) // create a second GVK @@ -170,50 +170,43 @@ func TestCacheManager_listAndSyncData(t *testing.T) { // TestCacheManager_makeUpdates tests that we can remove and add gvks to the data store. func TestCacheManager_makeUpdates(t *testing.T) { cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + // seed one gvk configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - fooGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Foo"} - barGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Bar"} - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + _, err := opaClient.AddData(ctx, cm) + require.NoError(t, err, "creating ConfigMap config-test-1 in opa") + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} pod := unstructuredFor(podGVK, "pod-1") require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - // seed gvks - cacheManager.gvksToRemove.Add(fooGVK, barGVK) - cacheManager.gvksToSync.Add(configMapGVK) + // prep gvkAggregator for updates to be picked up in makeUpdates cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK}) - // after the updates we should not see any gvks that - // were removed but should see what was in the aggregator - // and the new gvks to sync. - cacheManager.makeUpdates(ctx) - - // check that the "work queues" were drained - require.Len(t, cacheManager.gvksToSync.Items(), 0) - require.Len(t, cacheManager.gvksToRemove.Items(), 0) + gvksInAgg := watch.NewSet() + gvksInAgg.Add(cacheManager.gvkAggregator.ListAllGVKs()...) + cacheManager.makeUpdates(ctx, gvksInAgg, false) // expect the following instances to be in the data store expected := map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) - require.True(t, ok) require.Equal(t, 2, opaClient.Len()) require.True(t, opaClient.Contains(expected)) // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + // cm was not actually created thru the client, so no need for this. + // require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") } -// TestCacheManager_WatchGVKsToSync also tests replay. -func TestCacheManager_WatchGVKsToSync(t *testing.T) { +// TestCacheManager_AddSourceRemoveSource makes sure that we can add and remove multiple sources +// and changes to the underlying cache are reflected. +func TestCacheManager_AddSourceRemoveSource(t *testing.T) { cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} @@ -233,7 +226,7 @@ func TestCacheManager_WatchGVKsToSync(t *testing.T) { require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} - require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}, nil, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}, syncSourceOne.Source, syncSourceOne.ID)) expected := map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -241,18 +234,7 @@ func TestCacheManager_WatchGVKsToSync(t *testing.T) { {Gvk: podGVK, Key: "default/pod-1"}: nil, } - reqCondition := func(expected map[fakes.OpaKey]interface{}) func() bool { - return func() bool { - if opaClient.Len() != len(expected) { - return false - } - if opaClient.Contains(expected) { - return true - } - return false - } - } - require.Eventually(t, reqCondition(expected), testutils.TenSecondWaitFor, testutils.OneSecondTick) + require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) // now assert that the gvkAggregator looks as expected cacheManager.gvkAggregator.IsPresent(configMapGVK) @@ -264,13 +246,13 @@ func TestCacheManager_WatchGVKsToSync(t *testing.T) { require.True(t, foundPod) // now remove the podgvk and make sure we don't have pods in the cache anymore - require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{configMapGVK}, nil, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK}, syncSourceOne.Source, syncSourceOne.ID)) expected = map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } - require.Eventually(t, reqCondition(expected), testutils.TenSecondWaitFor, testutils.OneSecondTick) + require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) // now assert that the gvkAggregator looks as expected cacheManager.gvkAggregator.IsPresent(configMapGVK) gvks = cacheManager.gvkAggregator.List(syncSourceOne) @@ -282,7 +264,7 @@ func TestCacheManager_WatchGVKsToSync(t *testing.T) { // now make sure that adding another sync source with the same gvk has no side effects syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{configMapGVK}, nil, syncSourceTwo.Source, syncSourceTwo.ID)) + require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK}, syncSourceTwo.Source, syncSourceTwo.ID)) reqConditionForAgg := func() bool { cacheManager.gvkAggregator.IsPresent(configMapGVK) @@ -306,43 +288,106 @@ func TestCacheManager_WatchGVKsToSync(t *testing.T) { return true } - require.Eventually(t, reqConditionForAgg, testutils.TenSecondWaitFor, testutils.OneSecondTick) + require.Eventually(t, reqConditionForAgg, testutils.TenSecond, testutils.OneSecond) - // now add pod2 - require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{podGVK}, nil, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{podGVK}, syncSourceOne.Source, syncSourceOne.ID)) expected2 := map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, reqCondition(expected2), testutils.TenSecondWaitFor, testutils.OneSecondTick) + require.Eventually(t, expectedCheck(opaClient, expected2), testutils.TenSecond, testutils.OneSecond) // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed - require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{}, nil, syncSourceTwo.Source, syncSourceTwo.ID)) + require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{}, syncSourceTwo.Source, syncSourceTwo.ID)) expected3 := map[fakes.OpaKey]interface{}{ // config maps no longer required by any sync source //{Gvk: configMapGVK, Key: "default/config-test-1"}: nil, //{Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, reqCondition(expected3), testutils.TenSecondWaitFor, testutils.OneSecondTick) + require.Eventually(t, expectedCheck(opaClient, expected3), testutils.TenSecond, testutils.OneSecond) + + // now remove all the sources + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo.Source, syncSourceTwo.ID)) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne.Source, syncSourceOne.ID)) + + // and expect an empty cache and empty aggregator + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.TenSecond, testutils.OneSecond) + require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 0) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") +} + +// TestCacheManager_ExcludeProcesses makes sure that changing the process excluder +// in the cache manager triggers a re-evaluation of GVKs. +func TestCacheManager_ExcludeProcesses(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + } + require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK}, "aSourceType", "aSourceName")) + // check that everything is well added at first + require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) + + // make sure that replacing w same process excluder is a no op + sameExcluder := process.New() + sameExcluder.Add([]configv1alpha1.MatchEntry{ + // same excluder as the one in makeCacheManagerForTest + { + ExcludedNamespaces: []util.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + cacheManager.ExcludeProcesses(sameExcluder) + require.True(t, cacheManager.processExcluder.Equals(sameExcluder)) - // now process exclude the remaing gvk, it should get removed now - blankExcluder := process.New() - blankExcluder.Add([]configv1alpha1.MatchEntry{ + // now process exclude the remaing gvk, it should get removed by the background process. + excluder := process.New() + excluder.Add([]configv1alpha1.MatchEntry{ + // exclude the "default" namespace { ExcludedNamespaces: []util.Wildcard{"default"}, Processes: []string{"sync"}, }, + { + ExcludedNamespaces: []util.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, }) - require.NoError(t, cacheManager.WatchGVKsToSync(ctx, []schema.GroupVersionKind{podGVK}, blankExcluder, syncSourceOne.Source, syncSourceOne.ID)) - expected4 := map[fakes.OpaKey]interface{}{} - require.Eventually(t, reqCondition(expected4), testutils.TenSecondWaitFor, testutils.OneSecondTick) + cacheManager.ExcludeProcesses(excluder) + + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.TenSecond, testutils.ThreeSecond) + // make sure the gvk is still in gvkAggregator + require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 1) + require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") +} + +func expectedCheck(opaClient *fakes.FakeOpa, expected map[fakes.OpaKey]interface{}) func() bool { + return func() bool { + if opaClient.Len() != len(expected) { + return false + } + if opaClient.Contains(expected) { + return true + } + + return false + } } func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index f3f59c12e08..a67b3b08253 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -225,8 +225,15 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.tracker.DisableStats() } - if err := r.cacheManager.WatchGVKsToSync(ctx, gvksToSync, newExcluder, "config", request.NamespacedName.Name); err != nil { - return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) + r.cacheManager.ExcludeProcesses(newExcluder) + if len(gvksToSync) > 0 { + if err := r.cacheManager.AddSource(ctx, gvksToSync, "config", request.NamespacedName.Name); err != nil { + return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) + } + } else { + if err := r.cacheManager.RemoveSource(ctx, "config", request.NamespacedName.Name); err != nil { + return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error removing syncOny gvks from sync process: %w", err) + } } return reconcile.Result{}, nil diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 0f15d1aeff8..814701e26d1 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -94,7 +94,9 @@ func setupManager(t *testing.T) (manager.Manager, *watch.Manager) { } func TestReconcile(t *testing.T) { + ctx, cancelFunc := context.WithCancel(context.Background()) g := gomega.NewGomegaWithT(t) + instance := &configv1alpha1.Config{ ObjectMeta: metav1.ObjectMeta{ Name: "config", @@ -151,15 +153,16 @@ func TestReconcile(t *testing.T) { Reader: c, }) require.NoError(t, err) - rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) + + // start the cache manager + go cacheManager.Start(ctx) + + rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) + require.NoError(t, err) recFn, requests := SetupTestReconcile(rec) - err = add(mgr, recFn) - if err != nil { - t.Fatal(err) - } + require.NoError(t, add(mgr, recFn)) - ctx, cancelFunc := context.WithCancel(context.Background()) testutils.StartManager(ctx, t, mgr) once := gosync.Once{} testMgrStopped := func() { @@ -185,8 +188,10 @@ func TestReconcile(t *testing.T) { }() g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + g.Eventually(func() int { + return len(wm.GetManagedGVK()) + }).WithTimeout(timeout).ShouldNot(gomega.Equal(0)) gvks := wm.GetManagedGVK() - g.Eventually(len(gvks), timeout).ShouldNot(gomega.Equal(0)) wantGVKs := []schema.GroupVersionKind{ {Group: "", Version: "v1", Kind: "Namespace"}, From 1cd478a610257036d89e34e74db6d2ee2107e730 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:27:12 +0000 Subject: [PATCH 10/58] add cacheManager as a runnable Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/controller.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index af9568ee5b9..68248f86b2a 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -186,8 +186,11 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { if err != nil { return err } - if err := cm.Start(context.TODO()); err != nil { - return fmt.Errorf("error starting cache manager: %w", err) + + // Adding the CacheManager as a runnable; + // manager will start CacheManager. + if err := m.Add(cm); err != nil { + return fmt.Errorf("error adding cache manager as a runnable: %w", err) } syncAdder := syncc.Adder{ From 9eb8f35e759f30e40bea4933f4696045601c38b4 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 6 Jul 2023 18:49:06 +0000 Subject: [PATCH 11/58] rename makeUpdates, comments Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 9 ++++++--- pkg/controller/cachemanager/cachemanager_test.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index 68a1dccb391..9876c76a44f 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -227,13 +227,15 @@ func (c *CacheManager) updateDatastore(ctx context.Context) { case <-ctx.Done(): return case <-c.replayTicker.C: + // snapshot the current spec so we can make a step upgrade + // to the contests of the opa cache. c.mu.RLock() currentGVKsInAgg := watch.NewSet() currentGVKsInAgg.Add(c.gvkAggregator.ListAllGVKs()...) excluderChanged := c.excluderChanged c.mu.RUnlock() - c.makeUpdates(ctx, currentGVKsInAgg, excluderChanged) + c.makeUpdatesForSpecInTime(ctx, currentGVKsInAgg, excluderChanged) } } } @@ -255,8 +257,9 @@ func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupV return gvksSuccessfullySynced } -// makeUpdates performs a conditional wipe followed by a replay if necessary. -func (c *CacheManager) makeUpdates(ctx context.Context, currentGVKsInAgg *watch.Set, excluderChanged bool) { +// makeUpdatesForSpecInTime performs a conditional wipe followed by a replay if necessary as +// given by the current spec (currentGVKsInAgg, excluderChanged) at the time of the call. +func (c *CacheManager) makeUpdatesForSpecInTime(ctx context.Context, currentGVKsInAgg *watch.Set, excluderChanged bool) { if c.watchedSet.Equals(currentGVKsInAgg) && !excluderChanged { return // nothing to do if both sets are the same and the excluder didn't change } diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index 1929f8e2baa..39247fc05be 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -188,7 +188,7 @@ func TestCacheManager_makeUpdates(t *testing.T) { gvksInAgg := watch.NewSet() gvksInAgg.Add(cacheManager.gvkAggregator.ListAllGVKs()...) - cacheManager.makeUpdates(ctx, gvksInAgg, false) + cacheManager.makeUpdatesForSpecInTime(ctx, gvksInAgg, false) // expect the following instances to be in the data store expected := map[fakes.OpaKey]interface{}{ From 7fbb1c48cf2602d4278cbcb4111a843244bf29ff Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 6 Jul 2023 20:56:22 +0000 Subject: [PATCH 12/58] pass reader for cm Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/controller.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 68248f86b2a..ebb85cd8136 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -182,7 +182,15 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { if err != nil { return err } - cm, err := cm.NewCacheManager(&cm.CacheManagerConfig{Opa: filteredOpa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, ProcessExcluder: deps.ProcessExcluder, Registrar: w, WatchedSet: deps.WatchSet}) + cm, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + Opa: filteredOpa, + SyncMetricsCache: syncMetricsCache, + Tracker: deps.Tracker, + ProcessExcluder: deps.ProcessExcluder, + Registrar: w, + WatchedSet: deps.WatchSet, + Reader: m.GetCache(), + }) if err != nil { return err } From ba4981d264d1dd03c29965f5954aab269f6298e7 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 6 Jul 2023 21:41:05 +0000 Subject: [PATCH 13/58] review: consumer defines source, others Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 18 +++++++-------- .../cachemanager/cachemanager_test.go | 22 ++++++++++--------- pkg/controller/config/config_controller.go | 6 +++-- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index 9876c76a44f..fdb0be9b87a 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -93,24 +93,22 @@ func (c *CacheManager) Start(ctx context.Context) error { } // AddSource adjusts the watched set of gvks according to the newGVKs passed in -// for a given {syncSourceType, syncSourceName}. -func (c *CacheManager) AddSource(ctx context.Context, newGVKs []schema.GroupVersionKind, syncSourceType, syncSourceName string) error { +// for a given sourceKey. +func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() - opKey := aggregator.Key{Source: syncSourceType, ID: syncSourceName} - - if err := c.gvkAggregator.Upsert(opKey, newGVKs); err != nil { + if err := c.gvkAggregator.Upsert(sourceKey, newGVKs); err != nil { return fmt.Errorf("internal error adding source: %w", err) } return nil } -func (c *CacheManager) RemoveSource(ctx context.Context, syncSourceType, syncSourceName string) error { +func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() - if err := c.gvkAggregator.Remove(aggregator.Key{Source: syncSourceType, ID: syncSourceName}); err != nil { + if err := c.gvkAggregator.Remove(sourceKey); err != nil { return fmt.Errorf("internal error removing source: %w", err) } @@ -197,7 +195,7 @@ func (c *CacheManager) ReportSyncMetrics() { c.syncMetricsCache.ReportSync() } -func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.GroupVersionKind, reader client.Reader) error { +func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.GroupVersionKind) error { u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: gvk.Group, @@ -205,7 +203,7 @@ func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.Gro Kind: gvk.Kind + "List", }) - err := reader.List(ctx, u) + err := c.reader.List(ctx, u) if err != nil { return fmt.Errorf("replaying data for %+v: %w", gvk, err) } @@ -244,7 +242,7 @@ func (c *CacheManager) updateDatastore(ctx context.Context) { func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupVersionKind, reader client.Reader) *watch.Set { gvksSuccessfullySynced := watch.NewSet() for _, gvk := range gvks { - err := c.listAndSyncDataForGVK(ctx, gvk, c.reader) + err := c.listAndSyncDataForGVK(ctx, gvk) if err != nil { log.Error(err, "internal: error syncing gvks cache data") // we don't remove this gvk as we will try to re-add it later diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index 39247fc05be..bbdb05de697 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -113,7 +113,7 @@ func TestCacheManager_listAndSyncData(t *testing.T) { cm2 := unstructuredFor(configMapGVK, "config-test-2") require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") - require.NoError(t, cacheManager.listAndSyncDataForGVK(ctx, configMapGVK, c)) + require.NoError(t, cacheManager.listAndSyncDataForGVK(ctx, configMapGVK)) opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -226,7 +226,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} - require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) expected := map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -246,7 +246,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.True(t, foundPod) // now remove the podgvk and make sure we don't have pods in the cache anymore - require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK}, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected = map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -264,7 +264,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now make sure that adding another sync source with the same gvk has no side effects syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK}, syncSourceTwo.Source, syncSourceTwo.ID)) + require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) reqConditionForAgg := func() bool { cacheManager.gvkAggregator.IsPresent(configMapGVK) @@ -290,7 +290,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { } require.Eventually(t, reqConditionForAgg, testutils.TenSecond, testutils.OneSecond) - require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{podGVK}, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) expected2 := map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, @@ -299,7 +299,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, expected2), testutils.TenSecond, testutils.OneSecond) // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed - require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{}, syncSourceTwo.Source, syncSourceTwo.ID)) + require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) expected3 := map[fakes.OpaKey]interface{}{ // config maps no longer required by any sync source //{Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -309,8 +309,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, expected3), testutils.TenSecond, testutils.OneSecond) // now remove all the sources - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo.Source, syncSourceTwo.ID)) - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne.Source, syncSourceOne.ID)) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) // and expect an empty cache and empty aggregator require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.TenSecond, testutils.OneSecond) @@ -337,7 +337,9 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { expected := map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } - require.NoError(t, cacheManager.AddSource(ctx, []schema.GroupVersionKind{configMapGVK}, "aSourceType", "aSourceName")) + + syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} + require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) // check that everything is well added at first require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) @@ -353,7 +355,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { cacheManager.ExcludeProcesses(sameExcluder) require.True(t, cacheManager.processExcluder.Equals(sameExcluder)) - // now process exclude the remaing gvk, it should get removed by the background process. + // now process exclude the remaining gvk, it should get removed by the background process. excluder := process.New() excluder.Add([]configv1alpha1.MatchEntry{ // exclude the "default" namespace diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index a67b3b08253..00e7981ab15 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -28,6 +28,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -226,12 +227,13 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque } r.cacheManager.ExcludeProcesses(newExcluder) + configSourceKey := aggregator.Key{Source: "config", ID: request.NamespacedName.String()} if len(gvksToSync) > 0 { - if err := r.cacheManager.AddSource(ctx, gvksToSync, "config", request.NamespacedName.Name); err != nil { + if err := r.cacheManager.AddSource(ctx, configSourceKey, gvksToSync); err != nil { return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) } } else { - if err := r.cacheManager.RemoveSource(ctx, "config", request.NamespacedName.Name); err != nil { + if err := r.cacheManager.RemoveSource(ctx, configSourceKey); err != nil { return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error removing syncOny gvks from sync process: %w", err) } } From cf11ef1e6ec66fb1ae26fa3699fa7f5c891b49c5 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:31:03 +0000 Subject: [PATCH 14/58] lint fixes & more Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 10 +-- .../cachemanager/cachemanager_test.go | 36 ++++---- .../config/config_controller_test.go | 83 +++---------------- pkg/controller/controller.go | 2 +- pkg/syncutil/aggregator/aggregator.go | 12 +-- test/testutils/const.go | 8 ++ 6 files changed, 46 insertions(+), 105 deletions(-) create mode 100644 test/testutils/const.go diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index fdb0be9b87a..558ea77b3ca 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -21,7 +21,7 @@ import ( var log = logf.Log.WithName("cache-manager") -type CacheManagerConfig struct { +type Config struct { Opa syncutil.OpaDataClient SyncMetricsCache *syncutil.MetricsCache Tracker *readiness.Tracker @@ -45,13 +45,12 @@ type CacheManager struct { tracker *readiness.Tracker registrar *watch.Registrar watchedSet *watch.Set - replayErrChan chan error replayTicker time.Ticker reader client.Reader excluderChanged bool } -func NewCacheManager(config *CacheManagerConfig) (*CacheManager, error) { +func NewCacheManager(config *Config) (*CacheManager, error) { if config.WatchedSet == nil { return nil, fmt.Errorf("watchedSet must be non-nil") } @@ -116,17 +115,18 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke } func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { + c.mu.Lock() + defer c.mu.Unlock() + if c.processExcluder.Equals(newExcluder) { return } - c.mu.Lock() c.processExcluder.Replace(newExcluder) // there is a new excluder which means we need to schedule a wipe for any // previously watched GVKs to be re-added to get a chance to be evaluated // for this new process excluder. c.excluderChanged = true - c.mu.Unlock() } func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Unstructured) error { diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index bbdb05de697..e77a75ad3ac 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -184,7 +184,7 @@ func TestCacheManager_makeUpdates(t *testing.T) { require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") // prep gvkAggregator for updates to be picked up in makeUpdates - cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK}) + require.NoError(t, cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) gvksInAgg := watch.NewSet() gvksInAgg.Add(cacheManager.gvkAggregator.ListAllGVKs()...) @@ -234,7 +234,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) // now assert that the gvkAggregator looks as expected cacheManager.gvkAggregator.IsPresent(configMapGVK) @@ -252,7 +252,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) // now assert that the gvkAggregator looks as expected cacheManager.gvkAggregator.IsPresent(configMapGVK) gvks = cacheManager.gvkAggregator.List(syncSourceOne) @@ -282,13 +282,9 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { return false } _, found2 := gvks2[configMapGVK] - if !found2 { - return false - } - - return true + return found2 } - require.Eventually(t, reqConditionForAgg, testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, reqConditionForAgg, testutils.EventuallyTimeout, testutils.EventuallyTicker) require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) expected2 := map[fakes.OpaKey]interface{}{ @@ -296,24 +292,24 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected2), testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, expectedCheck(opaClient, expected2), testutils.EventuallyTimeout, testutils.EventuallyTicker) // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) expected3 := map[fakes.OpaKey]interface{}{ // config maps no longer required by any sync source - //{Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - //{Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected3), testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, expectedCheck(opaClient, expected3), testutils.EventuallyTimeout, testutils.EventuallyTicker) // now remove all the sources require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 0) // cleanup @@ -341,7 +337,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) // check that everything is well added at first - require.Eventually(t, expectedCheck(opaClient, expected), testutils.TenSecond, testutils.OneSecond) + require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) // make sure that replacing w same process excluder is a no op sameExcluder := process.New() @@ -370,7 +366,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { }) cacheManager.ExcludeProcesses(excluder) - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.TenSecond, testutils.ThreeSecond) + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) // make sure the gvk is still in gvkAggregator require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 1) require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) @@ -429,7 +425,8 @@ func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*Cach w, err := wm.NewRegistrar( "test-cache-manager", events) - cacheManager, err := NewCacheManager(&CacheManagerConfig{ + require.NoError(t, err) + cacheManager, err := NewCacheManager(&Config{ Opa: opaClient, SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, @@ -441,7 +438,10 @@ func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*Cach require.NoError(t, err) if startCache { - go cacheManager.Start(ctx) + go func() { + require.NoError(t, cacheManager.Start(ctx)) + }() + t.Cleanup(func() { ctx.Done() }) diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 814701e26d1..0b96681460b 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -42,7 +42,6 @@ import ( "golang.org/x/net/context" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -143,7 +142,7 @@ func TestReconcile(t *testing.T) { CtrlName, events) require.NoError(t, err) - cacheManager, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + cacheManager, err := cm.NewCacheManager(&cm.Config{ Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, @@ -155,7 +154,9 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) // start the cache manager - go cacheManager.Start(ctx) + go func() { + require.NoError(t, cacheManager.Start(ctx)) + }() rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) require.NoError(t, err) @@ -435,7 +436,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager if err != nil { return nil, fmt.Errorf("cannot create registrar: %w", err) } - cacheManager, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + cacheManager, err := cm.NewCacheManager(&cm.Config{ Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, @@ -447,7 +448,10 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager if err != nil { return nil, fmt.Errorf("error creating cache manager: %w", err) } - go cacheManager.Start(ctx) + go func() { + _ = cacheManager.Start(ctx) + }() + rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) if err != nil { return nil, fmt.Errorf("creating reconciler: %w", err) @@ -614,7 +618,7 @@ func TestConfig_Retries(t *testing.T) { CtrlName, events) require.NoError(t, err) - cacheManager, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + cacheManager, err := cm.NewCacheManager(&cm.Config{ Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, @@ -624,7 +628,9 @@ func TestConfig_Retries(t *testing.T) { Reader: c, }) require.NoError(t, err) - go cacheManager.Start(ctx) + go func() { + require.NoError(t, cacheManager.Start(ctx)) + }() rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) err = add(mgr, rec) @@ -764,66 +770,3 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns type testExpectations interface { IsExpecting(gvk schema.GroupVersionKind, nsName types.NamespacedName) bool } - -// ensureDeleted -// -// This package uses the same API server process across multiple test functions. -// The residual state from a previous test function can cause flakes. -// -// To ensure a clean slate, we must verify that any previously applied Config object -// has been fully removed before applying our new object. -func ensureDeleted(ctx context.Context, c client.Client, toDelete client.Object) func() error { - gvk := toDelete.GetObjectKind().GroupVersionKind() - key := client.ObjectKeyFromObject(toDelete) - - return func() error { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - - err := c.Get(ctx, key, u) - if apierrors.IsNotFound(err) { - return nil - } else if err != nil { - return err - } - - if !u.GetDeletionTimestamp().IsZero() { - return fmt.Errorf("waiting for deletion: %v %v", gvk, key) - } - - err = c.Delete(ctx, u) - if err != nil { - return fmt.Errorf("deleting %v %v: %w", gvk, key, err) - } - - return fmt.Errorf("queued %v %v for deletion", gvk, key) - } -} - -// ensureCreated attempts to create toCreate in Client c as toCreate existed when ensureCreated was called. -func ensureCreated(ctx context.Context, c client.Client, toCreate client.Object) func() error { - gvk := toCreate.GetObjectKind().GroupVersionKind() - key := client.ObjectKeyFromObject(toCreate) - - // As ensureCreated returns a closure, it is possible that the value toCreate will be modified after ensureCreated - // is called but before the closure is called. Creating a copy here ensures the object to be created is consistent - // with the way it existed when ensureCreated was called. - toCreateCopy := toCreate.DeepCopyObject() - - return func() error { - instance, ok := toCreateCopy.(client.Object) - if !ok { - return fmt.Errorf("instance was %T which is not a client.Object", instance) - } - - err := c.Create(ctx, instance) - if apierrors.IsAlreadyExists(err) { - return fmt.Errorf("a copy of %v %v already exists - run ensureDeleted to ensure a fresh copy exists for testing", - gvk, key) - } else if err != nil { - return fmt.Errorf("creating %v %v: %w", gvk, key, err) - } - - return nil - } -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index ebb85cd8136..1946d08d32a 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -182,7 +182,7 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { if err != nil { return err } - cm, err := cm.NewCacheManager(&cm.CacheManagerConfig{ + cm, err := cm.NewCacheManager(&cm.Config{ Opa: filteredOpa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, diff --git a/pkg/syncutil/aggregator/aggregator.go b/pkg/syncutil/aggregator/aggregator.go index 9925991316d..57f2572618c 100644 --- a/pkg/syncutil/aggregator/aggregator.go +++ b/pkg/syncutil/aggregator/aggregator.go @@ -103,8 +103,7 @@ func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { b.mu.Lock() defer b.mu.Unlock() - gvks, _ := b.store[k] - return gvks + return b.store[k] } func (b *GVKAgreggator) ListAllGVKs() []schema.GroupVersionKind { @@ -118,15 +117,6 @@ func (b *GVKAgreggator) ListAllGVKs() []schema.GroupVersionKind { return allGVKs } -func (b *GVKAgreggator) Clear() { - b.mu.Lock() - defer b.mu.Unlock() - - b.store = make(map[Key]map[schema.GroupVersionKind]struct{}) - b.reverseStore = make(map[schema.GroupVersionKind]map[Key]struct{}) -} - - func (b *GVKAgreggator) pruneReverseStore(gvks map[schema.GroupVersionKind]struct{}, k Key) error { for gvk := range gvks { keySet, found := b.reverseStore[gvk] diff --git a/test/testutils/const.go b/test/testutils/const.go new file mode 100644 index 00000000000..e89412410b7 --- /dev/null +++ b/test/testutils/const.go @@ -0,0 +1,8 @@ +package testutils + +import "time" + +const ( + EventuallyTimeout = 10 * time.Second + EventuallyTicker = 1 * time.Second +) From 21b92e9aa571fa55adf2c9db9407205696e15466 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 6 Jul 2023 23:22:57 +0000 Subject: [PATCH 15/58] after origin/master rebase Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index e77a75ad3ac..a0314130adf 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -9,10 +9,9 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" - "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" @@ -344,7 +343,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { sameExcluder.Add([]configv1alpha1.MatchEntry{ // same excluder as the one in makeCacheManagerForTest { - ExcludedNamespaces: []util.Wildcard{"kube-system"}, + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, Processes: []string{"sync"}, }, }) @@ -356,11 +355,11 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { excluder.Add([]configv1alpha1.MatchEntry{ // exclude the "default" namespace { - ExcludedNamespaces: []util.Wildcard{"default"}, + ExcludedNamespaces: []wildcard.Wildcard{"default"}, Processes: []string{"sync"}, }, { - ExcludedNamespaces: []util.Wildcard{"kube-system"}, + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, Processes: []string{"sync"}, }, }) @@ -417,7 +416,7 @@ func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*Cach processExcluder := process.Get() processExcluder.Add([]configv1alpha1.MatchEntry{ { - ExcludedNamespaces: []util.Wildcard{"kube-system"}, + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, Processes: []string{"sync"}, }, }) From 29d747f53fbea72867eee137ad679fa6738d485a Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 11 Jul 2023 01:50:48 +0000 Subject: [PATCH 16/58] spli tests into unit, e2e Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 10 +- .../cachemanager/cachemanager_suite_test.go | 77 ++++++++++ .../cachemanager/cachemanager_test.go | 135 +----------------- .../cachemanager/cachemanager_unit_test.go | 81 +++++++++++ 4 files changed, 164 insertions(+), 139 deletions(-) create mode 100644 pkg/controller/cachemanager/cachemanager_suite_test.go create mode 100644 pkg/controller/cachemanager/cachemanager_unit_test.go diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index 558ea77b3ca..79acd74da33 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -225,15 +225,15 @@ func (c *CacheManager) updateDatastore(ctx context.Context) { case <-ctx.Done(): return case <-c.replayTicker.C: - // snapshot the current spec so we can make a step upgrade - // to the contests of the opa cache. + // snapshot the current spec so we can make a point in + // time change to the contents of the opa cache. c.mu.RLock() currentGVKsInAgg := watch.NewSet() currentGVKsInAgg.Add(c.gvkAggregator.ListAllGVKs()...) excluderChanged := c.excluderChanged c.mu.RUnlock() - c.makeUpdatesForSpecInTime(ctx, currentGVKsInAgg, excluderChanged) + c.makeUpdatesForSpec(ctx, currentGVKsInAgg, excluderChanged) } } } @@ -255,9 +255,9 @@ func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupV return gvksSuccessfullySynced } -// makeUpdatesForSpecInTime performs a conditional wipe followed by a replay if necessary as +// makeUpdatesForSpec performs a conditional wipe followed by a replay if necessary as // given by the current spec (currentGVKsInAgg, excluderChanged) at the time of the call. -func (c *CacheManager) makeUpdatesForSpecInTime(ctx context.Context, currentGVKsInAgg *watch.Set, excluderChanged bool) { +func (c *CacheManager) makeUpdatesForSpec(ctx context.Context, currentGVKsInAgg *watch.Set, excluderChanged bool) { if c.watchedSet.Equals(currentGVKsInAgg) && !excluderChanged { return // nothing to do if both sets are the same and the excluder didn't change } diff --git a/pkg/controller/cachemanager/cachemanager_suite_test.go b/pkg/controller/cachemanager/cachemanager_suite_test.go new file mode 100644 index 00000000000..5c4cc48ca4b --- /dev/null +++ b/pkg/controller/cachemanager/cachemanager_suite_test.go @@ -0,0 +1,77 @@ +package cachemanager + +import ( + "context" + "testing" + + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" + "github.com/stretchr/testify/require" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + testutils.StartControlPlane(m, &cfg, 3) +} + +func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*CacheManager, client.Client, context.Context) { + ctx, cancelFunc := context.WithCancel(context.Background()) + mgr, wm := testutils.SetupManager(t, cfg) + + c := testclient.NewRetryClient(mgr.GetClient()) + opaClient := &fakes.FakeOpa{} + tracker, err := readiness.SetupTracker(mgr, false, false, false) + require.NoError(t, err) + processExcluder := process.Get() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + events := make(chan event.GenericEvent, 1024) + w, err := wm.NewRegistrar( + "test-cache-manager", + events) + require.NoError(t, err) + cacheManager, err := NewCacheManager(&Config{ + Opa: opaClient, + SyncMetricsCache: syncutil.NewMetricsCache(), + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watch.NewSet(), + Registrar: w, + Reader: c, + }) + require.NoError(t, err) + + if startCache { + go func() { + require.NoError(t, cacheManager.Start(ctx)) + }() + + t.Cleanup(func() { + ctx.Done() + }) + } + + if startManager { + testutils.StartManager(ctx, t, mgr) + } + + t.Cleanup(func() { + cancelFunc() + }) + return cacheManager, c, ctx +} diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index a0314130adf..679752449a1 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -1,102 +1,20 @@ package cachemanager import ( - "context" "testing" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" - "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" - testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" ) -var cfg *rest.Config - -func TestMain(m *testing.M) { - testutils.StartControlPlane(m, &cfg, 3) -} - -// TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. -func TestCacheManager_AddObject_RemoveObject(t *testing.T) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - - pod := fakes.Pod( - fakes.WithNamespace("test-ns"), - fakes.WithName("test-name"), - ) - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - - // test that pod is cache managed - opaClient, ok := cm.opa.(*fakes.FakeOpa) - require.True(t, ok) - require.True(t, opaClient.HasGVK(pod.GroupVersionKind())) - - // now remove the object and verify it's removed - require.NoError(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) -} - -// TestCacheManager_processExclusion makes sure that we don't add objects that are process excluded. -func TestCacheManager_processExclusion(t *testing.T) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - processExcluder := process.Get() - processExcluder.Add([]configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{"test-ns-excluded"}, - Processes: []string{"sync"}, - }, - }) - cm.processExcluder.Replace(processExcluder) - - pod := fakes.Pod( - fakes.WithNamespace("test-ns-excluded"), - fakes.WithName("test-name"), - ) - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - - // test that pod from excluded namespace is not cache managed - opaClient, ok := cm.opa.(*fakes.FakeOpa) - require.True(t, ok) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) -} - -// TestCacheManager_errors tests that the cache manager responds to errors from the opa client. -func TestCacheManager_errors(t *testing.T) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - opaClient, ok := cm.opa.(*fakes.FakeOpa) - require.True(t, ok) - opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err - - pod := fakes.Pod( - fakes.WithNamespace("test-ns"), - fakes.WithName("test-name"), - ) - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - - // test that cm bubbles up the errors - require.ErrorContains(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") - require.ErrorContains(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") -} - // TestCacheManager_listAndSyncData tests that the cache manager can add gvks to the data store. func TestCacheManager_listAndSyncData(t *testing.T) { cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) @@ -187,7 +105,7 @@ func TestCacheManager_makeUpdates(t *testing.T) { gvksInAgg := watch.NewSet() gvksInAgg.Add(cacheManager.gvkAggregator.ListAllGVKs()...) - cacheManager.makeUpdatesForSpecInTime(ctx, gvksInAgg, false) + cacheManager.makeUpdatesForSpec(ctx, gvksInAgg, false) // expect the following instances to be in the data store expected := map[fakes.OpaKey]interface{}{ @@ -404,54 +322,3 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns } return u } - -func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*CacheManager, client.Client, context.Context) { - ctx, cancelFunc := context.WithCancel(context.Background()) - mgr, wm := testutils.SetupManager(t, cfg) - - c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &fakes.FakeOpa{} - tracker, err := readiness.SetupTracker(mgr, false, false, false) - require.NoError(t, err) - processExcluder := process.Get() - processExcluder.Add([]configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - events := make(chan event.GenericEvent, 1024) - w, err := wm.NewRegistrar( - "test-cache-manager", - events) - require.NoError(t, err) - cacheManager, err := NewCacheManager(&Config{ - Opa: opaClient, - SyncMetricsCache: syncutil.NewMetricsCache(), - Tracker: tracker, - ProcessExcluder: processExcluder, - WatchedSet: watch.NewSet(), - Registrar: w, - Reader: c, - }) - require.NoError(t, err) - - if startCache { - go func() { - require.NoError(t, cacheManager.Start(ctx)) - }() - - t.Cleanup(func() { - ctx.Done() - }) - } - - if startManager { - testutils.StartManager(ctx, t, mgr) - } - - t.Cleanup(func() { - cancelFunc() - }) - return cacheManager, c, ctx -} diff --git a/pkg/controller/cachemanager/cachemanager_unit_test.go b/pkg/controller/cachemanager/cachemanager_unit_test.go new file mode 100644 index 00000000000..bd786d5fd07 --- /dev/null +++ b/pkg/controller/cachemanager/cachemanager_unit_test.go @@ -0,0 +1,81 @@ +package cachemanager + +import ( + "testing" + + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. +func TestCacheManager_AddObject_RemoveObject(t *testing.T) { + cm, _, ctx := makeCacheManagerForTest(t, false, false) + + pod := fakes.Pod( + fakes.WithNamespace("test-ns"), + fakes.WithName("test-name"), + ) + unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) + + require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + + // test that pod is cache managed + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) + require.True(t, opaClient.HasGVK(pod.GroupVersionKind())) + + // now remove the object and verify it's removed + require.NoError(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) +} + +// TestCacheManager_processExclusion makes sure that we don't add objects that are process excluded. +func TestCacheManager_processExclusion(t *testing.T) { + cm, _, ctx := makeCacheManagerForTest(t, false, false) + processExcluder := process.Get() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"test-ns-excluded"}, + Processes: []string{"sync"}, + }, + }) + cm.processExcluder.Replace(processExcluder) + + pod := fakes.Pod( + fakes.WithNamespace("test-ns-excluded"), + fakes.WithName("test-name"), + ) + unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) + require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + + // test that pod from excluded namespace is not cache managed + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) + require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) +} + +// TestCacheManager_errors tests that the cache manager responds to errors from the opa client. +func TestCacheManager_errors(t *testing.T) { + cm, _, ctx := makeCacheManagerForTest(t, false, false) + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) + opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err + + pod := fakes.Pod( + fakes.WithNamespace("test-ns"), + fakes.WithName("test-name"), + ) + unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) + + // test that cm bubbles up the errors + require.ErrorContains(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") + require.ErrorContains(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") +} From 6e8348036a4072aeb46d1a499382d5ae51c26650 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 11 Jul 2023 20:10:49 +0000 Subject: [PATCH 17/58] mediate cm funcs Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/sync/sync_controller.go | 7 +++---- pkg/syncutil/opadataclient.go | 10 ++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/controller/cachemanager/sync/sync_controller.go b/pkg/controller/cachemanager/sync/sync_controller.go index 9c518c481d0..76256c20555 100644 --- a/pkg/controller/cachemanager/sync/sync_controller.go +++ b/pkg/controller/cachemanager/sync/sync_controller.go @@ -20,7 +20,6 @@ import ( "time" "github.com/go-logr/logr" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" @@ -41,7 +40,7 @@ import ( var log = logf.Log.WithName("controller").WithValues("metaKind", "Sync") type Adder struct { - CacheManager *cm.CacheManager + CacheManager syncutil.CacheManagerMediator Events <-chan event.GenericEvent } @@ -65,7 +64,7 @@ func (a *Adder) Add(mgr manager.Manager) error { func newReconciler( mgr manager.Manager, reporter syncutil.Reporter, - cmt *cm.CacheManager, + cmt syncutil.CacheManagerMediator, ) reconcile.Reconciler { return &ReconcileSync{ reader: mgr.GetCache(), @@ -103,7 +102,7 @@ type ReconcileSync struct { scheme *runtime.Scheme log logr.Logger reporter syncutil.Reporter - cm *cm.CacheManager + cm syncutil.CacheManagerMediator } // +kubebuilder:rbac:groups=constraints.gatekeeper.sh,resources=*,verbs=get;list;watch;create;update;patch;delete diff --git a/pkg/syncutil/opadataclient.go b/pkg/syncutil/opadataclient.go index 63acd1ddfac..fe2c3749bbe 100644 --- a/pkg/syncutil/opadataclient.go +++ b/pkg/syncutil/opadataclient.go @@ -20,9 +20,19 @@ import ( "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" ) +// CacheManagerMediator is an interface for mediating +// with a CacheManager but not actually depending on an instance. +type CacheManagerMediator interface { + AddObject(ctx context.Context, instance *unstructured.Unstructured) error + RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error + + ReportSyncMetrics() +} + // OpaDataClient is an interface for caching data. type OpaDataClient interface { AddData(ctx context.Context, data interface{}) (*types.Responses, error) From 33ceeeaf93430bc3bcdc64bd625d3db55c79f46e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 11 Jul 2023 20:15:29 +0000 Subject: [PATCH 18/58] review: filteredClient is cm, replay in bckg, watch gvks asap Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/cachemanager/cachemanager.go | 247 +++++++--- .../cachemanager_integration_test.go | 288 ++++++++++++ .../cachemanager/cachemanager_suite_test.go | 31 ++ .../cachemanager/cachemanager_test.go | 420 +++++++----------- .../cachemanager/cachemanager_unit_test.go | 81 ---- .../cachemanager/sync/sync_controller.go | 4 +- pkg/controller/controller.go | 3 +- pkg/syncutil/opadataclient.go | 41 -- 8 files changed, 645 insertions(+), 470 deletions(-) create mode 100644 pkg/controller/cachemanager/cachemanager_integration_test.go delete mode 100644 pkg/controller/cachemanager/cachemanager_unit_test.go diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/controller/cachemanager/cachemanager.go index 79acd74da33..c06da39e9ab 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/controller/cachemanager/cachemanager.go @@ -15,6 +15,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -33,21 +34,20 @@ type Config struct { } type CacheManager struct { - // the processExcluder and the gvkAggregator define what the underlying - // cache should look like. we refer to those two as "the spec" processExcluder *process.Excluder gvkAggregator *aggregator.GVKAgreggator - // mu guards access to any part of the spec above + gvksToRelist *watch.Set + excluderChanged bool + // mu guards access to any of the fields above mu sync.RWMutex - opa syncutil.OpaDataClient - syncMetricsCache *syncutil.MetricsCache - tracker *readiness.Tracker - registrar *watch.Registrar - watchedSet *watch.Set - replayTicker time.Ticker - reader client.Reader - excluderChanged bool + opa syncutil.OpaDataClient + syncMetricsCache *syncutil.MetricsCache + tracker *readiness.Tracker + registrar *watch.Registrar + watchedSet *watch.Set + cacheManagementTicker time.Ticker + reader client.Reader } func NewCacheManager(config *Config) (*CacheManager, error) { @@ -78,14 +78,14 @@ func NewCacheManager(config *Config) (*CacheManager, error) { } cm.gvkAggregator = aggregator.NewGVKAggregator() - - cm.replayTicker = *time.NewTicker(3 * time.Second) + cm.gvksToRelist = watch.NewSet() + cm.cacheManagementTicker = *time.NewTicker(3 * time.Second) return cm, nil } func (c *CacheManager) Start(ctx context.Context) error { - go c.updateDatastore(ctx) + go c.manageCache(ctx) <-ctx.Done() return nil @@ -97,9 +97,52 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, c.mu.Lock() defer c.mu.Unlock() + // for this source, find the net new gvks; + // we will establish new watches for them. + netNewGVKs := []schema.GroupVersionKind{} + for _, gvk := range newGVKs { + if !c.gvkAggregator.IsPresent(gvk) { + netNewGVKs = append(netNewGVKs, gvk) + } + } + if err := c.gvkAggregator.Upsert(sourceKey, newGVKs); err != nil { return fmt.Errorf("internal error adding source: %w", err) } + // as a result of upserting the new gvks for the source key, some gvks + // may become unreferenced and need to be deleted; this will be handled async + // in the manageCache loop. + + newGvkWatchSet := watch.NewSet() + newGvkWatchSet.AddSet(c.watchedSet) + newGvkWatchSet.Add(netNewGVKs...) + + if newGvkWatchSet.Size() != 0 { + // watch the net new gvks + if err := c.replaceWatchSet(ctx, newGvkWatchSet); err != nil { + return fmt.Errorf("error watching new gvks: %w", err) + } + } + + return nil +} + +func (c *CacheManager) replaceWatchSet(ctx context.Context, newWatchSet *watch.Set) error { + // assumes caller has lock + + var innerError error + c.watchedSet.Replace(newWatchSet, func() { + // *Note the following steps are not transactional with respect to admission control + + // Important: dynamic watches update must happen *after* updating our watchSet. + // Otherwise, the sync controller will drop events for the newly watched kinds. + // Defer error handling so object re-sync happens even if the watch is hard + // errored due to a missing GVK in the watch set. + innerError = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) + }) + if innerError != nil { + return innerError + } return nil } @@ -107,9 +150,11 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() + if err := c.gvkAggregator.Remove(sourceKey); err != nil { return fmt.Errorf("internal error removing source: %w", err) } + // watchSet update will happen async-ly in manageCache return nil } @@ -130,6 +175,11 @@ func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { } func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Unstructured) error { + // only perform work for watched gvks + if gvk := instance.GroupVersionKind(); !c.watchedSet.Contains(gvk) { + return nil + } + isNamespaceExcluded, err := c.processExcluder.IsNamespaceExcluded(process.Sync, instance) if err != nil { return fmt.Errorf("error while excluding namespaces: %w", err) @@ -168,6 +218,11 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns } func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error { + // only perform work for watched gvks + if gvk := instance.GroupVersionKind(); !c.watchedSet.Contains(gvk) { + return nil + } + if _, err := c.opa.RemoveData(ctx, instance); err != nil { return err } @@ -219,27 +274,95 @@ func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.Gro return nil } -func (c *CacheManager) updateDatastore(ctx context.Context) { +func (c *CacheManager) manageCache(ctx context.Context) { + stopChan := make(chan bool, 1) + gvkErrdChan := make(chan schema.GroupVersionKind, 1024) + gvksFailingTolist := watch.NewSet() + + gvksFailingToListReconciler := func(stopChan <-chan bool) { + for { + select { + case <-stopChan: + return + case gvk := <-gvkErrdChan: + gvksFailingTolist.Add(gvk) + } + } + } + for { select { case <-ctx.Done(): + close(stopChan) + close(gvkErrdChan) return - case <-c.replayTicker.C: - // snapshot the current spec so we can make a point in - // time change to the contents of the opa cache. - c.mu.RLock() - currentGVKsInAgg := watch.NewSet() - currentGVKsInAgg.Add(c.gvkAggregator.ListAllGVKs()...) - excluderChanged := c.excluderChanged - c.mu.RUnlock() - - c.makeUpdatesForSpec(ctx, currentGVKsInAgg, excluderChanged) + case <-c.cacheManagementTicker.C: + c.mu.Lock() + c.makeUpdates(ctx) + + // spin up new goroutines to relist if new gvks to relist are + // populated from makeUpdates. + if c.gvksToRelist.Size() != 0 { + // stop any goroutines that were relisting before + stopChan <- true + + // also try to catch any gvks that are in the aggregator + // but are failing to list from a previous replay. + for _, gvk := range gvksFailingTolist.Items() { + if c.gvkAggregator.IsPresent(gvk) { + c.gvksToRelist.Add(gvk) + } + } + + // save all gvks that need relisting + gvksToRelistForLoop := c.gvksToRelist.Items() + + // clean state + gvksFailingTolist = watch.NewSet() + c.gvksToRelist = watch.NewSet() + + stopChan = make(chan bool) + + go c.replayLoop(ctx, gvksToRelistForLoop, stopChan) + go gvksFailingToListReconciler(stopChan) + } + c.mu.Unlock() + } + } +} + +func (c *CacheManager) replayLoop(ctx context.Context, gvksToRelist []schema.GroupVersionKind, stopChan <-chan bool) { + for _, gvk := range gvksToRelist { + select { + case <-ctx.Done(): + return + case <-stopChan: + return + default: + backoff := wait.Backoff{ + Duration: time.Second, + Factor: 2, + Jitter: 0.1, + Steps: 3, + } + + operation := func() (bool, error) { + if err := c.listAndSyncDataForGVK(ctx, gvk); err != nil { + return false, err + } + + return true, nil + } + + if err := wait.ExponentialBackoff(backoff, operation); err != nil { + log.Error(err, "internal: error listings gvk cache data", "gvk", gvk) + } } } } // listAndSyncData returns a set of gvks that were successfully listed and synced. -func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupVersionKind, reader client.Reader) *watch.Set { +func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupVersionKind) *watch.Set { gvksSuccessfullySynced := watch.NewSet() for _, gvk := range gvks { err := c.listAndSyncDataForGVK(ctx, gvk) @@ -255,64 +378,40 @@ func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupV return gvksSuccessfullySynced } -// makeUpdatesForSpec performs a conditional wipe followed by a replay if necessary as +// makeUpdates performs a conditional wipe followed by a replay if necessary as // given by the current spec (currentGVKsInAgg, excluderChanged) at the time of the call. -func (c *CacheManager) makeUpdatesForSpec(ctx context.Context, currentGVKsInAgg *watch.Set, excluderChanged bool) { - if c.watchedSet.Equals(currentGVKsInAgg) && !excluderChanged { - return // nothing to do if both sets are the same and the excluder didn't change - } - - // replace the current watch set for the sync_controller to pick up - // any updates on said GVKs. - // also save the current watch set to make cache changes later - oldWatchSet := watch.NewSet() - oldWatchSet.AddSet(c.watchedSet) +func (c *CacheManager) makeUpdates(ctx context.Context) { + // assumes the caller has lock - var innerError error - c.watchedSet.Replace(currentGVKsInAgg, func() { - // *Note the following steps are not transactional with respect to admission control* + currentGVKsInAgg := watch.NewSet() + currentGVKsInAgg.Add(c.gvkAggregator.ListAllGVKs()...) - // Important: dynamic watches update must happen *after* updating our watchSet. - // Otherwise, the sync controller will drop events for the newly watched kinds. - // Defer error handling so object re-sync happens even if the watch is hard - // errored due to a missing GVK in the watch set. - innerError = c.registrar.ReplaceWatch(ctx, currentGVKsInAgg.Items()) - }) - if innerError != nil { - log.Error(innerError, "internal: error replacing watch set") + if c.watchedSet.Equals(currentGVKsInAgg) && !c.excluderChanged { + // nothing to do if both sets are the same and the excluder didn't change + // and there are no gvks that need relisting from a previous wipe + return } - gvksToDelete := oldWatchSet.Difference(currentGVKsInAgg).Items() - newGVKsToSync := currentGVKsInAgg.Difference(oldWatchSet) + gvksToDelete := c.watchedSet.Difference(currentGVKsInAgg) + newGVKsToSync := currentGVKsInAgg.Difference(c.watchedSet) + gvksToReplay := c.watchedSet.Intersection(currentGVKsInAgg) + + if gvksToDelete.Size() != 0 || newGVKsToSync.Size() != 0 { + // in this case we need to replace the watch set again since there + // is drift between the aggregator and the currently watched gvks + if err := c.replaceWatchSet(ctx, currentGVKsInAgg); err != nil { + log.Error(err, "internal: error replacing watch set") + } + } // remove any gvks not needing to be synced anymore // or re evaluate all if the excluder changed. - if len(gvksToDelete) > 0 || excluderChanged { + if gvksToDelete.Size() > 0 || c.excluderChanged { if err := c.wipeData(ctx); err != nil { log.Error(err, "internal: error wiping cache") + } else { + c.excluderChanged = false } - - if excluderChanged { - c.unsetExcluderChanged() - } - - // everything that gets wiped needs to be readded - newGVKsToSync.AddSet(currentGVKsInAgg) + c.gvksToRelist.AddSet(gvksToReplay) } - - // sync net new gvks and potentially replayed gvks from the cache wipe above - gvksSynced := c.listAndSyncData(ctx, newGVKsToSync.Items(), c.reader) - - gvksNotSynced := gvksSynced.Difference(newGVKsToSync) - for _, gvk := range gvksNotSynced.Items() { - log.Info(fmt.Sprintf("failed to sync gvk: %s; will retry", gvk)) - } -} - -func (c *CacheManager) unsetExcluderChanged() { - c.mu.Lock() - defer c.mu.Unlock() - - // unset the excluderChanged bool now - c.excluderChanged = false } diff --git a/pkg/controller/cachemanager/cachemanager_integration_test.go b/pkg/controller/cachemanager/cachemanager_integration_test.go new file mode 100644 index 00000000000..2a6f9b1ac56 --- /dev/null +++ b/pkg/controller/cachemanager/cachemanager_integration_test.go @@ -0,0 +1,288 @@ +package cachemanager + +import ( + "testing" + + configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" + "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// TestCacheManager_listAndSyncData tests that the cache manager can add gvks to the data store. +func TestCacheManager_listAndSyncData(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) + + configMapGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "ConfigMap", + } + // Create configMaps to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + cm2 := unstructuredFor(configMapGVK, "config-test-2") + require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + + cacheManager.watchedSet.Add(configMapGVK) + require.NoError(t, cacheManager.listAndSyncDataForGVK(ctx, configMapGVK)) + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + } + + require.Equal(t, 2, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // wipe cache + require.NoError(t, cacheManager.wipeData(ctx)) + require.False(t, opaClient.Contains(expected)) + + // create a second GVK + podGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + } + // Create pods to test for + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + pod2 := unstructuredFor(podGVK, "pod-2") + require.NoError(t, c.Create(ctx, pod2), "creating Pod pod-2") + + pod3 := unstructuredFor(podGVK, "pod-3") + require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") + + cacheManager.watchedSet.Add(podGVK) + syncedSet := cacheManager.listAndSyncData(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}) + require.ElementsMatch(t, syncedSet.Items(), []schema.GroupVersionKind{configMapGVK, podGVK}) + + expected = map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + {Gvk: podGVK, Key: "default/pod-2"}: nil, + {Gvk: podGVK, Key: "default/pod-3"}: nil, + } + + require.Equal(t, 5, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") + require.NoError(t, c.Delete(ctx, pod3), "deleting Pod pod-3") + require.NoError(t, c.Delete(ctx, pod2), "deleting Pod pod-2") +} + +// TestCacheManager_AddSourceRemoveSource makes sure that we can add and remove multiple sources +// and changes to the underlying cache are reflected. +func TestCacheManager_AddSourceRemoveSource(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + + // Create configMaps to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + cm2 := unstructuredFor(configMapGVK, "config-test-2") + require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + + syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) + + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + + require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + + // now assert that the gvkAggregator looks as expected + cacheManager.gvkAggregator.IsPresent(configMapGVK) + gvks := cacheManager.gvkAggregator.List(syncSourceOne) + require.Len(t, gvks, 2) + _, foundConfigMap := gvks[configMapGVK] + require.True(t, foundConfigMap) + _, foundPod := gvks[podGVK] + require.True(t, foundPod) + + // now remove the podgvk and make sure we don't have pods in the cache anymore + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + + expected = map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + } + require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + // now assert that the gvkAggregator looks as expected + cacheManager.gvkAggregator.IsPresent(configMapGVK) + gvks = cacheManager.gvkAggregator.List(syncSourceOne) + require.Len(t, gvks, 1) + _, foundConfigMap = gvks[configMapGVK] + require.True(t, foundConfigMap) + _, foundPod = gvks[podGVK] + require.False(t, foundPod) + + // now make sure that adding another sync source with the same gvk has no side effects + syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} + require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) + + reqConditionForAgg := func() bool { + cacheManager.gvkAggregator.IsPresent(configMapGVK) + gvks := cacheManager.gvkAggregator.List(syncSourceOne) + if len(gvks) != 1 { + return false + } + _, found := gvks[configMapGVK] + if !found { + return false + } + + gvks2 := cacheManager.gvkAggregator.List(syncSourceTwo) + if len(gvks2) != 1 { + return false + } + _, found2 := gvks2[configMapGVK] + return found2 + } + require.Eventually(t, reqConditionForAgg, testutils.EventuallyTimeout, testutils.EventuallyTicker) + + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) + expected2 := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + require.Eventually(t, expectedCheck(opaClient, expected2), testutils.EventuallyTimeout, testutils.EventuallyTicker) + + // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed + require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) + expected3 := map[fakes.OpaKey]interface{}{ + // config maps no longer required by any sync source + // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + require.Eventually(t, expectedCheck(opaClient, expected3), testutils.EventuallyTimeout, testutils.EventuallyTicker) + + // now remove all the sources + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) + + // and expect an empty cache and empty aggregator + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 0) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") +} + +// TestCacheManager_ExcludeProcesses makes sure that changing the process excluder +// in the cache manager triggers a re-evaluation of GVKs. +func TestCacheManager_ExcludeProcesses(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + } + + syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} + require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) + // check that everything is well added at first + require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + + // make sure that replacing w same process excluder is a no op + sameExcluder := process.New() + sameExcluder.Add([]configv1alpha1.MatchEntry{ + // same excluder as the one in makeCacheManagerForTest + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + cacheManager.ExcludeProcesses(sameExcluder) + require.True(t, cacheManager.processExcluder.Equals(sameExcluder)) + + // now process exclude the remaining gvk, it should get removed by the background process. + excluder := process.New() + excluder.Add([]configv1alpha1.MatchEntry{ + // exclude the "default" namespace + { + ExcludedNamespaces: []wildcard.Wildcard{"default"}, + Processes: []string{"sync"}, + }, + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + cacheManager.ExcludeProcesses(excluder) + + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) + // make sure the gvk is still in gvkAggregator + require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 1) + require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") +} + +func expectedCheck(opaClient *fakes.FakeOpa, expected map[fakes.OpaKey]interface{}) func() bool { + return func() bool { + if opaClient.Len() != len(expected) { + return false + } + if opaClient.Contains(expected) { + return true + } + + return false + } +} + +func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetName(name) + u.SetNamespace("default") + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + return u +} diff --git a/pkg/controller/cachemanager/cachemanager_suite_test.go b/pkg/controller/cachemanager/cachemanager_suite_test.go index 5c4cc48ca4b..5e25847840f 100644 --- a/pkg/controller/cachemanager/cachemanager_suite_test.go +++ b/pkg/controller/cachemanager/cachemanager_suite_test.go @@ -5,6 +5,7 @@ import ( "testing" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" @@ -57,6 +58,11 @@ func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*Cach require.NoError(t, err) if startCache { + syncAdder := syncc.Adder{ + Events: events, + CacheManager: cacheManager, + } + require.NoError(t, syncAdder.Add(mgr), "registering sync controller") go func() { require.NoError(t, cacheManager.Start(ctx)) }() @@ -75,3 +81,28 @@ func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*Cach }) return cacheManager, c, ctx } + +// makeUnitCacheManagerForTest creates a cache manager without starting the controller-runtime manager +// and without starting the cache manager background process. Note that this also means that the +// watch manager is not started and the sync controller is not started. +func makeUnitCacheManagerForTest(t *testing.T) (*CacheManager, context.Context) { + cm, _, ctx := makeCacheManagerForTest(t, false, false) + return cm, ctx +} + +func newSyncExcluderFor(nsToExclude string) *process.Excluder { + excluder := process.New() + excluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{wildcard.Wildcard(nsToExclude)}, + Processes: []string{"sync"}, + }, + // exclude kube-system by default to prevent noise + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + + return excluder +} diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/controller/cachemanager/cachemanager_test.go index 679752449a1..7730647eed4 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/controller/cachemanager/cachemanager_test.go @@ -7,86 +7,16 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" - "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" - "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) -// TestCacheManager_listAndSyncData tests that the cache manager can add gvks to the data store. -func TestCacheManager_listAndSyncData(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) - - configMapGVK := schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "ConfigMap", - } - // Create configMaps to test for - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - cm2 := unstructuredFor(configMapGVK, "config-test-2") - require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") - - require.NoError(t, cacheManager.listAndSyncDataForGVK(ctx, configMapGVK)) - - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) - require.True(t, ok) - expected := map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - } - - require.Equal(t, 2, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) - - // wipe cache - require.NoError(t, cacheManager.wipeData(ctx)) - require.False(t, opaClient.Contains(expected)) - - // create a second GVK - podGVK := schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", - } - // Create pods to test for - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - - pod2 := unstructuredFor(podGVK, "pod-2") - require.NoError(t, c.Create(ctx, pod2), "creating Pod pod-2") - - pod3 := unstructuredFor(podGVK, "pod-3") - require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") - - syncedSet := cacheManager.listAndSyncData(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}, c) - require.ElementsMatch(t, syncedSet.Items(), []schema.GroupVersionKind{configMapGVK, podGVK}) - - expected = map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - {Gvk: podGVK, Key: "default/pod-2"}: nil, - {Gvk: podGVK, Key: "default/pod-3"}: nil, - } - - require.Equal(t, 5, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") - require.NoError(t, c.Delete(ctx, pod3), "deleting Pod pod-3") - require.NoError(t, c.Delete(ctx, pod2), "deleting Pod pod-2") -} - -// TestCacheManager_makeUpdates tests that we can remove and add gvks to the data store. +// TestCacheManager_makeUpdates tests that we can add gvks to the data store. func TestCacheManager_makeUpdates(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) + cacheManager, ctx := makeUnitCacheManagerForTest(t) opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -94,231 +24,181 @@ func TestCacheManager_makeUpdates(t *testing.T) { configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} cm := unstructuredFor(configMapGVK, "config-test-1") _, err := opaClient.AddData(ctx, cm) - require.NoError(t, err, "creating ConfigMap config-test-1 in opa") - - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + require.NoError(t, err, "adding ConfigMap config-test-1 in opa") + cacheManager.watchedSet.Add(configMapGVK) // prep gvkAggregator for updates to be picked up in makeUpdates + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} require.NoError(t, cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) - gvksInAgg := watch.NewSet() - gvksInAgg.Add(cacheManager.gvkAggregator.ListAllGVKs()...) - cacheManager.makeUpdatesForSpec(ctx, gvksInAgg, false) - - // expect the following instances to be in the data store - expected := map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - require.Equal(t, 2, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) - - // cleanup - // cm was not actually created thru the client, so no need for this. - // require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") + cacheManager.makeUpdates(ctx) + + // two things should happen: + // - the cache manager starts watching the pod gvk + // - the cache manager stops watching the configMap gvk + require.False(t, opaClient.HasGVK(configMapGVK)) + require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK}) } -// TestCacheManager_AddSourceRemoveSource makes sure that we can add and remove multiple sources -// and changes to the underlying cache are reflected. -func TestCacheManager_AddSourceRemoveSource(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) +// TestCacheManager_makeUpdates_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. +func TestCacheManager_makeUpdates_excluderChanges(t *testing.T) { + cacheManager, ctx := makeUnitCacheManagerForTest(t) + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + // seed gvks configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - - // Create configMaps to test for cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + cm.SetNamespace("excluded-ns") + _, err := opaClient.AddData(ctx, cm) + require.NoError(t, err, "adding ConfigMap config-test-1 in opa") + cacheManager.watchedSet.Add(configMapGVK) - cm2 := unstructuredFor(configMapGVK, "config-test-2") - require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + pod := unstructuredFor(configMapGVK, "pod-test-1") + pod.SetNamespace("excluded-ns") + _, err = opaClient.AddData(ctx, pod) + require.NoError(t, err, "adding Pod pod-test-1 in opa") + cacheManager.watchedSet.Add(podGVK) + require.NoError(t, cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK, configMapGVK})) + + cacheManager.ExcludeProcesses(newSyncExcluderFor("excluded-ns")) + cacheManager.makeUpdates(ctx) + + // the cache manager should not be watching any of the gvks that are now excluded + require.False(t, opaClient.HasGVK(configMapGVK)) + require.False(t, opaClient.HasGVK(podGVK)) +} - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") +// TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. +func TestCacheManager_AddObject_RemoveObject(t *testing.T) { + cm, ctx := makeUnitCacheManagerForTest(t) - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cm.opa.(*fakes.FakeOpa) require.True(t, ok) - syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - - expected := map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - - require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) - - // now assert that the gvkAggregator looks as expected - cacheManager.gvkAggregator.IsPresent(configMapGVK) - gvks := cacheManager.gvkAggregator.List(syncSourceOne) - require.Len(t, gvks, 2) - _, foundConfigMap := gvks[configMapGVK] - require.True(t, foundConfigMap) - _, foundPod := gvks[podGVK] - require.True(t, foundPod) - - // now remove the podgvk and make sure we don't have pods in the cache anymore - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - - expected = map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - } - require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) - // now assert that the gvkAggregator looks as expected - cacheManager.gvkAggregator.IsPresent(configMapGVK) - gvks = cacheManager.gvkAggregator.List(syncSourceOne) - require.Len(t, gvks, 1) - _, foundConfigMap = gvks[configMapGVK] - require.True(t, foundConfigMap) - _, foundPod = gvks[podGVK] - require.False(t, foundPod) - - // now make sure that adding another sync source with the same gvk has no side effects - syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - - reqConditionForAgg := func() bool { - cacheManager.gvkAggregator.IsPresent(configMapGVK) - gvks := cacheManager.gvkAggregator.List(syncSourceOne) - if len(gvks) != 1 { - return false - } - _, found := gvks[configMapGVK] - if !found { - return false - } - - gvks2 := cacheManager.gvkAggregator.List(syncSourceTwo) - if len(gvks2) != 1 { - return false - } - _, found2 := gvks2[configMapGVK] - return found2 - } - require.Eventually(t, reqConditionForAgg, testutils.EventuallyTimeout, testutils.EventuallyTicker) - - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - expected2 := map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - require.Eventually(t, expectedCheck(opaClient, expected2), testutils.EventuallyTimeout, testutils.EventuallyTicker) - - // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed - require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) - expected3 := map[fakes.OpaKey]interface{}{ - // config maps no longer required by any sync source - // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - require.Eventually(t, expectedCheck(opaClient, expected3), testutils.EventuallyTimeout, testutils.EventuallyTicker) - - // now remove all the sources - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) - - // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) - require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 0) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") -} + pod := fakes.Pod( + fakes.WithNamespace("test-ns"), + fakes.WithName("test-name"), + ) + unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) -// TestCacheManager_ExcludeProcesses makes sure that changing the process excluder -// in the cache manager triggers a re-evaluation of GVKs. -func TestCacheManager_ExcludeProcesses(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + // when gvk is watched, we expect Add, Remove to work + cm.watchedSet.Add(pod.GroupVersionKind()) - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) - require.True(t, ok) + require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + require.True(t, opaClient.HasGVK(pod.GroupVersionKind())) - expected := map[fakes.OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - } + // now remove the object and verify it's removed + require.NoError(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) - syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) - // check that everything is well added at first - require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + cm.watchedSet.Remove(pod.GroupVersionKind()) + require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) // we drop calls for gvks that are not watched +} - // make sure that replacing w same process excluder is a no op - sameExcluder := process.New() - sameExcluder.Add([]configv1alpha1.MatchEntry{ - // same excluder as the one in makeCacheManagerForTest +// TestCacheManager_AddObject_processExclusion makes sure that we don't add objects that are process excluded. +func TestCacheManager_AddObject_processExclusion(t *testing.T) { + cm, ctx := makeUnitCacheManagerForTest(t) + processExcluder := process.Get() + processExcluder.Add([]configv1alpha1.MatchEntry{ { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + ExcludedNamespaces: []wildcard.Wildcard{"test-ns-excluded"}, Processes: []string{"sync"}, }, }) - cacheManager.ExcludeProcesses(sameExcluder) - require.True(t, cacheManager.processExcluder.Equals(sameExcluder)) + cm.processExcluder.Replace(processExcluder) - // now process exclude the remaining gvk, it should get removed by the background process. - excluder := process.New() - excluder.Add([]configv1alpha1.MatchEntry{ - // exclude the "default" namespace - { - ExcludedNamespaces: []wildcard.Wildcard{"default"}, - Processes: []string{"sync"}, - }, - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - cacheManager.ExcludeProcesses(excluder) + pod := fakes.Pod( + fakes.WithNamespace("test-ns-excluded"), + fakes.WithName("test-name"), + ) + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) - // make sure the gvk is still in gvkAggregator - require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 1) - require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) + require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + // test that pod from excluded namespace is not cache managed + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) + require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) + require.False(t, opaClient.Contains(map[fakes.OpaKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) } -func expectedCheck(opaClient *fakes.FakeOpa, expected map[fakes.OpaKey]interface{}) func() bool { - return func() bool { - if opaClient.Len() != len(expected) { - return false - } - if opaClient.Contains(expected) { - return true - } - - return false - } +// TestCacheManager_errors tests that the cache manager responds to errors from the opa client. +func TestCacheManager_errors(t *testing.T) { + cm, ctx := makeUnitCacheManagerForTest(t) + opaClient, ok := cm.opa.(*fakes.FakeOpa) + require.True(t, ok) + opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err + + pod := fakes.Pod( + fakes.WithNamespace("test-ns"), + fakes.WithName("test-name"), + ) + unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) + require.NoError(t, err) + cm.watchedSet.Add(pod.GroupVersionKind()) + + // test that cm bubbles up the errors + require.ErrorContains(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") + require.ErrorContains(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") } -func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - u.SetName(name) - u.SetNamespace("default") - if gvk.Kind == "Pod" { - u.Object["spec"] = map[string]interface{}{ - "containers": []map[string]interface{}{ - { - "name": "foo-container", - "image": "foo-image", - }, - }, - } - } - return u +// TestCacheManager_AddSource tests that we can modify the gvk aggregator and watched set when adding a new source. +func TestCacheManager_AddSource(t *testing.T) { + cacheManager, ctx := makeUnitCacheManagerForTest(t) + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + nsGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} + sourceA := aggregator.Key{Source: "a", ID: "source"} + sourceB := aggregator.Key{Source: "b", ID: "source"} + + // given two sources with overlapping gvks ... + require.NoError(t, cacheManager.AddSource(ctx, sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + + // ... expect the aggregator to dedup + require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK, configMapGVK}) + + // adding a source without a previously added gvk ... + require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{configMapGVK})) + // ... should not remove any gvks that are still referenced by other sources + require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + + // adding a source that modifies the only reference to a gvk ... + require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{nsGVK})) + + // ... will effectively remove the gvk from the aggregator + require.False(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + require.True(t, cacheManager.gvkAggregator.IsPresent(nsGVK)) +} + +// TestCacheManager_RemoveSource tests that we can modify the gvk aggregator when removing a source. +func TestCacheManager_RemoveSource(t *testing.T) { + cacheManager, ctx := makeUnitCacheManagerForTest(t) + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + sourceA := aggregator.Key{Source: "a", ID: "source"} + sourceB := aggregator.Key{Source: "b", ID: "source"} + + // seed the gvk aggregator + require.NoError(t, cacheManager.gvkAggregator.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.gvkAggregator.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + + // removing a source that is not the only one referencing a gvk ... + require.NoError(t, cacheManager.RemoveSource(ctx, sourceB)) + // ... should not remove any gvks that are still referenced by other sources + require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + require.False(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + + require.NoError(t, cacheManager.RemoveSource(ctx, sourceA)) + require.False(t, cacheManager.gvkAggregator.IsPresent(podGVK)) } diff --git a/pkg/controller/cachemanager/cachemanager_unit_test.go b/pkg/controller/cachemanager/cachemanager_unit_test.go deleted file mode 100644 index bd786d5fd07..00000000000 --- a/pkg/controller/cachemanager/cachemanager_unit_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package cachemanager - -import ( - "testing" - - configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" - "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" - "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" -) - -// TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. -func TestCacheManager_AddObject_RemoveObject(t *testing.T) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - - pod := fakes.Pod( - fakes.WithNamespace("test-ns"), - fakes.WithName("test-name"), - ) - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - - // test that pod is cache managed - opaClient, ok := cm.opa.(*fakes.FakeOpa) - require.True(t, ok) - require.True(t, opaClient.HasGVK(pod.GroupVersionKind())) - - // now remove the object and verify it's removed - require.NoError(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) -} - -// TestCacheManager_processExclusion makes sure that we don't add objects that are process excluded. -func TestCacheManager_processExclusion(t *testing.T) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - processExcluder := process.Get() - processExcluder.Add([]configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{"test-ns-excluded"}, - Processes: []string{"sync"}, - }, - }) - cm.processExcluder.Replace(processExcluder) - - pod := fakes.Pod( - fakes.WithNamespace("test-ns-excluded"), - fakes.WithName("test-name"), - ) - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - - // test that pod from excluded namespace is not cache managed - opaClient, ok := cm.opa.(*fakes.FakeOpa) - require.True(t, ok) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) -} - -// TestCacheManager_errors tests that the cache manager responds to errors from the opa client. -func TestCacheManager_errors(t *testing.T) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - opaClient, ok := cm.opa.(*fakes.FakeOpa) - require.True(t, ok) - opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err - - pod := fakes.Pod( - fakes.WithNamespace("test-ns"), - fakes.WithName("test-name"), - ) - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - - // test that cm bubbles up the errors - require.ErrorContains(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") - require.ErrorContains(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") -} diff --git a/pkg/controller/cachemanager/sync/sync_controller.go b/pkg/controller/cachemanager/sync/sync_controller.go index 76256c20555..242e8217681 100644 --- a/pkg/controller/cachemanager/sync/sync_controller.go +++ b/pkg/controller/cachemanager/sync/sync_controller.go @@ -64,14 +64,14 @@ func (a *Adder) Add(mgr manager.Manager) error { func newReconciler( mgr manager.Manager, reporter syncutil.Reporter, - cmt syncutil.CacheManagerMediator, + cm syncutil.CacheManagerMediator, ) reconcile.Reconciler { return &ReconcileSync{ reader: mgr.GetCache(), scheme: mgr.GetScheme(), log: log, reporter: reporter, - cm: cmt, + cm: cm, } } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 1946d08d32a..2c990a76ba1 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -174,7 +174,6 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { // Events will be used to receive events from dynamic watches registered // via the registrar below. events := make(chan event.GenericEvent, 1024) - filteredOpa := syncutil.NewFilteredOpaDataClient(deps.Opa, deps.WatchSet) syncMetricsCache := syncutil.NewMetricsCache() w, err := deps.WatchManger.NewRegistrar( config.CtrlName, @@ -183,7 +182,7 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { return err } cm, err := cm.NewCacheManager(&cm.Config{ - Opa: filteredOpa, + Opa: deps.Opa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, ProcessExcluder: deps.ProcessExcluder, diff --git a/pkg/syncutil/opadataclient.go b/pkg/syncutil/opadataclient.go index fe2c3749bbe..3febc82b70c 100644 --- a/pkg/syncutil/opadataclient.go +++ b/pkg/syncutil/opadataclient.go @@ -19,9 +19,7 @@ import ( "context" "github.com/open-policy-agent/frameworks/constraint/pkg/types" - "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "sigs.k8s.io/controller-runtime/pkg/client" ) // CacheManagerMediator is an interface for mediating @@ -38,42 +36,3 @@ type OpaDataClient interface { AddData(ctx context.Context, data interface{}) (*types.Responses, error) RemoveData(ctx context.Context, data interface{}) (*types.Responses, error) } - -// FilteredDataClient is an OpaDataClient which drops any unwatched resources. -type FilteredDataClient struct { - watched *watch.Set - opa OpaDataClient -} - -func NewFilteredOpaDataClient(opa OpaDataClient, watchSet *watch.Set) *FilteredDataClient { - return &FilteredDataClient{ - watched: watchSet, - opa: opa, - } -} - -// AddData adds data to the opa cache if that data is currently being watched. -// Unwatched data is silently dropped with no error. -func (f *FilteredDataClient) AddData(ctx context.Context, data interface{}) (*types.Responses, error) { - if obj, ok := data.(client.Object); ok { - gvk := obj.GetObjectKind().GroupVersionKind() - if !f.watched.Contains(gvk) { - return &types.Responses{}, nil - } - } - - return f.opa.AddData(ctx, data) -} - -// RemoveData removes data from the opa cache if that data is currently being watched. -// Unwatched data is silently dropped with no error. -func (f *FilteredDataClient) RemoveData(ctx context.Context, data interface{}) (*types.Responses, error) { - if obj, ok := data.(client.Object); ok { - gvk := obj.GetObjectKind().GroupVersionKind() - if !f.watched.Contains(gvk) { - return &types.Responses{}, nil - } - } - - return f.opa.RemoveData(ctx, data) -} From 30aa4cf3601c7858ab2374463a4588ac531ca2be Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 24 Jul 2023 17:39:16 +0000 Subject: [PATCH 19/58] review: update watch set on RemoveSource - leave cachemanager in syncutil - naming - plus other points of feedback Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller.go | 2 +- .../config/config_controller_test.go | 4 +- pkg/controller/controller.go | 4 +- .../sync/sync_controller.go | 0 .../cachemanager/cachemanager.go | 178 +++++++----------- .../cachemanager_integration_test.go | 30 +-- .../cachemanager/cachemanager_suite_test.go | 2 +- .../cachemanager/cachemanager_test.go | 55 +++--- 8 files changed, 117 insertions(+), 158 deletions(-) rename pkg/controller/{cachemanager => }/sync/sync_controller.go (100%) rename pkg/{controller => syncutil}/cachemanager/cachemanager.go (66%) rename pkg/{controller => syncutil}/cachemanager/cachemanager_integration_test.go (91%) rename pkg/{controller => syncutil}/cachemanager/cachemanager_suite_test.go (99%) rename pkg/{controller => syncutil}/cachemanager/cachemanager_test.go (79%) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 00e7981ab15..6a260a5be02 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -22,13 +22,13 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 0b96681460b..3a064862640 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -26,12 +26,12 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" - syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2c990a76ba1..82769f8b979 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -24,16 +24,16 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager" - syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" corev1 "k8s.io/api/core/v1" diff --git a/pkg/controller/cachemanager/sync/sync_controller.go b/pkg/controller/sync/sync_controller.go similarity index 100% rename from pkg/controller/cachemanager/sync/sync_controller.go rename to pkg/controller/sync/sync_controller.go diff --git a/pkg/controller/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go similarity index 66% rename from pkg/controller/cachemanager/cachemanager.go rename to pkg/syncutil/cachemanager/cachemanager.go index c06da39e9ab..e0d7a20d480 100644 --- a/pkg/controller/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -34,20 +34,21 @@ type Config struct { } type CacheManager struct { - processExcluder *process.Excluder - gvkAggregator *aggregator.GVKAgreggator - gvksToRelist *watch.Set - excluderChanged bool + processExcluder *process.Excluder + specifiedGVKs *aggregator.GVKAgreggator + gvksToList *watch.Set + gvksToDeleteFromCache *watch.Set + excluderChanged bool // mu guards access to any of the fields above mu sync.RWMutex - opa syncutil.OpaDataClient - syncMetricsCache *syncutil.MetricsCache - tracker *readiness.Tracker - registrar *watch.Registrar - watchedSet *watch.Set - cacheManagementTicker time.Ticker - reader client.Reader + opa syncutil.OpaDataClient + syncMetricsCache *syncutil.MetricsCache + tracker *readiness.Tracker + registrar *watch.Registrar + watchedSet *watch.Set + backgroundManagementTicker time.Ticker + reader client.Reader } func NewCacheManager(config *Config) (*CacheManager, error) { @@ -67,21 +68,19 @@ func NewCacheManager(config *Config) (*CacheManager, error) { return nil, fmt.Errorf("reader must be non-nil") } - cm := &CacheManager{ - opa: config.Opa, - syncMetricsCache: config.SyncMetricsCache, - tracker: config.Tracker, - processExcluder: config.ProcessExcluder, - registrar: config.Registrar, - watchedSet: config.WatchedSet, - reader: config.Reader, - } - - cm.gvkAggregator = aggregator.NewGVKAggregator() - cm.gvksToRelist = watch.NewSet() - cm.cacheManagementTicker = *time.NewTicker(3 * time.Second) - - return cm, nil + return &CacheManager{ + opa: config.Opa, + syncMetricsCache: config.SyncMetricsCache, + tracker: config.Tracker, + processExcluder: config.ProcessExcluder, + registrar: config.Registrar, + watchedSet: config.WatchedSet, + reader: config.Reader, + specifiedGVKs: aggregator.NewGVKAggregator(), + gvksToList: watch.NewSet(), + backgroundManagementTicker: *time.NewTicker(3 * time.Second), + gvksToDeleteFromCache: watch.NewSet(), + }, nil } func (c *CacheManager) Start(ctx context.Context) error { @@ -97,38 +96,34 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, c.mu.Lock() defer c.mu.Unlock() - // for this source, find the net new gvks; - // we will establish new watches for them. - netNewGVKs := []schema.GroupVersionKind{} - for _, gvk := range newGVKs { - if !c.gvkAggregator.IsPresent(gvk) { - netNewGVKs = append(netNewGVKs, gvk) - } - } - - if err := c.gvkAggregator.Upsert(sourceKey, newGVKs); err != nil { + if err := c.specifiedGVKs.Upsert(sourceKey, newGVKs); err != nil { return fmt.Errorf("internal error adding source: %w", err) } // as a result of upserting the new gvks for the source key, some gvks // may become unreferenced and need to be deleted; this will be handled async // in the manageCache loop. - newGvkWatchSet := watch.NewSet() - newGvkWatchSet.AddSet(c.watchedSet) - newGvkWatchSet.Add(netNewGVKs...) - - if newGvkWatchSet.Size() != 0 { - // watch the net new gvks - if err := c.replaceWatchSet(ctx, newGvkWatchSet); err != nil { - return fmt.Errorf("error watching new gvks: %w", err) - } + // make changes to the watches + if err := c.replaceWatchSet(ctx); err != nil { + return fmt.Errorf("error watching new gvks: %w", err) } return nil } -func (c *CacheManager) replaceWatchSet(ctx context.Context, newWatchSet *watch.Set) error { - // assumes caller has lock +// replaceWatchSet looks at the specifiedGVKs and makes changes to the registrar's watch set. +// assumes caller has lock. +func (c *CacheManager) replaceWatchSet(ctx context.Context) error { + newWatchSet := watch.NewSet() + newWatchSet.Add(c.specifiedGVKs.ListAllGVKs()...) + + if newWatchSet.Equals(c.watchedSet) { + // nothing to do as the sets are equal + return nil + } + + // record any gvks that need to be deleted + c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) var innerError error c.watchedSet.Replace(newWatchSet, func() { @@ -140,21 +135,22 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context, newWatchSet *watch.S // errored due to a missing GVK in the watch set. innerError = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) }) - if innerError != nil { - return innerError - } - return nil + return innerError } func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() - if err := c.gvkAggregator.Remove(sourceKey); err != nil { + if err := c.specifiedGVKs.Remove(sourceKey); err != nil { return fmt.Errorf("internal error removing source: %w", err) } - // watchSet update will happen async-ly in manageCache + + // make changes to the watches + if err := c.replaceWatchSet(ctx); err != nil { + return fmt.Errorf("error removing watches for source %s: %w", sourceKey, err) + } return nil } @@ -250,7 +246,7 @@ func (c *CacheManager) ReportSyncMetrics() { c.syncMetricsCache.ReportSync() } -func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.GroupVersionKind) error { +func (c *CacheManager) syncGVKInstances(ctx context.Context, gvk schema.GroupVersionKind) error { u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: gvk.Group, @@ -275,6 +271,7 @@ func (c *CacheManager) listAndSyncDataForGVK(ctx context.Context, gvk schema.Gro } func (c *CacheManager) manageCache(ctx context.Context) { + // stopChan is used to stop any list operations still in progress stopChan := make(chan bool, 1) gvkErrdChan := make(chan schema.GroupVersionKind, 1024) gvksFailingTolist := watch.NewSet() @@ -296,32 +293,33 @@ func (c *CacheManager) manageCache(ctx context.Context) { close(stopChan) close(gvkErrdChan) return - case <-c.cacheManagementTicker.C: + case <-c.backgroundManagementTicker.C: c.mu.Lock() - c.makeUpdates(ctx) + c.wipeCacheIfNeeded(ctx) // spin up new goroutines to relist if new gvks to relist are // populated from makeUpdates. - if c.gvksToRelist.Size() != 0 { + if c.gvksToList.Size() != 0 { // stop any goroutines that were relisting before + // as we may no longer be interested in those gvks stopChan <- true // also try to catch any gvks that are in the aggregator // but are failing to list from a previous replay. for _, gvk := range gvksFailingTolist.Items() { - if c.gvkAggregator.IsPresent(gvk) { - c.gvksToRelist.Add(gvk) + if c.specifiedGVKs.IsPresent(gvk) { + c.gvksToList.Add(gvk) } } // save all gvks that need relisting - gvksToRelistForLoop := c.gvksToRelist.Items() + gvksToRelistForLoop := c.gvksToList.Items() // clean state gvksFailingTolist = watch.NewSet() - c.gvksToRelist = watch.NewSet() + c.gvksToList = watch.NewSet() - stopChan = make(chan bool) + stopChan = make(chan bool, 1) go c.replayLoop(ctx, gvksToRelistForLoop, stopChan) go gvksFailingToListReconciler(stopChan) @@ -347,7 +345,7 @@ func (c *CacheManager) replayLoop(ctx context.Context, gvksToRelist []schema.Gro } operation := func() (bool, error) { - if err := c.listAndSyncDataForGVK(ctx, gvk); err != nil { + if err := c.syncGVKInstances(ctx, gvk); err != nil { return false, err } @@ -361,57 +359,21 @@ func (c *CacheManager) replayLoop(ctx context.Context, gvksToRelist []schema.Gro } } -// listAndSyncData returns a set of gvks that were successfully listed and synced. -func (c *CacheManager) listAndSyncData(ctx context.Context, gvks []schema.GroupVersionKind) *watch.Set { - gvksSuccessfullySynced := watch.NewSet() - for _, gvk := range gvks { - err := c.listAndSyncDataForGVK(ctx, gvk) - if err != nil { - log.Error(err, "internal: error syncing gvks cache data") - // we don't remove this gvk as we will try to re-add it later - // we also don't return on this error to be able to list and sync - // other gvks in order to protect against a bad gvk. - } else { - gvksSuccessfullySynced.Add(gvk) - } - } - return gvksSuccessfullySynced -} - -// makeUpdates performs a conditional wipe followed by a replay if necessary as -// given by the current spec (currentGVKsInAgg, excluderChanged) at the time of the call. -func (c *CacheManager) makeUpdates(ctx context.Context) { - // assumes the caller has lock - - currentGVKsInAgg := watch.NewSet() - currentGVKsInAgg.Add(c.gvkAggregator.ListAllGVKs()...) - - if c.watchedSet.Equals(currentGVKsInAgg) && !c.excluderChanged { - // nothing to do if both sets are the same and the excluder didn't change - // and there are no gvks that need relisting from a previous wipe - return - } - - gvksToDelete := c.watchedSet.Difference(currentGVKsInAgg) - newGVKsToSync := currentGVKsInAgg.Difference(c.watchedSet) - gvksToReplay := c.watchedSet.Intersection(currentGVKsInAgg) - - if gvksToDelete.Size() != 0 || newGVKsToSync.Size() != 0 { - // in this case we need to replace the watch set again since there - // is drift between the aggregator and the currently watched gvks - if err := c.replaceWatchSet(ctx, currentGVKsInAgg); err != nil { - log.Error(err, "internal: error replacing watch set") - } - } - +// wipeCacheIfNeeded performs a cache wipe if there are any gvks needing to be removed +// from the cache or if the excluder has changed. It also marks which gvks need to be +// re listed again in the opa cache after the wipe. +// assumes the caller has lock. +func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { // remove any gvks not needing to be synced anymore // or re evaluate all if the excluder changed. - if gvksToDelete.Size() > 0 || c.excluderChanged { + if c.gvksToDeleteFromCache.Size() > 0 || c.excluderChanged { if err := c.wipeData(ctx); err != nil { log.Error(err, "internal: error wiping cache") } else { + c.gvksToDeleteFromCache = watch.NewSet() c.excluderChanged = false } - c.gvksToRelist.AddSet(gvksToReplay) + + c.gvksToList.AddSet(c.watchedSet) } } diff --git a/pkg/controller/cachemanager/cachemanager_integration_test.go b/pkg/syncutil/cachemanager/cachemanager_integration_test.go similarity index 91% rename from pkg/controller/cachemanager/cachemanager_integration_test.go rename to pkg/syncutil/cachemanager/cachemanager_integration_test.go index 2a6f9b1ac56..e3b620694fe 100644 --- a/pkg/controller/cachemanager/cachemanager_integration_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_integration_test.go @@ -14,8 +14,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -// TestCacheManager_listAndSyncData tests that the cache manager can add gvks to the data store. -func TestCacheManager_listAndSyncData(t *testing.T) { +// TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the opa client. +func TestCacheManager_syncGVKInstances(t *testing.T) { cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) configMapGVK := schema.GroupVersionKind{ @@ -30,7 +30,7 @@ func TestCacheManager_listAndSyncData(t *testing.T) { require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") cacheManager.watchedSet.Add(configMapGVK) - require.NoError(t, cacheManager.listAndSyncDataForGVK(ctx, configMapGVK)) + require.NoError(t, cacheManager.syncGVKInstances(ctx, configMapGVK)) opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -63,8 +63,8 @@ func TestCacheManager_listAndSyncData(t *testing.T) { require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") cacheManager.watchedSet.Add(podGVK) - syncedSet := cacheManager.listAndSyncData(ctx, []schema.GroupVersionKind{configMapGVK, podGVK}) - require.ElementsMatch(t, syncedSet.Items(), []schema.GroupVersionKind{configMapGVK, podGVK}) + require.NoError(t, cacheManager.syncGVKInstances(ctx, configMapGVK)) + require.NoError(t, cacheManager.syncGVKInstances(ctx, podGVK)) expected = map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -118,8 +118,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) // now assert that the gvkAggregator looks as expected - cacheManager.gvkAggregator.IsPresent(configMapGVK) - gvks := cacheManager.gvkAggregator.List(syncSourceOne) + cacheManager.specifiedGVKs.IsPresent(configMapGVK) + gvks := cacheManager.specifiedGVKs.List(syncSourceOne) require.Len(t, gvks, 2) _, foundConfigMap := gvks[configMapGVK] require.True(t, foundConfigMap) @@ -135,8 +135,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { } require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) // now assert that the gvkAggregator looks as expected - cacheManager.gvkAggregator.IsPresent(configMapGVK) - gvks = cacheManager.gvkAggregator.List(syncSourceOne) + cacheManager.specifiedGVKs.IsPresent(configMapGVK) + gvks = cacheManager.specifiedGVKs.List(syncSourceOne) require.Len(t, gvks, 1) _, foundConfigMap = gvks[configMapGVK] require.True(t, foundConfigMap) @@ -148,8 +148,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) reqConditionForAgg := func() bool { - cacheManager.gvkAggregator.IsPresent(configMapGVK) - gvks := cacheManager.gvkAggregator.List(syncSourceOne) + cacheManager.specifiedGVKs.IsPresent(configMapGVK) + gvks := cacheManager.specifiedGVKs.List(syncSourceOne) if len(gvks) != 1 { return false } @@ -158,7 +158,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { return false } - gvks2 := cacheManager.gvkAggregator.List(syncSourceTwo) + gvks2 := cacheManager.specifiedGVKs.List(syncSourceTwo) if len(gvks2) != 1 { return false } @@ -191,7 +191,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // and expect an empty cache and empty aggregator require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) - require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 0) + require.True(t, len(cacheManager.specifiedGVKs.ListAllGVKs()) == 0) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") @@ -249,8 +249,8 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) // make sure the gvk is still in gvkAggregator - require.True(t, len(cacheManager.gvkAggregator.ListAllGVKs()) == 1) - require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + require.True(t, len(cacheManager.specifiedGVKs.ListAllGVKs()) == 1) + require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") diff --git a/pkg/controller/cachemanager/cachemanager_suite_test.go b/pkg/syncutil/cachemanager/cachemanager_suite_test.go similarity index 99% rename from pkg/controller/cachemanager/cachemanager_suite_test.go rename to pkg/syncutil/cachemanager/cachemanager_suite_test.go index 5e25847840f..9000df0e32a 100644 --- a/pkg/controller/cachemanager/cachemanager_suite_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_suite_test.go @@ -5,8 +5,8 @@ import ( "testing" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/cachemanager/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" diff --git a/pkg/controller/cachemanager/cachemanager_test.go b/pkg/syncutil/cachemanager/cachemanager_test.go similarity index 79% rename from pkg/controller/cachemanager/cachemanager_test.go rename to pkg/syncutil/cachemanager/cachemanager_test.go index 7730647eed4..6b9f5be4904 100644 --- a/pkg/controller/cachemanager/cachemanager_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_test.go @@ -14,8 +14,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -// TestCacheManager_makeUpdates tests that we can add gvks to the data store. -func TestCacheManager_makeUpdates(t *testing.T) { +// TestCacheManager_wipeCacheIfNeeded. +func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { cacheManager, ctx := makeUnitCacheManagerForTest(t) opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -25,23 +25,20 @@ func TestCacheManager_makeUpdates(t *testing.T) { cm := unstructuredFor(configMapGVK, "config-test-1") _, err := opaClient.AddData(ctx, cm) require.NoError(t, err, "adding ConfigMap config-test-1 in opa") - cacheManager.watchedSet.Add(configMapGVK) // prep gvkAggregator for updates to be picked up in makeUpdates podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - require.NoError(t, cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.specifiedGVKs.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) - cacheManager.makeUpdates(ctx) + cacheManager.gvksToDeleteFromCache.Add(configMapGVK) + cacheManager.wipeCacheIfNeeded(ctx) - // two things should happen: - // - the cache manager starts watching the pod gvk - // - the cache manager stops watching the configMap gvk require.False(t, opaClient.HasGVK(configMapGVK)) - require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK}) + require.ElementsMatch(t, cacheManager.specifiedGVKs.ListAllGVKs(), []schema.GroupVersionKind{podGVK}) } -// TestCacheManager_makeUpdates_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. -func TestCacheManager_makeUpdates_excluderChanges(t *testing.T) { +// TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. +func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { cacheManager, ctx := makeUnitCacheManagerForTest(t) opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -60,14 +57,14 @@ func TestCacheManager_makeUpdates_excluderChanges(t *testing.T) { _, err = opaClient.AddData(ctx, pod) require.NoError(t, err, "adding Pod pod-test-1 in opa") cacheManager.watchedSet.Add(podGVK) - require.NoError(t, cacheManager.gvkAggregator.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK, configMapGVK})) cacheManager.ExcludeProcesses(newSyncExcluderFor("excluded-ns")) - cacheManager.makeUpdates(ctx) + cacheManager.wipeCacheIfNeeded(ctx) // the cache manager should not be watching any of the gvks that are now excluded require.False(t, opaClient.HasGVK(configMapGVK)) require.False(t, opaClient.HasGVK(podGVK)) + require.False(t, cacheManager.excluderChanged) } // TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. @@ -128,8 +125,8 @@ func TestCacheManager_AddObject_processExclusion(t *testing.T) { require.False(t, opaClient.Contains(map[fakes.OpaKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) } -// TestCacheManager_errors tests that the cache manager responds to errors from the opa client. -func TestCacheManager_errors(t *testing.T) { +// TestCacheManager_opaClient_errors tests that the cache manager responds to errors from the opa client. +func TestCacheManager_opaClient_errors(t *testing.T) { cm, ctx := makeUnitCacheManagerForTest(t) opaClient, ok := cm.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -150,7 +147,7 @@ func TestCacheManager_errors(t *testing.T) { // TestCacheManager_AddSource tests that we can modify the gvk aggregator and watched set when adding a new source. func TestCacheManager_AddSource(t *testing.T) { - cacheManager, ctx := makeUnitCacheManagerForTest(t) + cacheManager, _, ctx := makeCacheManagerForTest(t, false, true) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} nsGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} @@ -162,43 +159,43 @@ func TestCacheManager_AddSource(t *testing.T) { require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) // ... expect the aggregator to dedup - require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) - require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK, configMapGVK}) // adding a source without a previously added gvk ... require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{configMapGVK})) // ... should not remove any gvks that are still referenced by other sources - require.True(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) - require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) // adding a source that modifies the only reference to a gvk ... require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{nsGVK})) // ... will effectively remove the gvk from the aggregator - require.False(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) - require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) - require.True(t, cacheManager.gvkAggregator.IsPresent(nsGVK)) + require.False(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(nsGVK)) } // TestCacheManager_RemoveSource tests that we can modify the gvk aggregator when removing a source. func TestCacheManager_RemoveSource(t *testing.T) { - cacheManager, ctx := makeUnitCacheManagerForTest(t) + cacheManager, _, ctx := makeCacheManagerForTest(t, false, true) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} sourceA := aggregator.Key{Source: "a", ID: "source"} sourceB := aggregator.Key{Source: "b", ID: "source"} // seed the gvk aggregator - require.NoError(t, cacheManager.gvkAggregator.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.gvkAggregator.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + require.NoError(t, cacheManager.specifiedGVKs.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.specifiedGVKs.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) // removing a source that is not the only one referencing a gvk ... require.NoError(t, cacheManager.RemoveSource(ctx, sourceB)) // ... should not remove any gvks that are still referenced by other sources - require.True(t, cacheManager.gvkAggregator.IsPresent(podGVK)) - require.False(t, cacheManager.gvkAggregator.IsPresent(configMapGVK)) + require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) + require.False(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) require.NoError(t, cacheManager.RemoveSource(ctx, sourceA)) - require.False(t, cacheManager.gvkAggregator.IsPresent(podGVK)) + require.False(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) } From ead18f2606c251145c81e0b0831f5b13f7c93cb3 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:32:08 +0000 Subject: [PATCH 20/58] use a set to record failing gvks Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/cachemanager/cachemanager.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index e0d7a20d480..5342c2bd3ae 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -270,28 +270,17 @@ func (c *CacheManager) syncGVKInstances(ctx context.Context, gvk schema.GroupVer return nil } +var gvksFailingTolist *watch.Set + func (c *CacheManager) manageCache(ctx context.Context) { // stopChan is used to stop any list operations still in progress stopChan := make(chan bool, 1) - gvkErrdChan := make(chan schema.GroupVersionKind, 1024) - gvksFailingTolist := watch.NewSet() - - gvksFailingToListReconciler := func(stopChan <-chan bool) { - for { - select { - case <-stopChan: - return - case gvk := <-gvkErrdChan: - gvksFailingTolist.Add(gvk) - } - } - } + gvksFailingTolist = watch.NewSet() for { select { case <-ctx.Done(): close(stopChan) - close(gvkErrdChan) return case <-c.backgroundManagementTicker.C: c.mu.Lock() @@ -322,7 +311,6 @@ func (c *CacheManager) manageCache(ctx context.Context) { stopChan = make(chan bool, 1) go c.replayLoop(ctx, gvksToRelistForLoop, stopChan) - go gvksFailingToListReconciler(stopChan) } c.mu.Unlock() } @@ -354,6 +342,7 @@ func (c *CacheManager) replayLoop(ctx context.Context, gvksToRelist []schema.Gro if err := wait.ExponentialBackoff(backoff, operation); err != nil { log.Error(err, "internal: error listings gvk cache data", "gvk", gvk) + gvksFailingTolist.Add(gvk) } } } From a42ef436d9c38e51421b8323b8acfe569f8089d9 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 24 Jul 2023 19:07:17 +0000 Subject: [PATCH 21/58] rework the filtered point Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/cachemanager/cachemanager.go | 42 +++++++++++------------ 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index 5342c2bd3ae..35eb8f5fc9a 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -171,14 +171,11 @@ func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { } func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Unstructured) error { - // only perform work for watched gvks - if gvk := instance.GroupVersionKind(); !c.watchedSet.Contains(gvk) { - return nil - } + gvk := instance.GroupVersionKind() isNamespaceExcluded, err := c.processExcluder.IsNamespaceExcluded(process.Sync, instance) if err != nil { - return fmt.Errorf("error while excluding namespaces: %w", err) + return fmt.Errorf("error while excluding namespaces for gvk: %s: %w", gvk, err) } // bail because it means we should not be @@ -189,17 +186,19 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns } syncKey := syncutil.GetKeyForSyncMetrics(instance.GetNamespace(), instance.GetName()) - _, err = c.opa.AddData(ctx, instance) - if err != nil { - c.syncMetricsCache.AddObject( - syncKey, - syncutil.Tags{ - Kind: instance.GetKind(), - Status: metrics.ErrorStatus, - }, - ) - - return err + if c.watchedSet.Contains(gvk) { + _, err = c.opa.AddData(ctx, instance) + if err != nil { + c.syncMetricsCache.AddObject( + syncKey, + syncutil.Tags{ + Kind: instance.GetKind(), + Status: metrics.ErrorStatus, + }, + ) + + return err + } } c.tracker.ForData(instance.GroupVersionKind()).Observe(instance) @@ -214,13 +213,12 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns } func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error { - // only perform work for watched gvks - if gvk := instance.GroupVersionKind(); !c.watchedSet.Contains(gvk) { - return nil - } + gvk := instance.GroupVersionKind() - if _, err := c.opa.RemoveData(ctx, instance); err != nil { - return err + if c.watchedSet.Contains(gvk) { + if _, err := c.opa.RemoveData(ctx, instance); err != nil { + return err + } } // only delete from metrics map if the data removal was succcesful From 12f39da298e115b17afde67c54b3a5c2a3918cad Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 25 Jul 2023 23:03:21 +0000 Subject: [PATCH 22/58] review: naming, docs, comments, polish Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/aggregator/aggregator.go | 3 +- pkg/syncutil/cachemanager/cachemanager.go | 62 ++++++++++--------- .../cachemanager_integration_test.go | 33 +++++----- .../cachemanager/cachemanager_test.go | 2 +- test/testutils/const.go | 8 --- 5 files changed, 56 insertions(+), 52 deletions(-) delete mode 100644 test/testutils/const.go diff --git a/pkg/syncutil/aggregator/aggregator.go b/pkg/syncutil/aggregator/aggregator.go index 57f2572618c..6127b505bce 100644 --- a/pkg/syncutil/aggregator/aggregator.go +++ b/pkg/syncutil/aggregator/aggregator.go @@ -106,7 +106,8 @@ func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { return b.store[k] } -func (b *GVKAgreggator) ListAllGVKs() []schema.GroupVersionKind { +// GVKs returns a list of all of the schema.GroupVersionKind that are aggregated. +func (b *GVKAgreggator) GVKs() []schema.GroupVersionKind { b.mu.Lock() defer b.mu.Unlock() diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index 35eb8f5fc9a..f2df971a499 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -20,7 +20,16 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -var log = logf.Log.WithName("cache-manager") +var ( + log = logf.Log.WithName("cache-manager") + backoff = wait.Backoff{ + Duration: time.Second, + Factor: 2, + Jitter: 0.1, + Steps: 3, + } +) +var gvksFailingTolist *watch.Set type Config struct { Opa syncutil.OpaDataClient @@ -49,6 +58,9 @@ type CacheManager struct { watchedSet *watch.Set backgroundManagementTicker time.Ticker reader client.Reader + + // stopChan is used to stop any list operations still in progress + stopChan chan bool } func NewCacheManager(config *Config) (*CacheManager, error) { @@ -80,6 +92,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { gvksToList: watch.NewSet(), backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), + stopChan: make(chan bool, 1), }, nil } @@ -92,6 +105,7 @@ func (c *CacheManager) Start(ctx context.Context) error { // AddSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. +// It errors out if there is an issue removing the Key internally or replacing the watches. func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() @@ -115,7 +129,7 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, // assumes caller has lock. func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() - newWatchSet.Add(c.specifiedGVKs.ListAllGVKs()...) + newWatchSet.Add(c.specifiedGVKs.GVKs()...) if newWatchSet.Equals(c.watchedSet) { // nothing to do as the sets are equal @@ -139,6 +153,8 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } +// RemoveSource removes the watches of the GVKs for a given aggregator.Key. +// It errors out if there is an issue removing the Key internally or replacing the watches. func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() @@ -149,12 +165,14 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke // make changes to the watches if err := c.replaceWatchSet(ctx); err != nil { - return fmt.Errorf("error removing watches for source %s: %w", sourceKey, err) + return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } return nil } +// ExcludeProcesses swaps the current process excluder with the new *process.Excluder. +// It's a no-op if the two excluder are equal. func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { c.mu.Lock() defer c.mu.Unlock() @@ -175,11 +193,11 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns isNamespaceExcluded, err := c.processExcluder.IsNamespaceExcluded(process.Sync, instance) if err != nil { - return fmt.Errorf("error while excluding namespaces for gvk: %s: %w", gvk, err) + return fmt.Errorf("error while excluding namespaces for gvk: %v: %w", gvk.String(), err) } // bail because it means we should not be - // syncing this gvk + // syncing this gvk's objects as it is namespace excluded. if isNamespaceExcluded { c.tracker.ForData(instance.GroupVersionKind()).CancelExpect(instance) return nil @@ -244,7 +262,7 @@ func (c *CacheManager) ReportSyncMetrics() { c.syncMetricsCache.ReportSync() } -func (c *CacheManager) syncGVKInstances(ctx context.Context, gvk schema.GroupVersionKind) error { +func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) error { u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: gvk.Group, @@ -257,8 +275,6 @@ func (c *CacheManager) syncGVKInstances(ctx context.Context, gvk schema.GroupVer return fmt.Errorf("replaying data for %+v: %w", gvk, err) } - defer c.ReportSyncMetrics() - for i := range u.Items { if err := c.AddObject(ctx, &u.Items[i]); err != nil { return fmt.Errorf("adding data for %+v: %w", gvk, err) @@ -268,17 +284,13 @@ func (c *CacheManager) syncGVKInstances(ctx context.Context, gvk schema.GroupVer return nil } -var gvksFailingTolist *watch.Set - func (c *CacheManager) manageCache(ctx context.Context) { - // stopChan is used to stop any list operations still in progress - stopChan := make(chan bool, 1) gvksFailingTolist = watch.NewSet() for { select { case <-ctx.Done(): - close(stopChan) + close(c.stopChan) return case <-c.backgroundManagementTicker.C: c.mu.Lock() @@ -289,7 +301,7 @@ func (c *CacheManager) manageCache(ctx context.Context) { if c.gvksToList.Size() != 0 { // stop any goroutines that were relisting before // as we may no longer be interested in those gvks - stopChan <- true + c.stopChan <- true // also try to catch any gvks that are in the aggregator // but are failing to list from a previous replay. @@ -306,32 +318,25 @@ func (c *CacheManager) manageCache(ctx context.Context) { gvksFailingTolist = watch.NewSet() c.gvksToList = watch.NewSet() - stopChan = make(chan bool, 1) + c.stopChan = make(chan bool, 1) - go c.replayLoop(ctx, gvksToRelistForLoop, stopChan) + go c.replayGVKs(ctx, gvksToRelistForLoop) } c.mu.Unlock() } } } -func (c *CacheManager) replayLoop(ctx context.Context, gvksToRelist []schema.GroupVersionKind, stopChan <-chan bool) { +func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.GroupVersionKind) { for _, gvk := range gvksToRelist { select { case <-ctx.Done(): return - case <-stopChan: + case <-c.stopChan: return default: - backoff := wait.Backoff{ - Duration: time.Second, - Factor: 2, - Jitter: 0.1, - Steps: 3, - } - operation := func() (bool, error) { - if err := c.syncGVKInstances(ctx, gvk); err != nil { + if err := c.syncGVK(ctx, gvk); err != nil { return false, err } @@ -344,6 +349,8 @@ func (c *CacheManager) replayLoop(ctx context.Context, gvksToRelist []schema.Gro } } } + + c.ReportSyncMetrics() } // wipeCacheIfNeeded performs a cache wipe if there are any gvks needing to be removed @@ -359,8 +366,7 @@ func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { } else { c.gvksToDeleteFromCache = watch.NewSet() c.excluderChanged = false + c.gvksToList.AddSet(c.watchedSet) } - - c.gvksToList.AddSet(c.watchedSet) } } diff --git a/pkg/syncutil/cachemanager/cachemanager_integration_test.go b/pkg/syncutil/cachemanager/cachemanager_integration_test.go index e3b620694fe..7a920c88adf 100644 --- a/pkg/syncutil/cachemanager/cachemanager_integration_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_integration_test.go @@ -2,18 +2,23 @@ package cachemanager import ( "testing" + "time" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" - "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ) +const ( + eventuallyTimeout = 10 * time.Second + eventuallyTicker = 2 * time.Second +) + // TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the opa client. func TestCacheManager_syncGVKInstances(t *testing.T) { cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) @@ -30,7 +35,7 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") cacheManager.watchedSet.Add(configMapGVK) - require.NoError(t, cacheManager.syncGVKInstances(ctx, configMapGVK)) + require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) require.True(t, ok) @@ -63,8 +68,8 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") cacheManager.watchedSet.Add(podGVK) - require.NoError(t, cacheManager.syncGVKInstances(ctx, configMapGVK)) - require.NoError(t, cacheManager.syncGVKInstances(ctx, podGVK)) + require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) + require.NoError(t, cacheManager.syncGVK(ctx, podGVK)) expected = map[fakes.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -115,7 +120,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected cacheManager.specifiedGVKs.IsPresent(configMapGVK) @@ -133,7 +138,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected cacheManager.specifiedGVKs.IsPresent(configMapGVK) gvks = cacheManager.specifiedGVKs.List(syncSourceOne) @@ -165,7 +170,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { _, found2 := gvks2[configMapGVK] return found2 } - require.Eventually(t, reqConditionForAgg, testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, reqConditionForAgg, eventuallyTimeout, eventuallyTicker) require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) expected2 := map[fakes.OpaKey]interface{}{ @@ -173,7 +178,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected2), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, expected2), eventuallyTimeout, eventuallyTicker) // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) @@ -183,15 +188,15 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected3), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, expected3), eventuallyTimeout, eventuallyTicker) // now remove all the sources require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) - require.True(t, len(cacheManager.specifiedGVKs.ListAllGVKs()) == 0) + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.True(t, len(cacheManager.specifiedGVKs.GVKs()) == 0) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") @@ -218,7 +223,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) // check that everything is well added at first - require.Eventually(t, expectedCheck(opaClient, expected), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // make sure that replacing w same process excluder is a no op sameExcluder := process.New() @@ -247,9 +252,9 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { }) cacheManager.ExcludeProcesses(excluder) - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), testutils.EventuallyTimeout, testutils.EventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) // make sure the gvk is still in gvkAggregator - require.True(t, len(cacheManager.specifiedGVKs.ListAllGVKs()) == 1) + require.True(t, len(cacheManager.specifiedGVKs.GVKs()) == 1) require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) // cleanup diff --git a/pkg/syncutil/cachemanager/cachemanager_test.go b/pkg/syncutil/cachemanager/cachemanager_test.go index 6b9f5be4904..54edd863470 100644 --- a/pkg/syncutil/cachemanager/cachemanager_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_test.go @@ -34,7 +34,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { cacheManager.wipeCacheIfNeeded(ctx) require.False(t, opaClient.HasGVK(configMapGVK)) - require.ElementsMatch(t, cacheManager.specifiedGVKs.ListAllGVKs(), []schema.GroupVersionKind{podGVK}) + require.ElementsMatch(t, cacheManager.specifiedGVKs.GVKs(), []schema.GroupVersionKind{podGVK}) } // TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. diff --git a/test/testutils/const.go b/test/testutils/const.go deleted file mode 100644 index e89412410b7..00000000000 --- a/test/testutils/const.go +++ /dev/null @@ -1,8 +0,0 @@ -package testutils - -import "time" - -const ( - EventuallyTimeout = 10 * time.Second - EventuallyTicker = 1 * time.Second -) From b67121b86db08c4a9b4dabf3d274c28893bf6b71 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 25 Jul 2023 23:20:36 +0000 Subject: [PATCH 23/58] review: gate gvks to list differently Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/cachemanager/cachemanager.go | 34 +++++++---------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index f2df971a499..3bbfbbea9e6 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -29,7 +29,6 @@ var ( Steps: 3, } ) -var gvksFailingTolist *watch.Set type Config struct { Opa syncutil.OpaDataClient @@ -43,9 +42,10 @@ type Config struct { } type CacheManager struct { + watchedSet *watch.Set processExcluder *process.Excluder specifiedGVKs *aggregator.GVKAgreggator - gvksToList *watch.Set + needToList bool gvksToDeleteFromCache *watch.Set excluderChanged bool // mu guards access to any of the fields above @@ -55,7 +55,6 @@ type CacheManager struct { syncMetricsCache *syncutil.MetricsCache tracker *readiness.Tracker registrar *watch.Registrar - watchedSet *watch.Set backgroundManagementTicker time.Ticker reader client.Reader @@ -89,7 +88,6 @@ func NewCacheManager(config *Config) (*CacheManager, error) { watchedSet: config.WatchedSet, reader: config.Reader, specifiedGVKs: aggregator.NewGVKAggregator(), - gvksToList: watch.NewSet(), backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), stopChan: make(chan bool, 1), @@ -105,7 +103,7 @@ func (c *CacheManager) Start(ctx context.Context) error { // AddSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. -// It errors out if there is an issue removing the Key internally or replacing the watches. +// It errors out if there is an issue adding the Key internally or replacing the watches. func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() @@ -172,7 +170,7 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke } // ExcludeProcesses swaps the current process excluder with the new *process.Excluder. -// It's a no-op if the two excluder are equal. +// It's a no-op if the two excluders are equal. func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { c.mu.Lock() defer c.mu.Unlock() @@ -285,8 +283,6 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) } func (c *CacheManager) manageCache(ctx context.Context) { - gvksFailingTolist = watch.NewSet() - for { select { case <-ctx.Done(): @@ -298,26 +294,16 @@ func (c *CacheManager) manageCache(ctx context.Context) { // spin up new goroutines to relist if new gvks to relist are // populated from makeUpdates. - if c.gvksToList.Size() != 0 { + if c.needToList { // stop any goroutines that were relisting before // as we may no longer be interested in those gvks c.stopChan <- true - // also try to catch any gvks that are in the aggregator - // but are failing to list from a previous replay. - for _, gvk := range gvksFailingTolist.Items() { - if c.specifiedGVKs.IsPresent(gvk) { - c.gvksToList.Add(gvk) - } - } - - // save all gvks that need relisting - gvksToRelistForLoop := c.gvksToList.Items() + // assume all gvks need to be relist + gvksToRelistForLoop := c.specifiedGVKs.GVKs() // clean state - gvksFailingTolist = watch.NewSet() - c.gvksToList = watch.NewSet() - + c.needToList = false c.stopChan = make(chan bool, 1) go c.replayGVKs(ctx, gvksToRelistForLoop) @@ -345,7 +331,7 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro if err := wait.ExponentialBackoff(backoff, operation); err != nil { log.Error(err, "internal: error listings gvk cache data", "gvk", gvk) - gvksFailingTolist.Add(gvk) + // this gvk will be retried next time we relist from manageCache } } } @@ -366,7 +352,7 @@ func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { } else { c.gvksToDeleteFromCache = watch.NewSet() c.excluderChanged = false - c.gvksToList.AddSet(c.watchedSet) + c.needToList = true } } } From 80210a68fdc3ebadf9509d8b4bdb585afcc19f81 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:57:34 +0000 Subject: [PATCH 24/58] make the reply goroutine resilient Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/cachemanager/cachemanager.go | 48 +++++++++++++---------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index 3bbfbbea9e6..a2c8f31d13c 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -134,7 +134,7 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return nil } - // record any gvks that need to be deleted + // record any gvks that need to be deleted from the opa cache. c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) var innerError error @@ -292,8 +292,7 @@ func (c *CacheManager) manageCache(ctx context.Context) { c.mu.Lock() c.wipeCacheIfNeeded(ctx) - // spin up new goroutines to relist if new gvks to relist are - // populated from makeUpdates. + // spin up new goroutines to relist gvks as there has been a wipe if c.needToList { // stop any goroutines that were relisting before // as we may no longer be interested in those gvks @@ -314,29 +313,36 @@ func (c *CacheManager) manageCache(ctx context.Context) { } func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.GroupVersionKind) { - for _, gvk := range gvksToRelist { - select { - case <-ctx.Done(): - return - case <-c.stopChan: - return - default: - operation := func() (bool, error) { - if err := c.syncGVK(ctx, gvk); err != nil { - return false, err + gvksSet := watch.NewSet() + gvksSet.Add(gvksToRelist...) + + for gvksSet.Size() != 0 { + gvkItems := gvksSet.Items() + + for _, gvk := range gvkItems { + select { + case <-ctx.Done(): + return + case <-c.stopChan: + return + default: + operation := func() (bool, error) { + if err := c.syncGVK(ctx, gvk); err != nil { + return false, err + } + return true, nil } - return true, nil - } - - if err := wait.ExponentialBackoff(backoff, operation); err != nil { - log.Error(err, "internal: error listings gvk cache data", "gvk", gvk) - // this gvk will be retried next time we relist from manageCache + if err := wait.ExponentialBackoff(backoff, operation); err != nil { + log.Error(err, "internal: error listings gvk cache data", "gvk", gvk) + } else { + gvksSet.Remove(gvk) + } } } - } - c.ReportSyncMetrics() + c.ReportSyncMetrics() + } } // wipeCacheIfNeeded performs a cache wipe if there are any gvks needing to be removed From 1f58357b2d01e1e2db6c00f84613cdd0663eafaf Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:59:12 +0000 Subject: [PATCH 25/58] test replay resiliency in cm Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../config/config_controller_test.go | 2 +- pkg/controller/config/fakes_test.go | 35 ----------- pkg/fakes/reader.go | 19 ++++++ .../cachemanager_integration_test.go | 63 +++++++++++++++++++ 4 files changed, 83 insertions(+), 36 deletions(-) delete mode 100644 pkg/controller/config/fakes_test.go create mode 100644 pkg/fakes/reader.go diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 3a064862640..ffe62e3eeb2 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -645,7 +645,7 @@ func TestConfig_Retries(t *testing.T) { // Use our special hookReader to inject controlled failures failPlease := make(chan string, 1) - rec.reader = hookReader{ + rec.reader = fakes.HookReader{ Reader: mgr.GetCache(), ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // Return an error the first go-around. diff --git a/pkg/controller/config/fakes_test.go b/pkg/controller/config/fakes_test.go deleted file mode 100644 index 3c209d4e3d4..00000000000 --- a/pkg/controller/config/fakes_test.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// hookReader is a client.Reader with overrideable methods. -type hookReader struct { - client.Reader - ListFunc func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error -} - -func (r hookReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - if r.ListFunc != nil { - return r.ListFunc(ctx, list, opts...) - } - return r.Reader.List(ctx, list, opts...) -} diff --git a/pkg/fakes/reader.go b/pkg/fakes/reader.go new file mode 100644 index 00000000000..0d930404413 --- /dev/null +++ b/pkg/fakes/reader.go @@ -0,0 +1,19 @@ +package fakes + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type HookReader struct { + client.Reader + ListFunc func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error +} + +func (r HookReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if r.ListFunc != nil { + return r.ListFunc(ctx, list, opts...) + } + return r.Reader.List(ctx, list, opts...) +} diff --git a/pkg/syncutil/cachemanager/cachemanager_integration_test.go b/pkg/syncutil/cachemanager/cachemanager_integration_test.go index 7a920c88adf..8fc3b04078e 100644 --- a/pkg/syncutil/cachemanager/cachemanager_integration_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_integration_test.go @@ -1,6 +1,8 @@ package cachemanager import ( + "context" + "fmt" "testing" "time" @@ -12,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -19,6 +22,66 @@ const ( eventuallyTicker = 2 * time.Second ) +// TestCacheManager_replay_retries tests that we can retry GVKs that error out in the reply goroutine. +func TestCacheManager_replay_retries(t *testing.T) { + cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + + failPlease := make(chan string, 2) + cacheManager.reader = fakes.HookReader{ + Reader: c, + ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + // Return an error the first go-around. + var failKind string + select { + case failKind = <-failPlease: + default: + } + if failKind != "" && list.GetObjectKind().GroupVersionKind().Kind == failKind { + return fmt.Errorf("synthetic failure") + } + return c.List(ctx, list, opts...) + }, + } + + opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + require.True(t, ok) + + // seed one gvk + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) + + expected := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + + require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) + + // set up a scenario where the list from replay will fail a few times + failPlease <- "ConfigMapList" + failPlease <- "ConfigMapList" + + // this call should make schedule a cache wipe and a replay for the configMapGVK + require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + + expected2 := map[fakes.OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + } + require.Eventually(t, expectedCheck(opaClient, expected2), eventuallyTimeout, eventuallyTicker) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "creating ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, pod), "creating ConfigMap pod-1") +} + // TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the opa client. func TestCacheManager_syncGVKInstances(t *testing.T) { cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) From da647477a53cd990c8c69497f1985a392b4137e0 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 26 Jul 2023 22:33:56 +0000 Subject: [PATCH 26/58] review: always replace watch set plus - namings - comments Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/cachemanager/cachemanager.go | 40 +++++++++++-------- .../cachemanager_integration_test.go | 20 +++++----- .../cachemanager/cachemanager_test.go | 28 ++++++------- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/syncutil/cachemanager/cachemanager.go index a2c8f31d13c..f243ba7ae6d 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/syncutil/cachemanager/cachemanager.go @@ -44,7 +44,7 @@ type Config struct { type CacheManager struct { watchedSet *watch.Set processExcluder *process.Excluder - specifiedGVKs *aggregator.GVKAgreggator + gvksToSync *aggregator.GVKAgreggator needToList bool gvksToDeleteFromCache *watch.Set excluderChanged bool @@ -87,7 +87,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { registrar: config.Registrar, watchedSet: config.WatchedSet, reader: config.Reader, - specifiedGVKs: aggregator.NewGVKAggregator(), + gvksToSync: aggregator.NewGVKAggregator(), backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), stopChan: make(chan bool, 1), @@ -104,11 +104,12 @@ func (c *CacheManager) Start(ctx context.Context) error { // AddSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. // It errors out if there is an issue adding the Key internally or replacing the watches. +// Consumers are encouraged to retry on error. func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() - if err := c.specifiedGVKs.Upsert(sourceKey, newGVKs); err != nil { + if err := c.gvksToSync.Upsert(sourceKey, newGVKs); err != nil { return fmt.Errorf("internal error adding source: %w", err) } // as a result of upserting the new gvks for the source key, some gvks @@ -125,14 +126,10 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, // replaceWatchSet looks at the specifiedGVKs and makes changes to the registrar's watch set. // assumes caller has lock. +// replaceWatchSet may error out and that error is retryable. func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() - newWatchSet.Add(c.specifiedGVKs.GVKs()...) - - if newWatchSet.Equals(c.watchedSet) { - // nothing to do as the sets are equal - return nil - } + newWatchSet.Add(c.gvksToSync.GVKs()...) // record any gvks that need to be deleted from the opa cache. c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) @@ -143,8 +140,6 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { // Important: dynamic watches update must happen *after* updating our watchSet. // Otherwise, the sync controller will drop events for the newly watched kinds. - // Defer error handling so object re-sync happens even if the watch is hard - // errored due to a missing GVK in the watch set. innerError = c.registrar.ReplaceWatch(ctx, newWatchSet.Items()) }) @@ -153,11 +148,12 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { // RemoveSource removes the watches of the GVKs for a given aggregator.Key. // It errors out if there is an issue removing the Key internally or replacing the watches. +// Consumers are encouraged to retry on error. func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() - if err := c.specifiedGVKs.Remove(sourceKey); err != nil { + if err := c.gvksToSync.Remove(sourceKey); err != nil { return fmt.Errorf("internal error removing source: %w", err) } @@ -191,7 +187,7 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns isNamespaceExcluded, err := c.processExcluder.IsNamespaceExcluded(process.Sync, instance) if err != nil { - return fmt.Errorf("error while excluding namespaces for gvk: %v: %w", gvk.String(), err) + return fmt.Errorf("error while excluding namespaces for gvk: %+v: %w", gvk, err) } // bail because it means we should not be @@ -268,7 +264,17 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) Kind: gvk.Kind + "List", }) - err := c.reader.List(ctx, u) + var err error + c.mu.Lock() + if !c.watchedSet.Contains(gvk) { + // we are not actually wathcing the gvk + // so don't list instances for it. + err = nil + } else { + err = c.reader.List(ctx, u) + } + c.mu.Unlock() + if err != nil { return fmt.Errorf("replaying data for %+v: %w", gvk, err) } @@ -298,14 +304,14 @@ func (c *CacheManager) manageCache(ctx context.Context) { // as we may no longer be interested in those gvks c.stopChan <- true - // assume all gvks need to be relist - gvksToRelistForLoop := c.specifiedGVKs.GVKs() + // assume all gvks need to be relisted + gvksToRelist := c.gvksToSync.GVKs() // clean state c.needToList = false c.stopChan = make(chan bool, 1) - go c.replayGVKs(ctx, gvksToRelistForLoop) + go c.replayGVKs(ctx, gvksToRelist) } c.mu.Unlock() } diff --git a/pkg/syncutil/cachemanager/cachemanager_integration_test.go b/pkg/syncutil/cachemanager/cachemanager_integration_test.go index 8fc3b04078e..53568ca2a29 100644 --- a/pkg/syncutil/cachemanager/cachemanager_integration_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_integration_test.go @@ -186,8 +186,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected - cacheManager.specifiedGVKs.IsPresent(configMapGVK) - gvks := cacheManager.specifiedGVKs.List(syncSourceOne) + cacheManager.gvksToSync.IsPresent(configMapGVK) + gvks := cacheManager.gvksToSync.List(syncSourceOne) require.Len(t, gvks, 2) _, foundConfigMap := gvks[configMapGVK] require.True(t, foundConfigMap) @@ -203,8 +203,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { } require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected - cacheManager.specifiedGVKs.IsPresent(configMapGVK) - gvks = cacheManager.specifiedGVKs.List(syncSourceOne) + cacheManager.gvksToSync.IsPresent(configMapGVK) + gvks = cacheManager.gvksToSync.List(syncSourceOne) require.Len(t, gvks, 1) _, foundConfigMap = gvks[configMapGVK] require.True(t, foundConfigMap) @@ -216,8 +216,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) reqConditionForAgg := func() bool { - cacheManager.specifiedGVKs.IsPresent(configMapGVK) - gvks := cacheManager.specifiedGVKs.List(syncSourceOne) + cacheManager.gvksToSync.IsPresent(configMapGVK) + gvks := cacheManager.gvksToSync.List(syncSourceOne) if len(gvks) != 1 { return false } @@ -226,7 +226,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { return false } - gvks2 := cacheManager.specifiedGVKs.List(syncSourceTwo) + gvks2 := cacheManager.gvksToSync.List(syncSourceTwo) if len(gvks2) != 1 { return false } @@ -259,7 +259,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // and expect an empty cache and empty aggregator require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) - require.True(t, len(cacheManager.specifiedGVKs.GVKs()) == 0) + require.True(t, len(cacheManager.gvksToSync.GVKs()) == 0) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") @@ -317,8 +317,8 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) // make sure the gvk is still in gvkAggregator - require.True(t, len(cacheManager.specifiedGVKs.GVKs()) == 1) - require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) + require.True(t, len(cacheManager.gvksToSync.GVKs()) == 1) + require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") diff --git a/pkg/syncutil/cachemanager/cachemanager_test.go b/pkg/syncutil/cachemanager/cachemanager_test.go index 54edd863470..59c53cc5b06 100644 --- a/pkg/syncutil/cachemanager/cachemanager_test.go +++ b/pkg/syncutil/cachemanager/cachemanager_test.go @@ -28,13 +28,13 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { // prep gvkAggregator for updates to be picked up in makeUpdates podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - require.NoError(t, cacheManager.specifiedGVKs.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.gvksToSync.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) cacheManager.gvksToDeleteFromCache.Add(configMapGVK) cacheManager.wipeCacheIfNeeded(ctx) require.False(t, opaClient.HasGVK(configMapGVK)) - require.ElementsMatch(t, cacheManager.specifiedGVKs.GVKs(), []schema.GroupVersionKind{podGVK}) + require.ElementsMatch(t, cacheManager.gvksToSync.GVKs(), []schema.GroupVersionKind{podGVK}) } // TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. @@ -159,23 +159,23 @@ func TestCacheManager_AddSource(t *testing.T) { require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) // ... expect the aggregator to dedup - require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) - require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK, configMapGVK}) // adding a source without a previously added gvk ... require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{configMapGVK})) // ... should not remove any gvks that are still referenced by other sources - require.True(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) - require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) // adding a source that modifies the only reference to a gvk ... require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{nsGVK})) // ... will effectively remove the gvk from the aggregator - require.False(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) - require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) - require.True(t, cacheManager.specifiedGVKs.IsPresent(nsGVK)) + require.False(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(nsGVK)) } // TestCacheManager_RemoveSource tests that we can modify the gvk aggregator when removing a source. @@ -187,15 +187,15 @@ func TestCacheManager_RemoveSource(t *testing.T) { sourceB := aggregator.Key{Source: "b", ID: "source"} // seed the gvk aggregator - require.NoError(t, cacheManager.specifiedGVKs.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.specifiedGVKs.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + require.NoError(t, cacheManager.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) // removing a source that is not the only one referencing a gvk ... require.NoError(t, cacheManager.RemoveSource(ctx, sourceB)) // ... should not remove any gvks that are still referenced by other sources - require.True(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) - require.False(t, cacheManager.specifiedGVKs.IsPresent(configMapGVK)) + require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) + require.False(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) require.NoError(t, cacheManager.RemoveSource(ctx, sourceA)) - require.False(t, cacheManager.specifiedGVKs.IsPresent(podGVK)) + require.False(t, cacheManager.gvksToSync.IsPresent(podGVK)) } From f32b9bf782791e04a8c6c012f55a3bb5c8ffef08 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 26 Jul 2023 23:02:29 +0000 Subject: [PATCH 27/58] refactor: move cachemanager to its own pkg Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/{syncutil => cachemanager}/aggregator/aggregator.go | 0 pkg/{syncutil => cachemanager}/aggregator/aggregator_test.go | 0 pkg/{syncutil => }/cachemanager/cachemanager.go | 4 ++-- .../cachemanager/cachemanager_integration_test.go | 2 +- pkg/{syncutil => }/cachemanager/cachemanager_suite_test.go | 2 +- pkg/{syncutil => }/cachemanager/cachemanager_test.go | 2 +- pkg/controller/config/config_controller.go | 4 ++-- pkg/controller/config/config_controller_test.go | 2 +- pkg/controller/controller.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) rename pkg/{syncutil => cachemanager}/aggregator/aggregator.go (100%) rename pkg/{syncutil => cachemanager}/aggregator/aggregator_test.go (100%) rename pkg/{syncutil => }/cachemanager/cachemanager.go (98%) rename pkg/{syncutil => }/cachemanager/cachemanager_integration_test.go (99%) rename pkg/{syncutil => }/cachemanager/cachemanager_suite_test.go (98%) rename pkg/{syncutil => }/cachemanager/cachemanager_test.go (99%) diff --git a/pkg/syncutil/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go similarity index 100% rename from pkg/syncutil/aggregator/aggregator.go rename to pkg/cachemanager/aggregator/aggregator.go diff --git a/pkg/syncutil/aggregator/aggregator_test.go b/pkg/cachemanager/aggregator/aggregator_test.go similarity index 100% rename from pkg/syncutil/aggregator/aggregator_test.go rename to pkg/cachemanager/aggregator/aggregator_test.go diff --git a/pkg/syncutil/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go similarity index 98% rename from pkg/syncutil/cachemanager/cachemanager.go rename to pkg/cachemanager/cachemanager.go index f243ba7ae6d..704f3c49b3a 100644 --- a/pkg/syncutil/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -6,11 +6,11 @@ import ( "sync" "time" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -267,7 +267,7 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) var err error c.mu.Lock() if !c.watchedSet.Contains(gvk) { - // we are not actually wathcing the gvk + // we are not actually watching this gvk anymore // so don't list instances for it. err = nil } else { diff --git a/pkg/syncutil/cachemanager/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_integration_test.go similarity index 99% rename from pkg/syncutil/cachemanager/cachemanager_integration_test.go rename to pkg/cachemanager/cachemanager_integration_test.go index 53568ca2a29..31e144022c5 100644 --- a/pkg/syncutil/cachemanager/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_integration_test.go @@ -7,9 +7,9 @@ import ( "time" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/syncutil/cachemanager/cachemanager_suite_test.go b/pkg/cachemanager/cachemanager_suite_test.go similarity index 98% rename from pkg/syncutil/cachemanager/cachemanager_suite_test.go rename to pkg/cachemanager/cachemanager_suite_test.go index 9000df0e32a..8a3f18eb2b4 100644 --- a/pkg/syncutil/cachemanager/cachemanager_suite_test.go +++ b/pkg/cachemanager/cachemanager_suite_test.go @@ -23,7 +23,7 @@ import ( var cfg *rest.Config func TestMain(m *testing.M) { - testutils.StartControlPlane(m, &cfg, 3) + testutils.StartControlPlane(m, &cfg, 2) } func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*CacheManager, client.Client, context.Context) { diff --git a/pkg/syncutil/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go similarity index 99% rename from pkg/syncutil/cachemanager/cachemanager_test.go rename to pkg/cachemanager/cachemanager_test.go index 59c53cc5b06..61518c8ac9a 100644 --- a/pkg/syncutil/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -4,9 +4,9 @@ import ( "testing" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 6a260a5be02..e10f2277250 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -22,13 +22,13 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/aggregator" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index ffe62e3eeb2..bf418aac09d 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -26,12 +26,12 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 82769f8b979..2b84d00f7de 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -24,6 +24,7 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" + cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" @@ -33,7 +34,6 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" corev1 "k8s.io/api/core/v1" From f835a0e3ca20f2268da5a0f03c2641fc3def84c3 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 26 Jul 2023 23:52:54 +0000 Subject: [PATCH 28/58] refactor: move OpaDataClient in cachemanager Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 11 +++++-- .../cachemanager_integration_test.go | 32 +++++++++---------- pkg/cachemanager/cachemanager_suite_test.go | 3 +- pkg/cachemanager/cachemanager_test.go | 14 ++++---- .../fakeopadataclient.go} | 5 ++- .../config/config_controller_test.go | 30 ++++++++--------- pkg/syncutil/opadataclient.go | 10 +----- 7 files changed, 51 insertions(+), 54 deletions(-) rename pkg/{fakes/opadataclient.go => cachemanager/fakeopadataclient.go} (96%) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 704f3c49b3a..b48730dc3a9 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" @@ -31,7 +32,7 @@ var ( ) type Config struct { - Opa syncutil.OpaDataClient + Opa OpaDataClient SyncMetricsCache *syncutil.MetricsCache Tracker *readiness.Tracker ProcessExcluder *process.Excluder @@ -51,7 +52,7 @@ type CacheManager struct { // mu guards access to any of the fields above mu sync.RWMutex - opa syncutil.OpaDataClient + opa OpaDataClient syncMetricsCache *syncutil.MetricsCache tracker *readiness.Tracker registrar *watch.Registrar @@ -62,6 +63,12 @@ type CacheManager struct { stopChan chan bool } +// OpaDataClient is an interface for caching data. +type OpaDataClient interface { + AddData(ctx context.Context, data interface{}) (*types.Responses, error) + RemoveData(ctx context.Context, data interface{}) (*types.Responses, error) +} + func NewCacheManager(config *Config) (*CacheManager, error) { if config.WatchedSet == nil { return nil, fmt.Errorf("watchedSet must be non-nil") diff --git a/pkg/cachemanager/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_integration_test.go index 31e144022c5..908034885e9 100644 --- a/pkg/cachemanager/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_integration_test.go @@ -43,7 +43,7 @@ func TestCacheManager_replay_retries(t *testing.T) { }, } - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) // seed one gvk @@ -58,7 +58,7 @@ func TestCacheManager_replay_retries(t *testing.T) { syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[fakes.OpaKey]interface{}{ + expected := map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } @@ -72,7 +72,7 @@ func TestCacheManager_replay_retries(t *testing.T) { // this call should make schedule a cache wipe and a replay for the configMapGVK require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected2 := map[fakes.OpaKey]interface{}{ + expected2 := map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } require.Eventually(t, expectedCheck(opaClient, expected2), eventuallyTimeout, eventuallyTicker) @@ -100,9 +100,9 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { cacheManager.watchedSet.Add(configMapGVK) require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) - expected := map[fakes.OpaKey]interface{}{ + expected := map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } @@ -134,7 +134,7 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) require.NoError(t, cacheManager.syncGVK(ctx, podGVK)) - expected = map[fakes.OpaKey]interface{}{ + expected = map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, @@ -171,13 +171,13 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { pod := unstructuredFor(podGVK, "pod-1") require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[fakes.OpaKey]interface{}{ + expected := map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, @@ -197,7 +197,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now remove the podgvk and make sure we don't have pods in the cache anymore require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected = map[fakes.OpaKey]interface{}{ + expected = map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } @@ -236,7 +236,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, reqConditionForAgg, eventuallyTimeout, eventuallyTicker) require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - expected2 := map[fakes.OpaKey]interface{}{ + expected2 := map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, @@ -245,7 +245,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) - expected3 := map[fakes.OpaKey]interface{}{ + expected3 := map[OpaKey]interface{}{ // config maps no longer required by any sync source // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, @@ -258,7 +258,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, map[OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) require.True(t, len(cacheManager.gvksToSync.GVKs()) == 0) // cleanup @@ -276,10 +276,10 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { cm := unstructuredFor(configMapGVK, "config-test-1") require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) - expected := map[fakes.OpaKey]interface{}{ + expected := map[OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } @@ -315,7 +315,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { }) cacheManager.ExcludeProcesses(excluder) - require.Eventually(t, expectedCheck(opaClient, map[fakes.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, map[OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) // make sure the gvk is still in gvkAggregator require.True(t, len(cacheManager.gvksToSync.GVKs()) == 1) require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) @@ -324,7 +324,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") } -func expectedCheck(opaClient *fakes.FakeOpa, expected map[fakes.OpaKey]interface{}) func() bool { +func expectedCheck(opaClient *FakeOpa, expected map[OpaKey]interface{}) func() bool { return func() bool { if opaClient.Len() != len(expected) { return false diff --git a/pkg/cachemanager/cachemanager_suite_test.go b/pkg/cachemanager/cachemanager_suite_test.go index 8a3f18eb2b4..b11c94a8807 100644 --- a/pkg/cachemanager/cachemanager_suite_test.go +++ b/pkg/cachemanager/cachemanager_suite_test.go @@ -7,7 +7,6 @@ import ( configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" - "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" @@ -31,7 +30,7 @@ func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*Cach mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &fakes.FakeOpa{} + opaClient := &FakeOpa{} tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) processExcluder := process.Get() diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 61518c8ac9a..b296861944b 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -17,7 +17,7 @@ import ( // TestCacheManager_wipeCacheIfNeeded. func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { cacheManager, ctx := makeUnitCacheManagerForTest(t) - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) // seed one gvk @@ -26,7 +26,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { _, err := opaClient.AddData(ctx, cm) require.NoError(t, err, "adding ConfigMap config-test-1 in opa") - // prep gvkAggregator for updates to be picked up in makeUpdates + // prep cachemanager for updates to be picked up in makeUpdates podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} require.NoError(t, cacheManager.gvksToSync.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) @@ -40,7 +40,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { // TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { cacheManager, ctx := makeUnitCacheManagerForTest(t) - opaClient, ok := cacheManager.opa.(*fakes.FakeOpa) + opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) // seed gvks @@ -71,7 +71,7 @@ func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { func TestCacheManager_AddObject_RemoveObject(t *testing.T) { cm, ctx := makeUnitCacheManagerForTest(t) - opaClient, ok := cm.opa.(*fakes.FakeOpa) + opaClient, ok := cm.opa.(*FakeOpa) require.True(t, ok) pod := fakes.Pod( @@ -119,16 +119,16 @@ func TestCacheManager_AddObject_processExclusion(t *testing.T) { require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) // test that pod from excluded namespace is not cache managed - opaClient, ok := cm.opa.(*fakes.FakeOpa) + opaClient, ok := cm.opa.(*FakeOpa) require.True(t, ok) require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) - require.False(t, opaClient.Contains(map[fakes.OpaKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) + require.False(t, opaClient.Contains(map[OpaKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) } // TestCacheManager_opaClient_errors tests that the cache manager responds to errors from the opa client. func TestCacheManager_opaClient_errors(t *testing.T) { cm, ctx := makeUnitCacheManagerForTest(t) - opaClient, ok := cm.opa.(*fakes.FakeOpa) + opaClient, ok := cm.opa.(*FakeOpa) require.True(t, ok) opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err diff --git a/pkg/fakes/opadataclient.go b/pkg/cachemanager/fakeopadataclient.go similarity index 96% rename from pkg/fakes/opadataclient.go rename to pkg/cachemanager/fakeopadataclient.go index 76c117e3e94..7d531218645 100644 --- a/pkg/fakes/opadataclient.go +++ b/pkg/cachemanager/fakeopadataclient.go @@ -11,7 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package fakes +package cachemanager import ( "context" @@ -19,7 +19,6 @@ import ( gosync "sync" constraintTypes "github.com/open-policy-agent/frameworks/constraint/pkg/types" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,7 +36,7 @@ type FakeOpa struct { needsToError bool } -var _ syncutil.OpaDataClient = &FakeOpa{} +var _ OpaDataClient = &FakeOpa{} // keyFor returns an opaKey for the provided resource. // Returns error if the resource is not a runtime.Object w/ metadata. diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index bf418aac09d..24679c2e826 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -26,7 +26,7 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" @@ -126,7 +126,7 @@ func TestReconcile(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &fakes.FakeOpa{} + opaClient := &cachemanager.FakeOpa{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) @@ -142,7 +142,7 @@ func TestReconcile(t *testing.T) { CtrlName, events) require.NoError(t, err) - cacheManager, err := cm.NewCacheManager(&cm.Config{ + cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, @@ -284,7 +284,7 @@ func TestReconcile(t *testing.T) { } require.NoError(t, c.Create(ctx, fooPod)) // fooPod should be namespace excluded, hence not synced - g.Eventually(opaClient.Contains(map[fakes.OpaKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}}), timeout).ShouldNot(gomega.BeTrue()) + g.Eventually(opaClient.Contains(map[cachemanager.OpaKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}}), timeout).ShouldNot(gomega.BeTrue()) require.NoError(t, c.Delete(ctx, fooPod)) testMgrStopped() @@ -407,11 +407,11 @@ func TestConfig_DeleteSyncResources(t *testing.T) { }, timeout).Should(gomega.BeTrue()) } -func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager, tracker *readiness.Tracker, events chan event.GenericEvent, reader client.Reader, useFakeOpa bool) (syncutil.OpaDataClient, error) { +func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager, tracker *readiness.Tracker, events chan event.GenericEvent, reader client.Reader, useFakeOpa bool) (cachemanager.OpaDataClient, error) { // initialize OPA - var opaClient syncutil.OpaDataClient + var opaClient cachemanager.OpaDataClient if useFakeOpa { - opaClient = &fakes.FakeOpa{} + opaClient = &cachemanager.FakeOpa{} } else { driver, err := rego.New(rego.Tracing(true)) if err != nil { @@ -436,7 +436,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager if err != nil { return nil, fmt.Errorf("cannot create registrar: %w", err) } - cacheManager, err := cm.NewCacheManager(&cm.Config{ + cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, @@ -505,7 +505,7 @@ func TestConfig_CacheContents(t *testing.T) { opa, err := setupController(ctx, mgr, wm, tracker, events, c, true) require.NoError(t, err, "failed to set up controller") - opaClient, ok := opa.(*fakes.FakeOpa) + opaClient, ok := opa.(*cachemanager.FakeOpa) require.True(t, ok) testutils.StartManager(ctx, t, mgr) @@ -521,7 +521,7 @@ func TestConfig_CacheContents(t *testing.T) { config := configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) require.NoError(t, c.Create(ctx, config), "creating Config config") - expected := map[fakes.OpaKey]interface{}{ + expected := map[cachemanager.OpaKey]interface{}{ {Gvk: nsGVK, Key: "default"}: nil, {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, // kube-system namespace is being excluded, it should not be in opa cache @@ -547,7 +547,7 @@ func TestConfig_CacheContents(t *testing.T) { // Expect our configMap to return at some point // TODO: In the future it will remain instead of having to repopulate. - expected = map[fakes.OpaKey]interface{}{ + expected = map[cachemanager.OpaKey]interface{}{ { Gvk: configMapGVK, Key: "default/config-test-1", @@ -557,7 +557,7 @@ func TestConfig_CacheContents(t *testing.T) { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "waiting for ConfigMap to repopulate in cache") - expected = map[fakes.OpaKey]interface{}{ + expected = map[cachemanager.OpaKey]interface{}{ { Gvk: configMapGVK, Key: "kube-system/config-test-2", @@ -602,7 +602,7 @@ func TestConfig_Retries(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &fakes.FakeOpa{} + opaClient := &cachemanager.FakeOpa{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { @@ -618,7 +618,7 @@ func TestConfig_Retries(t *testing.T) { CtrlName, events) require.NoError(t, err) - cacheManager, err := cm.NewCacheManager(&cm.Config{ + cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ Opa: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, @@ -699,7 +699,7 @@ func TestConfig_Retries(t *testing.T) { } }() - expected := map[fakes.OpaKey]interface{}{ + expected := map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } g.Eventually(func() bool { diff --git a/pkg/syncutil/opadataclient.go b/pkg/syncutil/opadataclient.go index 3febc82b70c..d2762906494 100644 --- a/pkg/syncutil/opadataclient.go +++ b/pkg/syncutil/opadataclient.go @@ -18,21 +18,13 @@ package syncutil import ( "context" - "github.com/open-policy-agent/frameworks/constraint/pkg/types" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) -// CacheManagerMediator is an interface for mediating -// with a CacheManager but not actually depending on an instance. +// CacheManagerMediator interface to use the CacheManager and avoid import cycles. type CacheManagerMediator interface { AddObject(ctx context.Context, instance *unstructured.Unstructured) error RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error ReportSyncMetrics() } - -// OpaDataClient is an interface for caching data. -type OpaDataClient interface { - AddData(ctx context.Context, data interface{}) (*types.Responses, error) - RemoveData(ctx context.Context, data interface{}) (*types.Responses, error) -} From f2f41d73ef7e2daedae42ed078469ed4c74436c2 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 27 Jul 2023 01:48:46 +0000 Subject: [PATCH 29/58] refactor: no mediator interface Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 12 +- pkg/cachemanager/cachemanager_suite_test.go | 107 --------- pkg/cachemanager/cachemanager_test.go | 188 +++++++++++++++- .../cachemanager_integration_test.go | 205 +++++++++--------- pkg/controller/sync/sync_controller.go | 7 +- pkg/syncutil/opadataclient.go | 30 --- 6 files changed, 298 insertions(+), 251 deletions(-) delete mode 100644 pkg/cachemanager/cachemanager_suite_test.go rename pkg/cachemanager/{ => cachemanager_test}/cachemanager_integration_test.go (68%) delete mode 100644 pkg/syncutil/opadataclient.go diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index b48730dc3a9..70ba21dfe6e 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -86,7 +86,11 @@ func NewCacheManager(config *Config) (*CacheManager, error) { return nil, fmt.Errorf("reader must be non-nil") } - return &CacheManager{ + if config.GVKAggregator == nil { + config.GVKAggregator = aggregator.NewGVKAggregator() + } + + cm := &CacheManager{ opa: config.Opa, syncMetricsCache: config.SyncMetricsCache, tracker: config.Tracker, @@ -94,11 +98,13 @@ func NewCacheManager(config *Config) (*CacheManager, error) { registrar: config.Registrar, watchedSet: config.WatchedSet, reader: config.Reader, - gvksToSync: aggregator.NewGVKAggregator(), + gvksToSync: config.GVKAggregator, backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), stopChan: make(chan bool, 1), - }, nil + } + + return cm, nil } func (c *CacheManager) Start(ctx context.Context) error { diff --git a/pkg/cachemanager/cachemanager_suite_test.go b/pkg/cachemanager/cachemanager_suite_test.go deleted file mode 100644 index b11c94a8807..00000000000 --- a/pkg/cachemanager/cachemanager_suite_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package cachemanager - -import ( - "context" - "testing" - - configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" - "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" - syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" - "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" - "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" - "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" - testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" - "github.com/open-policy-agent/gatekeeper/v3/test/testutils" - "github.com/stretchr/testify/require" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" -) - -var cfg *rest.Config - -func TestMain(m *testing.M) { - testutils.StartControlPlane(m, &cfg, 2) -} - -func makeCacheManagerForTest(t *testing.T, startCache, startManager bool) (*CacheManager, client.Client, context.Context) { - ctx, cancelFunc := context.WithCancel(context.Background()) - mgr, wm := testutils.SetupManager(t, cfg) - - c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &FakeOpa{} - tracker, err := readiness.SetupTracker(mgr, false, false, false) - require.NoError(t, err) - processExcluder := process.Get() - processExcluder.Add([]configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - events := make(chan event.GenericEvent, 1024) - w, err := wm.NewRegistrar( - "test-cache-manager", - events) - require.NoError(t, err) - cacheManager, err := NewCacheManager(&Config{ - Opa: opaClient, - SyncMetricsCache: syncutil.NewMetricsCache(), - Tracker: tracker, - ProcessExcluder: processExcluder, - WatchedSet: watch.NewSet(), - Registrar: w, - Reader: c, - }) - require.NoError(t, err) - - if startCache { - syncAdder := syncc.Adder{ - Events: events, - CacheManager: cacheManager, - } - require.NoError(t, syncAdder.Add(mgr), "registering sync controller") - go func() { - require.NoError(t, cacheManager.Start(ctx)) - }() - - t.Cleanup(func() { - ctx.Done() - }) - } - - if startManager { - testutils.StartManager(ctx, t, mgr) - } - - t.Cleanup(func() { - cancelFunc() - }) - return cacheManager, c, ctx -} - -// makeUnitCacheManagerForTest creates a cache manager without starting the controller-runtime manager -// and without starting the cache manager background process. Note that this also means that the -// watch manager is not started and the sync controller is not started. -func makeUnitCacheManagerForTest(t *testing.T) (*CacheManager, context.Context) { - cm, _, ctx := makeCacheManagerForTest(t, false, false) - return cm, ctx -} - -func newSyncExcluderFor(nsToExclude string) *process.Excluder { - excluder := process.New() - excluder.Add([]configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{wildcard.Wildcard(nsToExclude)}, - Processes: []string{"sync"}, - }, - // exclude kube-system by default to prevent noise - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - - return excluder -} diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index b296861944b..8a9b2bf63fc 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -1,22 +1,89 @@ package cachemanager import ( + "context" "testing" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" ) +var cfg *rest.Config + +func TestMain(m *testing.M) { + testutils.StartControlPlane(m, &cfg, 2) +} + +func unitCacheManagerForTest(t *testing.T) (*CacheManager, context.Context) { + cm, _, ctx := makeCacheManager(t) + return cm, ctx +} + +func makeCacheManager(t *testing.T) (*CacheManager, client.Client, context.Context) { + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + + ctx, cancelFunc := context.WithCancel(context.Background()) + + opaClient := &FakeOpa{} + tracker, err := readiness.SetupTracker(mgr, false, false, false) + require.NoError(t, err) + processExcluder := process.Get() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + events := make(chan event.GenericEvent, 1024) + w, err := wm.NewRegistrar( + "test-cache-manager", + events) + require.NoError(t, err) + + cacheManager, err := NewCacheManager(&Config{ + Opa: opaClient, + SyncMetricsCache: syncutil.NewMetricsCache(), + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watch.NewSet(), + Registrar: w, + Reader: c, + GVKAggregator: aggregator.NewGVKAggregator(), + }) + require.NoError(t, err) + + t.Cleanup(func() { + ctx.Done() + }) + + t.Cleanup(func() { + cancelFunc() + }) + + testutils.StartManager(ctx, t, mgr) + + return cacheManager, c, ctx +} + // TestCacheManager_wipeCacheIfNeeded. func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { - cacheManager, ctx := makeUnitCacheManagerForTest(t) + cacheManager, ctx := unitCacheManagerForTest(t) opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) @@ -26,7 +93,6 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { _, err := opaClient.AddData(ctx, cm) require.NoError(t, err, "adding ConfigMap config-test-1 in opa") - // prep cachemanager for updates to be picked up in makeUpdates podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} require.NoError(t, cacheManager.gvksToSync.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) @@ -37,9 +103,80 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { require.ElementsMatch(t, cacheManager.gvksToSync.GVKs(), []schema.GroupVersionKind{podGVK}) } +// TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the opa client. +func TestCacheManager_syncGVKInstances(t *testing.T) { + cacheManager, c, ctx := makeCacheManager(t) + + configMapGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "ConfigMap", + } + // Create configMaps to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + cm2 := unstructuredFor(configMapGVK, "config-test-2") + require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + + cacheManager.watchedSet.Add(configMapGVK) + require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) + + opaClient, ok := cacheManager.opa.(*FakeOpa) + require.True(t, ok) + expected := map[OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + } + + require.Equal(t, 2, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // wipe cache + require.NoError(t, cacheManager.wipeData(ctx)) + require.False(t, opaClient.Contains(expected)) + + // create a second GVK + podGVK := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + } + // Create pods to test for + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + pod2 := unstructuredFor(podGVK, "pod-2") + require.NoError(t, c.Create(ctx, pod2), "creating Pod pod-2") + + pod3 := unstructuredFor(podGVK, "pod-3") + require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") + + cacheManager.watchedSet.Add(podGVK) + require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) + require.NoError(t, cacheManager.syncGVK(ctx, podGVK)) + + expected = map[OpaKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + {Gvk: podGVK, Key: "default/pod-2"}: nil, + {Gvk: podGVK, Key: "default/pod-3"}: nil, + } + + require.Equal(t, 5, opaClient.Len()) + require.True(t, opaClient.Contains(expected)) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") + require.NoError(t, c.Delete(ctx, pod3), "deleting Pod pod-3") + require.NoError(t, c.Delete(ctx, pod2), "deleting Pod pod-2") +} + // TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { - cacheManager, ctx := makeUnitCacheManagerForTest(t) + cacheManager, ctx := unitCacheManagerForTest(t) opaClient, ok := cacheManager.opa.(*FakeOpa) require.True(t, ok) @@ -69,7 +206,7 @@ func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { // TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. func TestCacheManager_AddObject_RemoveObject(t *testing.T) { - cm, ctx := makeUnitCacheManagerForTest(t) + cm, ctx := unitCacheManagerForTest(t) opaClient, ok := cm.opa.(*FakeOpa) require.True(t, ok) @@ -98,7 +235,7 @@ func TestCacheManager_AddObject_RemoveObject(t *testing.T) { // TestCacheManager_AddObject_processExclusion makes sure that we don't add objects that are process excluded. func TestCacheManager_AddObject_processExclusion(t *testing.T) { - cm, ctx := makeUnitCacheManagerForTest(t) + cm, ctx := unitCacheManagerForTest(t) processExcluder := process.Get() processExcluder.Add([]configv1alpha1.MatchEntry{ { @@ -127,7 +264,7 @@ func TestCacheManager_AddObject_processExclusion(t *testing.T) { // TestCacheManager_opaClient_errors tests that the cache manager responds to errors from the opa client. func TestCacheManager_opaClient_errors(t *testing.T) { - cm, ctx := makeUnitCacheManagerForTest(t) + cm, ctx := unitCacheManagerForTest(t) opaClient, ok := cm.opa.(*FakeOpa) require.True(t, ok) opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err @@ -147,7 +284,7 @@ func TestCacheManager_opaClient_errors(t *testing.T) { // TestCacheManager_AddSource tests that we can modify the gvk aggregator and watched set when adding a new source. func TestCacheManager_AddSource(t *testing.T) { - cacheManager, _, ctx := makeCacheManagerForTest(t, false, true) + cacheManager, ctx := unitCacheManagerForTest(t) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} nsGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} @@ -180,7 +317,7 @@ func TestCacheManager_AddSource(t *testing.T) { // TestCacheManager_RemoveSource tests that we can modify the gvk aggregator when removing a source. func TestCacheManager_RemoveSource(t *testing.T) { - cacheManager, _, ctx := makeCacheManagerForTest(t, false, true) + cacheManager, ctx := unitCacheManagerForTest(t) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} sourceA := aggregator.Key{Source: "a", ID: "source"} @@ -199,3 +336,38 @@ func TestCacheManager_RemoveSource(t *testing.T) { require.NoError(t, cacheManager.RemoveSource(ctx, sourceA)) require.False(t, cacheManager.gvksToSync.IsPresent(podGVK)) } + +func newSyncExcluderFor(nsToExclude string) *process.Excluder { + excluder := process.New() + excluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{wildcard.Wildcard(nsToExclude)}, + Processes: []string{"sync"}, + }, + // exclude kube-system by default to prevent noise + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + + return excluder +} + +func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetName(name) + u.SetNamespace("default") + if gvk.Kind == "Pod" { + u.Object["spec"] = map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "foo-container", + "image": "foo-image", + }, + }, + } + } + return u +} diff --git a/pkg/cachemanager/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go similarity index 68% rename from pkg/cachemanager/cachemanager_integration_test.go rename to pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index 908034885e9..d6240f9b155 100644 --- a/pkg/cachemanager/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -1,4 +1,4 @@ -package cachemanager +package cachemanager_test import ( "context" @@ -7,14 +7,24 @@ import ( "time" configv1alpha1 "github.com/open-policy-agent/gatekeeper/v3/apis/config/v1alpha1" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" + "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" + "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" + testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" + "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" ) const ( @@ -22,12 +32,19 @@ const ( eventuallyTicker = 2 * time.Second ) +var cfg *rest.Config + +func TestMain(m *testing.M) { + testutils.StartControlPlane(m, &cfg, 3) +} + // TestCacheManager_replay_retries tests that we can retry GVKs that error out in the reply goroutine. func TestCacheManager_replay_retries(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) failPlease := make(chan string, 2) - cacheManager.reader = fakes.HookReader{ + reader := fakes.HookReader{ Reader: c, ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // Return an error the first go-around. @@ -43,7 +60,9 @@ func TestCacheManager_replay_retries(t *testing.T) { }, } - opaClient, ok := cacheManager.opa.(*FakeOpa) + cacheManager, opa, _, ctx := cacheManagerForTest(t, mgr, wm, reader) + + opaClient, ok := opa.(*cachemanager.FakeOpa) require.True(t, ok) // seed one gvk @@ -58,7 +77,7 @@ func TestCacheManager_replay_retries(t *testing.T) { syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[OpaKey]interface{}{ + expected := map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } @@ -72,7 +91,7 @@ func TestCacheManager_replay_retries(t *testing.T) { // this call should make schedule a cache wipe and a replay for the configMapGVK require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected2 := map[OpaKey]interface{}{ + expected2 := map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } require.Eventually(t, expectedCheck(opaClient, expected2), eventuallyTimeout, eventuallyTicker) @@ -82,81 +101,12 @@ func TestCacheManager_replay_retries(t *testing.T) { require.NoError(t, c.Delete(ctx, pod), "creating ConfigMap pod-1") } -// TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the opa client. -func TestCacheManager_syncGVKInstances(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, false, false) - - configMapGVK := schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "ConfigMap", - } - // Create configMaps to test for - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - cm2 := unstructuredFor(configMapGVK, "config-test-2") - require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") - - cacheManager.watchedSet.Add(configMapGVK) - require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) - - opaClient, ok := cacheManager.opa.(*FakeOpa) - require.True(t, ok) - expected := map[OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - } - - require.Equal(t, 2, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) - - // wipe cache - require.NoError(t, cacheManager.wipeData(ctx)) - require.False(t, opaClient.Contains(expected)) - - // create a second GVK - podGVK := schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", - } - // Create pods to test for - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - - pod2 := unstructuredFor(podGVK, "pod-2") - require.NoError(t, c.Create(ctx, pod2), "creating Pod pod-2") - - pod3 := unstructuredFor(podGVK, "pod-3") - require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") - - cacheManager.watchedSet.Add(podGVK) - require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) - require.NoError(t, cacheManager.syncGVK(ctx, podGVK)) - - expected = map[OpaKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - {Gvk: podGVK, Key: "default/pod-2"}: nil, - {Gvk: podGVK, Key: "default/pod-3"}: nil, - } - - require.Equal(t, 5, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") - require.NoError(t, c.Delete(ctx, pod3), "deleting Pod pod-3") - require.NoError(t, c.Delete(ctx, pod2), "deleting Pod pod-2") -} - // TestCacheManager_AddSourceRemoveSource makes sure that we can add and remove multiple sources // and changes to the underlying cache are reflected. func TestCacheManager_AddSourceRemoveSource(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + cacheManager, opa, agg, ctx := cacheManagerForTest(t, mgr, wm, c) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} @@ -171,13 +121,13 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { pod := unstructuredFor(podGVK, "pod-1") require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - opaClient, ok := cacheManager.opa.(*FakeOpa) + opaClient, ok := opa.(*cachemanager.FakeOpa) require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[OpaKey]interface{}{ + expected := map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, @@ -186,8 +136,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected - cacheManager.gvksToSync.IsPresent(configMapGVK) - gvks := cacheManager.gvksToSync.List(syncSourceOne) + agg.IsPresent(configMapGVK) + gvks := agg.List(syncSourceOne) require.Len(t, gvks, 2) _, foundConfigMap := gvks[configMapGVK] require.True(t, foundConfigMap) @@ -197,14 +147,14 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now remove the podgvk and make sure we don't have pods in the cache anymore require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected = map[OpaKey]interface{}{ + expected = map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected - cacheManager.gvksToSync.IsPresent(configMapGVK) - gvks = cacheManager.gvksToSync.List(syncSourceOne) + agg.IsPresent(configMapGVK) + gvks = agg.List(syncSourceOne) require.Len(t, gvks, 1) _, foundConfigMap = gvks[configMapGVK] require.True(t, foundConfigMap) @@ -216,8 +166,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) reqConditionForAgg := func() bool { - cacheManager.gvksToSync.IsPresent(configMapGVK) - gvks := cacheManager.gvksToSync.List(syncSourceOne) + agg.IsPresent(configMapGVK) + gvks := agg.List(syncSourceOne) if len(gvks) != 1 { return false } @@ -226,7 +176,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { return false } - gvks2 := cacheManager.gvksToSync.List(syncSourceTwo) + gvks2 := agg.List(syncSourceTwo) if len(gvks2) != 1 { return false } @@ -236,7 +186,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, reqConditionForAgg, eventuallyTimeout, eventuallyTicker) require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - expected2 := map[OpaKey]interface{}{ + expected2 := map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, @@ -245,7 +195,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) - expected3 := map[OpaKey]interface{}{ + expected3 := map[cachemanager.OpaKey]interface{}{ // config maps no longer required by any sync source // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, @@ -258,8 +208,8 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(opaClient, map[OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) - require.True(t, len(cacheManager.gvksToSync.GVKs()) == 0) + require.Eventually(t, expectedCheck(opaClient, map[cachemanager.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.True(t, len(agg.GVKs()) == 0) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") @@ -270,16 +220,18 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // TestCacheManager_ExcludeProcesses makes sure that changing the process excluder // in the cache manager triggers a re-evaluation of GVKs. func TestCacheManager_ExcludeProcesses(t *testing.T) { - cacheManager, c, ctx := makeCacheManagerForTest(t, true, true) + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + cacheManager, opa, agg, ctx := cacheManagerForTest(t, mgr, wm, c) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} cm := unstructuredFor(configMapGVK, "config-test-1") require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - opaClient, ok := cacheManager.opa.(*FakeOpa) + opaClient, ok := opa.(*cachemanager.FakeOpa) require.True(t, ok) - expected := map[OpaKey]interface{}{ + expected := map[cachemanager.OpaKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } @@ -298,7 +250,6 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { }, }) cacheManager.ExcludeProcesses(sameExcluder) - require.True(t, cacheManager.processExcluder.Equals(sameExcluder)) // now process exclude the remaining gvk, it should get removed by the background process. excluder := process.New() @@ -315,16 +266,16 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { }) cacheManager.ExcludeProcesses(excluder) - require.Eventually(t, expectedCheck(opaClient, map[OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(opaClient, map[cachemanager.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) // make sure the gvk is still in gvkAggregator - require.True(t, len(cacheManager.gvksToSync.GVKs()) == 1) - require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) + require.True(t, len(agg.GVKs()) == 1) + require.True(t, agg.IsPresent(configMapGVK)) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") } -func expectedCheck(opaClient *FakeOpa, expected map[OpaKey]interface{}) func() bool { +func expectedCheck(opaClient *cachemanager.FakeOpa, expected map[cachemanager.OpaKey]interface{}) func() bool { return func() bool { if opaClient.Len() != len(expected) { return false @@ -354,3 +305,57 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns } return u } + +func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, reader client.Reader) (*cachemanager.CacheManager, cachemanager.OpaDataClient, *aggregator.GVKAgreggator, context.Context) { + ctx, cancelFunc := context.WithCancel(context.Background()) + + opaClient := &cachemanager.FakeOpa{} + tracker, err := readiness.SetupTracker(mgr, false, false, false) + require.NoError(t, err) + processExcluder := process.Get() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, + Processes: []string{"sync"}, + }, + }) + events := make(chan event.GenericEvent, 1024) + w, err := wm.NewRegistrar( + "test-cache-manager", + events) + require.NoError(t, err) + + aggregator := aggregator.NewGVKAggregator() + cfg := &cachemanager.Config{ + Opa: opaClient, + SyncMetricsCache: syncutil.NewMetricsCache(), + Tracker: tracker, + ProcessExcluder: processExcluder, + WatchedSet: watch.NewSet(), + Registrar: w, + Reader: reader, + GVKAggregator: aggregator, + } + cacheManager, err := cachemanager.NewCacheManager(cfg) + require.NoError(t, err) + + syncAdder := sync.Adder{ + Events: events, + CacheManager: cacheManager, + } + require.NoError(t, syncAdder.Add(mgr), "registering sync controller") + go func() { + require.NoError(t, cacheManager.Start(ctx)) + }() + + t.Cleanup(func() { + ctx.Done() + }) + + testutils.StartManager(ctx, t, mgr) + + t.Cleanup(func() { + cancelFunc() + }) + return cacheManager, opaClient, aggregator, ctx +} diff --git a/pkg/controller/sync/sync_controller.go b/pkg/controller/sync/sync_controller.go index 242e8217681..a74655c2135 100644 --- a/pkg/controller/sync/sync_controller.go +++ b/pkg/controller/sync/sync_controller.go @@ -20,6 +20,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/logging" "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" @@ -40,7 +41,7 @@ import ( var log = logf.Log.WithName("controller").WithValues("metaKind", "Sync") type Adder struct { - CacheManager syncutil.CacheManagerMediator + CacheManager *cachemanager.CacheManager Events <-chan event.GenericEvent } @@ -64,7 +65,7 @@ func (a *Adder) Add(mgr manager.Manager) error { func newReconciler( mgr manager.Manager, reporter syncutil.Reporter, - cm syncutil.CacheManagerMediator, + cm *cachemanager.CacheManager, ) reconcile.Reconciler { return &ReconcileSync{ reader: mgr.GetCache(), @@ -102,7 +103,7 @@ type ReconcileSync struct { scheme *runtime.Scheme log logr.Logger reporter syncutil.Reporter - cm syncutil.CacheManagerMediator + cm *cachemanager.CacheManager } // +kubebuilder:rbac:groups=constraints.gatekeeper.sh,resources=*,verbs=get;list;watch;create;update;patch;delete diff --git a/pkg/syncutil/opadataclient.go b/pkg/syncutil/opadataclient.go deleted file mode 100644 index d2762906494..00000000000 --- a/pkg/syncutil/opadataclient.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package syncutil - -import ( - "context" - - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -) - -// CacheManagerMediator interface to use the CacheManager and avoid import cycles. -type CacheManagerMediator interface { - AddObject(ctx context.Context, instance *unstructured.Unstructured) error - RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error - - ReportSyncMetrics() -} From f8684ddacb16ebb72731dddffc51d29a7c4fe525 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 27 Jul 2023 01:57:02 +0000 Subject: [PATCH 30/58] refactor: move parser package per feedback Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/{syncutil => cachemanager}/parser/syncannotationreader.go | 0 .../parser/syncannotationreader_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pkg/{syncutil => cachemanager}/parser/syncannotationreader.go (100%) rename pkg/{syncutil => cachemanager}/parser/syncannotationreader_test.go (100%) diff --git a/pkg/syncutil/parser/syncannotationreader.go b/pkg/cachemanager/parser/syncannotationreader.go similarity index 100% rename from pkg/syncutil/parser/syncannotationreader.go rename to pkg/cachemanager/parser/syncannotationreader.go diff --git a/pkg/syncutil/parser/syncannotationreader_test.go b/pkg/cachemanager/parser/syncannotationreader_test.go similarity index 100% rename from pkg/syncutil/parser/syncannotationreader_test.go rename to pkg/cachemanager/parser/syncannotationreader_test.go From 71d77d4a09f6ebff09a1c9ac96ab239a35927239 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 27 Jul 2023 20:37:14 +0000 Subject: [PATCH 31/58] review: use anon funcs, naming, comments Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 77 +++++++++++++++++--------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 70ba21dfe6e..3559fb1331a 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -59,8 +59,8 @@ type CacheManager struct { backgroundManagementTicker time.Ticker reader client.Reader - // stopChan is used to stop any list operations still in progress - stopChan chan bool + // relistStopChan is used to stop any list operations still in progress + relistStopChan chan bool } // OpaDataClient is an interface for caching data. @@ -101,7 +101,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { gvksToSync: config.GVKAggregator, backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), - stopChan: make(chan bool, 1), + relistStopChan: make(chan bool, 1), } return cm, nil @@ -224,17 +224,17 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns return err } + + c.syncMetricsCache.AddObject(syncKey, syncutil.Tags{ + Kind: instance.GetKind(), + Status: metrics.ActiveStatus, + }) + c.syncMetricsCache.AddKind(instance.GetKind()) } c.tracker.ForData(instance.GroupVersionKind()).Observe(instance) - c.syncMetricsCache.AddObject(syncKey, syncutil.Tags{ - Kind: instance.GetKind(), - Status: metrics.ActiveStatus, - }) - c.syncMetricsCache.AddKind(instance.GetKind()) - - return err + return nil } func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error { @@ -246,7 +246,7 @@ func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured. } } - // only delete from metrics map if the data removal was succcesful + // only delete from metrics map if the data removal was succesful c.syncMetricsCache.DeleteObject(syncutil.GetKeyForSyncMetrics(instance.GetNamespace(), instance.GetName())) c.tracker.ForData(instance.GroupVersionKind()).CancelExpect(instance) @@ -278,15 +278,15 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) }) var err error - c.mu.Lock() - if !c.watchedSet.Contains(gvk) { - // we are not actually watching this gvk anymore - // so don't list instances for it. - err = nil - } else { - err = c.reader.List(ctx, u) - } - c.mu.Unlock() + func() { + c.mu.Lock() + defer c.mu.Unlock() + + // only call List if we are still watching the gvk. + if c.watchedSet.Contains(gvk) { + err = c.reader.List(ctx, u) + } + }() if err != nil { return fmt.Errorf("replaying data for %+v: %w", gvk, err) @@ -305,28 +305,31 @@ func (c *CacheManager) manageCache(ctx context.Context) { for { select { case <-ctx.Done(): - close(c.stopChan) + close(c.relistStopChan) return case <-c.backgroundManagementTicker.C: - c.mu.Lock() - c.wipeCacheIfNeeded(ctx) + func() { + c.mu.Lock() + defer c.mu.Unlock() - // spin up new goroutines to relist gvks as there has been a wipe - if c.needToList { - // stop any goroutines that were relisting before - // as we may no longer be interested in those gvks - c.stopChan <- true + c.wipeCacheIfNeeded(ctx) - // assume all gvks need to be relisted - gvksToRelist := c.gvksToSync.GVKs() + // spin up new goroutines to relist gvks as there has been a wipe + if c.needToList { + // stop any goroutines that were relisting before + // as we may no longer be interested in those gvks + c.relistStopChan <- true - // clean state - c.needToList = false - c.stopChan = make(chan bool, 1) + // assume all gvks need to be relisted + gvksToRelist := c.gvksToSync.GVKs() - go c.replayGVKs(ctx, gvksToRelist) - } - c.mu.Unlock() + // clean state + c.needToList = false + c.relistStopChan = make(chan bool, 1) + + go c.replayGVKs(ctx, gvksToRelist) + } + }() } } } @@ -342,7 +345,7 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro select { case <-ctx.Done(): return - case <-c.stopChan: + case <-c.relistStopChan: return default: operation := func() (bool, error) { From cb24e095e35dedb2a8f8b6c06f9b3628c5d7eeef Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 27 Jul 2023 20:45:41 +0000 Subject: [PATCH 32/58] fix lint Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 3559fb1331a..22d8bb5e525 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -246,7 +246,7 @@ func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured. } } - // only delete from metrics map if the data removal was succesful + // only delete from metrics map if the data removal was successful c.syncMetricsCache.DeleteObject(syncutil.GetKeyForSyncMetrics(instance.GetNamespace(), instance.GetName())) c.tracker.ForData(instance.GroupVersionKind()).CancelExpect(instance) From e62bea1a1ad611beff803cc417abfcd14a24da39 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 27 Jul 2023 21:41:44 +0000 Subject: [PATCH 33/58] review: dont use a buffered, sentinel ch Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 22d8bb5e525..9a1d18d3672 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -60,7 +60,7 @@ type CacheManager struct { reader client.Reader // relistStopChan is used to stop any list operations still in progress - relistStopChan chan bool + relistStopChan chan struct{} } // OpaDataClient is an interface for caching data. @@ -101,7 +101,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { gvksToSync: config.GVKAggregator, backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), - relistStopChan: make(chan bool, 1), + relistStopChan: make(chan struct{}), } return cm, nil @@ -305,7 +305,6 @@ func (c *CacheManager) manageCache(ctx context.Context) { for { select { case <-ctx.Done(): - close(c.relistStopChan) return case <-c.backgroundManagementTicker.C: func() { @@ -318,14 +317,14 @@ func (c *CacheManager) manageCache(ctx context.Context) { if c.needToList { // stop any goroutines that were relisting before // as we may no longer be interested in those gvks - c.relistStopChan <- true + close(c.relistStopChan) // assume all gvks need to be relisted gvksToRelist := c.gvksToSync.GVKs() // clean state c.needToList = false - c.relistStopChan = make(chan bool, 1) + c.relistStopChan = make(chan struct{}) go c.replayGVKs(ctx, gvksToRelist) } @@ -345,8 +344,10 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro select { case <-ctx.Done(): return - case <-c.relistStopChan: - return + case _, ok := <-c.relistStopChan: + if !ok { + return + } default: operation := func() (bool, error) { if err := c.syncGVK(ctx, gvk); err != nil { From ba4dbda14fde1ddb0c04c7da3d03e742bf879ab0 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:35:40 +0000 Subject: [PATCH 34/58] review: comments, renaming, read locks Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 50 +++++------ pkg/cachemanager/cachemanager_test.go | 68 +++++++-------- .../cachemanager_integration_test.go | 87 ++++++++----------- ...keopadataclient.go => fakecfdataclient.go} | 36 ++++---- .../config/config_controller_test.go | 28 +++--- pkg/controller/controller.go | 2 +- 6 files changed, 127 insertions(+), 144 deletions(-) rename pkg/cachemanager/{fakeopadataclient.go => fakecfdataclient.go} (68%) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 9a1d18d3672..67dc1581c4b 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -32,7 +32,7 @@ var ( ) type Config struct { - Opa OpaDataClient + CfClient CFDataClient SyncMetricsCache *syncutil.MetricsCache Tracker *readiness.Tracker ProcessExcluder *process.Excluder @@ -52,7 +52,7 @@ type CacheManager struct { // mu guards access to any of the fields above mu sync.RWMutex - opa OpaDataClient + cfClient CFDataClient syncMetricsCache *syncutil.MetricsCache tracker *readiness.Tracker registrar *watch.Registrar @@ -63,8 +63,8 @@ type CacheManager struct { relistStopChan chan struct{} } -// OpaDataClient is an interface for caching data. -type OpaDataClient interface { +// CFDataClient is an interface for caching data. +type CFDataClient interface { AddData(ctx context.Context, data interface{}) (*types.Responses, error) RemoveData(ctx context.Context, data interface{}) (*types.Responses, error) } @@ -91,7 +91,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { } cm := &CacheManager{ - opa: config.Opa, + cfClient: config.CfClient, syncMetricsCache: config.SyncMetricsCache, tracker: config.Tracker, processExcluder: config.ProcessExcluder, @@ -116,8 +116,7 @@ func (c *CacheManager) Start(ctx context.Context) error { // AddSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. -// It errors out if there is an issue adding the Key internally or replacing the watches. -// Consumers are encouraged to retry on error. +// Callers are responsible for retrying on error. func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() @@ -129,7 +128,6 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, // may become unreferenced and need to be deleted; this will be handled async // in the manageCache loop. - // make changes to the watches if err := c.replaceWatchSet(ctx); err != nil { return fmt.Errorf("error watching new gvks: %w", err) } @@ -137,14 +135,13 @@ func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, return nil } -// replaceWatchSet looks at the specifiedGVKs and makes changes to the registrar's watch set. +// replaceWatchSet looks at the gvksToSync and makes changes to the registrar's watch set. // assumes caller has lock. -// replaceWatchSet may error out and that error is retryable. +// On error, actual watch state may not align with intended watch state. func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() newWatchSet.Add(c.gvksToSync.GVKs()...) - // record any gvks that need to be deleted from the opa cache. c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) var innerError error @@ -160,8 +157,7 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { } // RemoveSource removes the watches of the GVKs for a given aggregator.Key. -// It errors out if there is an issue removing the Key internally or replacing the watches. -// Consumers are encouraged to retry on error. +// Callers are responsible for retrying on error. func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() @@ -170,7 +166,6 @@ func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Ke return fmt.Errorf("internal error removing source: %w", err) } - // make changes to the watches if err := c.replaceWatchSet(ctx); err != nil { return fmt.Errorf("error removing watches for source %v: %w", sourceKey, err) } @@ -195,6 +190,13 @@ func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { c.excluderChanged = true } +func (c *CacheManager) watchesGVK(gvk schema.GroupVersionKind) bool { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.watchedSet.Contains(gvk) +} + func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Unstructured) error { gvk := instance.GroupVersionKind() @@ -203,16 +205,14 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns return fmt.Errorf("error while excluding namespaces for gvk: %+v: %w", gvk, err) } - // bail because it means we should not be - // syncing this gvk's objects as it is namespace excluded. if isNamespaceExcluded { c.tracker.ForData(instance.GroupVersionKind()).CancelExpect(instance) return nil } syncKey := syncutil.GetKeyForSyncMetrics(instance.GetNamespace(), instance.GetName()) - if c.watchedSet.Contains(gvk) { - _, err = c.opa.AddData(ctx, instance) + if c.watchesGVK(gvk) { + _, err = c.cfClient.AddData(ctx, instance) if err != nil { c.syncMetricsCache.AddObject( syncKey, @@ -240,8 +240,8 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error { gvk := instance.GroupVersionKind() - if c.watchedSet.Contains(gvk) { - if _, err := c.opa.RemoveData(ctx, instance); err != nil { + if c.watchesGVK(gvk) { + if _, err := c.cfClient.RemoveData(ctx, instance); err != nil { return err } } @@ -254,7 +254,7 @@ func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured. } func (c *CacheManager) wipeData(ctx context.Context) error { - if _, err := c.opa.RemoveData(ctx, target.WipeData()); err != nil { + if _, err := c.cfClient.RemoveData(ctx, target.WipeData()); err != nil { return err } @@ -344,10 +344,8 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro select { case <-ctx.Done(): return - case _, ok := <-c.relistStopChan: - if !ok { - return - } + case <-c.relistStopChan: + return default: operation := func() (bool, error) { if err := c.syncGVK(ctx, gvk); err != nil { @@ -370,7 +368,7 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro // wipeCacheIfNeeded performs a cache wipe if there are any gvks needing to be removed // from the cache or if the excluder has changed. It also marks which gvks need to be -// re listed again in the opa cache after the wipe. +// re listed again in the cf data cache after the wipe. // assumes the caller has lock. func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { // remove any gvks not needing to be synced anymore diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 8a9b2bf63fc..2fce80c50ea 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -40,7 +40,7 @@ func makeCacheManager(t *testing.T) (*CacheManager, client.Client, context.Conte ctx, cancelFunc := context.WithCancel(context.Background()) - opaClient := &FakeOpa{} + cfClient := &FakeCfClient{} tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) processExcluder := process.Get() @@ -57,7 +57,7 @@ func makeCacheManager(t *testing.T) (*CacheManager, client.Client, context.Conte require.NoError(t, err) cacheManager, err := NewCacheManager(&Config{ - Opa: opaClient, + CfClient: cfClient, SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, ProcessExcluder: processExcluder, @@ -84,14 +84,14 @@ func makeCacheManager(t *testing.T) (*CacheManager, client.Client, context.Conte // TestCacheManager_wipeCacheIfNeeded. func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { cacheManager, ctx := unitCacheManagerForTest(t) - opaClient, ok := cacheManager.opa.(*FakeOpa) + cfClient, ok := cacheManager.cfClient.(*FakeCfClient) require.True(t, ok) // seed one gvk configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} cm := unstructuredFor(configMapGVK, "config-test-1") - _, err := opaClient.AddData(ctx, cm) - require.NoError(t, err, "adding ConfigMap config-test-1 in opa") + _, err := cfClient.AddData(ctx, cm) + require.NoError(t, err, "adding ConfigMap config-test-1 in cfClient") podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} require.NoError(t, cacheManager.gvksToSync.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) @@ -99,11 +99,11 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { cacheManager.gvksToDeleteFromCache.Add(configMapGVK) cacheManager.wipeCacheIfNeeded(ctx) - require.False(t, opaClient.HasGVK(configMapGVK)) + require.False(t, cfClient.HasGVK(configMapGVK)) require.ElementsMatch(t, cacheManager.gvksToSync.GVKs(), []schema.GroupVersionKind{podGVK}) } -// TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the opa client. +// TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the cfClient client. func TestCacheManager_syncGVKInstances(t *testing.T) { cacheManager, c, ctx := makeCacheManager(t) @@ -121,19 +121,19 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { cacheManager.watchedSet.Add(configMapGVK) require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) - opaClient, ok := cacheManager.opa.(*FakeOpa) + cfClient, ok := cacheManager.cfClient.(*FakeCfClient) require.True(t, ok) - expected := map[OpaKey]interface{}{ + expected := map[CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } - require.Equal(t, 2, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) + require.Equal(t, 2, cfClient.Len()) + require.True(t, cfClient.Contains(expected)) // wipe cache require.NoError(t, cacheManager.wipeData(ctx)) - require.False(t, opaClient.Contains(expected)) + require.False(t, cfClient.Contains(expected)) // create a second GVK podGVK := schema.GroupVersionKind{ @@ -155,7 +155,7 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) require.NoError(t, cacheManager.syncGVK(ctx, podGVK)) - expected = map[OpaKey]interface{}{ + expected = map[CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, @@ -163,8 +163,8 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { {Gvk: podGVK, Key: "default/pod-3"}: nil, } - require.Equal(t, 5, opaClient.Len()) - require.True(t, opaClient.Contains(expected)) + require.Equal(t, 5, cfClient.Len()) + require.True(t, cfClient.Contains(expected)) // cleanup require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") @@ -177,30 +177,30 @@ func TestCacheManager_syncGVKInstances(t *testing.T) { // TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { cacheManager, ctx := unitCacheManagerForTest(t) - opaClient, ok := cacheManager.opa.(*FakeOpa) + cfClient, ok := cacheManager.cfClient.(*FakeCfClient) require.True(t, ok) // seed gvks configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} cm := unstructuredFor(configMapGVK, "config-test-1") cm.SetNamespace("excluded-ns") - _, err := opaClient.AddData(ctx, cm) - require.NoError(t, err, "adding ConfigMap config-test-1 in opa") + _, err := cfClient.AddData(ctx, cm) + require.NoError(t, err, "adding ConfigMap config-test-1 in cfClient") cacheManager.watchedSet.Add(configMapGVK) podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} pod := unstructuredFor(configMapGVK, "pod-test-1") pod.SetNamespace("excluded-ns") - _, err = opaClient.AddData(ctx, pod) - require.NoError(t, err, "adding Pod pod-test-1 in opa") + _, err = cfClient.AddData(ctx, pod) + require.NoError(t, err, "adding Pod pod-test-1 in cfClient") cacheManager.watchedSet.Add(podGVK) cacheManager.ExcludeProcesses(newSyncExcluderFor("excluded-ns")) cacheManager.wipeCacheIfNeeded(ctx) // the cache manager should not be watching any of the gvks that are now excluded - require.False(t, opaClient.HasGVK(configMapGVK)) - require.False(t, opaClient.HasGVK(podGVK)) + require.False(t, cfClient.HasGVK(configMapGVK)) + require.False(t, cfClient.HasGVK(podGVK)) require.False(t, cacheManager.excluderChanged) } @@ -208,7 +208,7 @@ func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { func TestCacheManager_AddObject_RemoveObject(t *testing.T) { cm, ctx := unitCacheManagerForTest(t) - opaClient, ok := cm.opa.(*FakeOpa) + cfClient, ok := cm.cfClient.(*FakeCfClient) require.True(t, ok) pod := fakes.Pod( @@ -222,15 +222,15 @@ func TestCacheManager_AddObject_RemoveObject(t *testing.T) { cm.watchedSet.Add(pod.GroupVersionKind()) require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.True(t, opaClient.HasGVK(pod.GroupVersionKind())) + require.True(t, cfClient.HasGVK(pod.GroupVersionKind())) // now remove the object and verify it's removed require.NoError(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) + require.False(t, cfClient.HasGVK(pod.GroupVersionKind())) cm.watchedSet.Remove(pod.GroupVersionKind()) require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) // we drop calls for gvks that are not watched + require.False(t, cfClient.HasGVK(pod.GroupVersionKind())) // we drop calls for gvks that are not watched } // TestCacheManager_AddObject_processExclusion makes sure that we don't add objects that are process excluded. @@ -255,19 +255,19 @@ func TestCacheManager_AddObject_processExclusion(t *testing.T) { require.NoError(t, err) require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - // test that pod from excluded namespace is not cache managed - opaClient, ok := cm.opa.(*FakeOpa) + // test that pod from excluded namespace is not added to the cache + cfClient, ok := cm.cfClient.(*FakeCfClient) require.True(t, ok) - require.False(t, opaClient.HasGVK(pod.GroupVersionKind())) - require.False(t, opaClient.Contains(map[OpaKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) + require.False(t, cfClient.HasGVK(pod.GroupVersionKind())) + require.False(t, cfClient.Contains(map[CfDataKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) } -// TestCacheManager_opaClient_errors tests that the cache manager responds to errors from the opa client. -func TestCacheManager_opaClient_errors(t *testing.T) { +// TestCacheManager_cfClient_errors tests that the cache manager responds to errors from the cfClient client. +func TestCacheManager_cfClient_errors(t *testing.T) { cm, ctx := unitCacheManagerForTest(t) - opaClient, ok := cm.opa.(*FakeOpa) + cfClient, ok := cm.cfClient.(*FakeCfClient) require.True(t, ok) - opaClient.SetErroring(true) // This will cause AddObject, RemoveObject to err + cfClient.SetErroring(true) // This will cause AddObject, RemoveObject to err pod := fakes.Pod( fakes.WithNamespace("test-ns"), diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index d6240f9b155..a079b8d23df 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -60,9 +60,9 @@ func TestCacheManager_replay_retries(t *testing.T) { }, } - cacheManager, opa, _, ctx := cacheManagerForTest(t, mgr, wm, reader) + cacheManager, dataStore, _, ctx := cacheManagerForTest(t, mgr, wm, reader) - opaClient, ok := opa.(*cachemanager.FakeOpa) + cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) // seed one gvk @@ -77,24 +77,24 @@ func TestCacheManager_replay_retries(t *testing.T) { syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[cachemanager.OpaKey]interface{}{ + expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) // set up a scenario where the list from replay will fail a few times failPlease <- "ConfigMapList" failPlease <- "ConfigMapList" - // this call should make schedule a cache wipe and a replay for the configMapGVK + // this call should schedule a cache wipe and a replay for the configMapGVK require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected2 := map[cachemanager.OpaKey]interface{}{ + expected2 := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected2), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) // cleanup require.NoError(t, c.Delete(ctx, cm), "creating ConfigMap config-test-1") @@ -106,7 +106,7 @@ func TestCacheManager_replay_retries(t *testing.T) { func TestCacheManager_AddSourceRemoveSource(t *testing.T) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - cacheManager, opa, agg, ctx := cacheManagerForTest(t, mgr, wm, c) + cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} @@ -121,19 +121,19 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { pod := unstructuredFor(podGVK, "pod-1") require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - opaClient, ok := opa.(*cachemanager.FakeOpa) + cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[cachemanager.OpaKey]interface{}{ + expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected agg.IsPresent(configMapGVK) @@ -147,11 +147,11 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now remove the podgvk and make sure we don't have pods in the cache anymore require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected = map[cachemanager.OpaKey]interface{}{ + expected = map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected agg.IsPresent(configMapGVK) gvks = agg.List(syncSourceOne) @@ -164,51 +164,36 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now make sure that adding another sync source with the same gvk has no side effects syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - - reqConditionForAgg := func() bool { - agg.IsPresent(configMapGVK) - gvks := agg.List(syncSourceOne) - if len(gvks) != 1 { - return false - } - _, found := gvks[configMapGVK] - if !found { - return false - } - - gvks2 := agg.List(syncSourceTwo) - if len(gvks2) != 1 { - return false - } - _, found2 := gvks2[configMapGVK] - return found2 - } - require.Eventually(t, reqConditionForAgg, eventuallyTimeout, eventuallyTicker) + agg.IsPresent(configMapGVK) + gvks = agg.List(syncSourceTwo) + require.Len(t, gvks, 1) + _, foundConfigMap = gvks[configMapGVK] + require.True(t, foundConfigMap) require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - expected2 := map[cachemanager.OpaKey]interface{}{ + expected2 := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected2), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) - expected3 := map[cachemanager.OpaKey]interface{}{ + expected3 := map[cachemanager.CfDataKey]interface{}{ // config maps no longer required by any sync source // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, {Gvk: podGVK, Key: "default/pod-1"}: nil, } - require.Eventually(t, expectedCheck(opaClient, expected3), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, expected3), eventuallyTimeout, eventuallyTicker) // now remove all the sources require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(opaClient, map[cachemanager.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) require.True(t, len(agg.GVKs()) == 0) // cleanup @@ -222,23 +207,23 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { func TestCacheManager_ExcludeProcesses(t *testing.T) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - cacheManager, opa, agg, ctx := cacheManagerForTest(t, mgr, wm, c) + cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} cm := unstructuredFor(configMapGVK, "config-test-1") require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - opaClient, ok := opa.(*cachemanager.FakeOpa) + cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) - expected := map[cachemanager.OpaKey]interface{}{ + expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) - // check that everything is well added at first - require.Eventually(t, expectedCheck(opaClient, expected), eventuallyTimeout, eventuallyTicker) + // check that everything is correctly added at first + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) // make sure that replacing w same process excluder is a no op sameExcluder := process.New() @@ -266,7 +251,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { }) cacheManager.ExcludeProcesses(excluder) - require.Eventually(t, expectedCheck(opaClient, map[cachemanager.OpaKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) // make sure the gvk is still in gvkAggregator require.True(t, len(agg.GVKs()) == 1) require.True(t, agg.IsPresent(configMapGVK)) @@ -275,12 +260,12 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") } -func expectedCheck(opaClient *cachemanager.FakeOpa, expected map[cachemanager.OpaKey]interface{}) func() bool { +func expectedCheck(cfClient *cachemanager.FakeCfClient, expected map[cachemanager.CfDataKey]interface{}) func() bool { return func() bool { - if opaClient.Len() != len(expected) { + if cfClient.Len() != len(expected) { return false } - if opaClient.Contains(expected) { + if cfClient.Contains(expected) { return true } @@ -306,10 +291,10 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns return u } -func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, reader client.Reader) (*cachemanager.CacheManager, cachemanager.OpaDataClient, *aggregator.GVKAgreggator, context.Context) { +func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, reader client.Reader) (*cachemanager.CacheManager, cachemanager.CFDataClient, *aggregator.GVKAgreggator, context.Context) { ctx, cancelFunc := context.WithCancel(context.Background()) - opaClient := &cachemanager.FakeOpa{} + cfClient := &cachemanager.FakeCfClient{} tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) processExcluder := process.Get() @@ -327,7 +312,7 @@ func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, r aggregator := aggregator.NewGVKAggregator() cfg := &cachemanager.Config{ - Opa: opaClient, + CfClient: cfClient, SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, ProcessExcluder: processExcluder, @@ -357,5 +342,5 @@ func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, r t.Cleanup(func() { cancelFunc() }) - return cacheManager, opaClient, aggregator, ctx + return cacheManager, cfClient, aggregator, ctx } diff --git a/pkg/cachemanager/fakeopadataclient.go b/pkg/cachemanager/fakecfdataclient.go similarity index 68% rename from pkg/cachemanager/fakeopadataclient.go rename to pkg/cachemanager/fakecfdataclient.go index 7d531218645..e903deea117 100644 --- a/pkg/cachemanager/fakeopadataclient.go +++ b/pkg/cachemanager/fakecfdataclient.go @@ -24,37 +24,37 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type OpaKey struct { +type CfDataKey struct { Gvk schema.GroupVersionKind Key string } -// FakeOpa is an OpaDataClient for testing. -type FakeOpa struct { +// FakeCfClient is an CfDataClient for testing. +type FakeCfClient struct { mu gosync.Mutex - data map[OpaKey]interface{} + data map[CfDataKey]interface{} needsToError bool } -var _ OpaDataClient = &FakeOpa{} +var _ CFDataClient = &FakeCfClient{} -// keyFor returns an opaKey for the provided resource. +// keyFor returns a cfDataKey for the provided resource. // Returns error if the resource is not a runtime.Object w/ metadata. -func (f *FakeOpa) keyFor(obj interface{}) (OpaKey, error) { +func (f *FakeCfClient) keyFor(obj interface{}) (CfDataKey, error) { o, ok := obj.(client.Object) if !ok { - return OpaKey{}, fmt.Errorf("expected runtime.Object, got: %T", obj) + return CfDataKey{}, fmt.Errorf("expected runtime.Object, got: %T", obj) } gvk := o.GetObjectKind().GroupVersionKind() ns := o.GetNamespace() if ns == "" { - return OpaKey{Gvk: gvk, Key: o.GetName()}, nil + return CfDataKey{Gvk: gvk, Key: o.GetName()}, nil } - return OpaKey{Gvk: gvk, Key: fmt.Sprintf("%s/%s", ns, o.GetName())}, nil + return CfDataKey{Gvk: gvk, Key: fmt.Sprintf("%s/%s", ns, o.GetName())}, nil } -func (f *FakeOpa) AddData(ctx context.Context, data interface{}) (*constraintTypes.Responses, error) { +func (f *FakeCfClient) AddData(ctx context.Context, data interface{}) (*constraintTypes.Responses, error) { f.mu.Lock() defer f.mu.Unlock() @@ -68,14 +68,14 @@ func (f *FakeOpa) AddData(ctx context.Context, data interface{}) (*constraintTyp } if f.data == nil { - f.data = make(map[OpaKey]interface{}) + f.data = make(map[CfDataKey]interface{}) } f.data[key] = data return &constraintTypes.Responses{}, nil } -func (f *FakeOpa) RemoveData(ctx context.Context, data interface{}) (*constraintTypes.Responses, error) { +func (f *FakeCfClient) RemoveData(ctx context.Context, data interface{}) (*constraintTypes.Responses, error) { f.mu.Lock() defer f.mu.Unlock() @@ -84,7 +84,7 @@ func (f *FakeOpa) RemoveData(ctx context.Context, data interface{}) (*constraint } if target.IsWipeData(data) { - f.data = make(map[OpaKey]interface{}) + f.data = make(map[CfDataKey]interface{}) return &constraintTypes.Responses{}, nil } @@ -98,7 +98,7 @@ func (f *FakeOpa) RemoveData(ctx context.Context, data interface{}) (*constraint } // Contains returns true if all expected resources are in the cache. -func (f *FakeOpa) Contains(expected map[OpaKey]interface{}) bool { +func (f *FakeCfClient) Contains(expected map[CfDataKey]interface{}) bool { f.mu.Lock() defer f.mu.Unlock() @@ -111,7 +111,7 @@ func (f *FakeOpa) Contains(expected map[OpaKey]interface{}) bool { } // HasGVK returns true if the cache has any data of the requested kind. -func (f *FakeOpa) HasGVK(gvk schema.GroupVersionKind) bool { +func (f *FakeCfClient) HasGVK(gvk schema.GroupVersionKind) bool { f.mu.Lock() defer f.mu.Unlock() @@ -124,14 +124,14 @@ func (f *FakeOpa) HasGVK(gvk schema.GroupVersionKind) bool { } // Len returns the number of items in the cache. -func (f *FakeOpa) Len() int { +func (f *FakeCfClient) Len() int { f.mu.Lock() defer f.mu.Unlock() return len(f.data) } // SetErroring will error out on AddObject or RemoveObject. -func (f *FakeOpa) SetErroring(enabled bool) { +func (f *FakeCfClient) SetErroring(enabled bool) { f.mu.Lock() defer f.mu.Unlock() f.needsToError = enabled diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 24679c2e826..2067c9aed52 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -126,7 +126,7 @@ func TestReconcile(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &cachemanager.FakeOpa{} + opaClient := &cachemanager.FakeCfClient{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) @@ -143,7 +143,7 @@ func TestReconcile(t *testing.T) { events) require.NoError(t, err) cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ - Opa: opaClient, + CfClient: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, @@ -284,7 +284,7 @@ func TestReconcile(t *testing.T) { } require.NoError(t, c.Create(ctx, fooPod)) // fooPod should be namespace excluded, hence not synced - g.Eventually(opaClient.Contains(map[cachemanager.OpaKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}}), timeout).ShouldNot(gomega.BeTrue()) + g.Eventually(opaClient.Contains(map[cachemanager.CfDataKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}}), timeout).ShouldNot(gomega.BeTrue()) require.NoError(t, c.Delete(ctx, fooPod)) testMgrStopped() @@ -407,11 +407,11 @@ func TestConfig_DeleteSyncResources(t *testing.T) { }, timeout).Should(gomega.BeTrue()) } -func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager, tracker *readiness.Tracker, events chan event.GenericEvent, reader client.Reader, useFakeOpa bool) (cachemanager.OpaDataClient, error) { +func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager, tracker *readiness.Tracker, events chan event.GenericEvent, reader client.Reader, useFakeOpa bool) (cachemanager.CFDataClient, error) { // initialize OPA - var opaClient cachemanager.OpaDataClient + var opaClient cachemanager.CFDataClient if useFakeOpa { - opaClient = &cachemanager.FakeOpa{} + opaClient = &cachemanager.FakeCfClient{} } else { driver, err := rego.New(rego.Tracing(true)) if err != nil { @@ -437,7 +437,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager return nil, fmt.Errorf("cannot create registrar: %w", err) } cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ - Opa: opaClient, + CfClient: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, @@ -505,7 +505,7 @@ func TestConfig_CacheContents(t *testing.T) { opa, err := setupController(ctx, mgr, wm, tracker, events, c, true) require.NoError(t, err, "failed to set up controller") - opaClient, ok := opa.(*cachemanager.FakeOpa) + opaClient, ok := opa.(*cachemanager.FakeCfClient) require.True(t, ok) testutils.StartManager(ctx, t, mgr) @@ -521,7 +521,7 @@ func TestConfig_CacheContents(t *testing.T) { config := configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) require.NoError(t, c.Create(ctx, config), "creating Config config") - expected := map[cachemanager.OpaKey]interface{}{ + expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: nsGVK, Key: "default"}: nil, {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, // kube-system namespace is being excluded, it should not be in opa cache @@ -547,7 +547,7 @@ func TestConfig_CacheContents(t *testing.T) { // Expect our configMap to return at some point // TODO: In the future it will remain instead of having to repopulate. - expected = map[cachemanager.OpaKey]interface{}{ + expected = map[cachemanager.CfDataKey]interface{}{ { Gvk: configMapGVK, Key: "default/config-test-1", @@ -557,7 +557,7 @@ func TestConfig_CacheContents(t *testing.T) { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "waiting for ConfigMap to repopulate in cache") - expected = map[cachemanager.OpaKey]interface{}{ + expected = map[cachemanager.CfDataKey]interface{}{ { Gvk: configMapGVK, Key: "kube-system/config-test-2", @@ -602,7 +602,7 @@ func TestConfig_Retries(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &cachemanager.FakeOpa{} + opaClient := &cachemanager.FakeCfClient{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { @@ -619,7 +619,7 @@ func TestConfig_Retries(t *testing.T) { events) require.NoError(t, err) cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ - Opa: opaClient, + CfClient: opaClient, SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, @@ -699,7 +699,7 @@ func TestConfig_Retries(t *testing.T) { } }() - expected := map[cachemanager.OpaKey]interface{}{ + expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, } g.Eventually(func() bool { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2b84d00f7de..889d97015e5 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -182,7 +182,7 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { return err } cm, err := cm.NewCacheManager(&cm.Config{ - Opa: deps.Opa, + CfClient: deps.Opa, SyncMetricsCache: syncMetricsCache, Tracker: deps.Tracker, ProcessExcluder: deps.ProcessExcluder, From b78d3928a5df7540dad3f549439beced4215db70 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:43:17 +0000 Subject: [PATCH 35/58] fix: return cpy Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/aggregator/aggregator.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index 6127b505bce..e84f2fc2c8d 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -103,7 +103,12 @@ func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { b.mu.Lock() defer b.mu.Unlock() - return b.store[k] + v := b.store[k] + cpy := make(map[schema.GroupVersionKind]struct{}, len(v)) + for key, value := range v { + cpy[key] = value + } + return cpy } // GVKs returns a list of all of the schema.GroupVersionKind that are aggregated. From a5ffd53610ee8507ae2c8c3d3e0ccf37a3e10a61 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 31 Jul 2023 18:59:01 +0000 Subject: [PATCH 36/58] review: UpsertSource, short circuits Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 59 ++++++++++++------- pkg/cachemanager/cachemanager_test.go | 8 +-- .../cachemanager_integration_test.go | 16 ++--- pkg/controller/config/config_controller.go | 10 +--- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 67dc1581c4b..c9ef6adb894 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -114,16 +114,23 @@ func (c *CacheManager) Start(ctx context.Context) error { return nil } -// AddSource adjusts the watched set of gvks according to the newGVKs passed in +// UpsertSource adjusts the watched set of gvks according to the newGVKs passed in // for a given sourceKey. // Callers are responsible for retrying on error. -func (c *CacheManager) AddSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { +func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() - if err := c.gvksToSync.Upsert(sourceKey, newGVKs); err != nil { - return fmt.Errorf("internal error adding source: %w", err) + if len(newGVKs) > 0 { + if err := c.gvksToSync.Upsert(sourceKey, newGVKs); err != nil { + return fmt.Errorf("internal error adding source: %w", err) + } + } else { + if err := c.gvksToSync.Remove(sourceKey); err != nil { + return fmt.Errorf("internal error removing source: %w", err) + } } + // as a result of upserting the new gvks for the source key, some gvks // may become unreferenced and need to be deleted; this will be handled async // in the manageCache loop. @@ -289,7 +296,7 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) }() if err != nil { - return fmt.Errorf("replaying data for %+v: %w", gvk, err) + return fmt.Errorf("listing data for %+v: %w", gvk, err) } for i := range u.Items { @@ -313,21 +320,30 @@ func (c *CacheManager) manageCache(ctx context.Context) { c.wipeCacheIfNeeded(ctx) - // spin up new goroutines to relist gvks as there has been a wipe - if c.needToList { - // stop any goroutines that were relisting before - // as we may no longer be interested in those gvks - close(c.relistStopChan) + if !c.needToList { + // this means that there are no changes needed + // such that any gvks need to be relisted. + // any in flight goroutines can finish relisiting. + return + } - // assume all gvks need to be relisted - gvksToRelist := c.gvksToSync.GVKs() + // otherwise, spin up new goroutines to relist gvks as there has been a wipe - // clean state - c.needToList = false - c.relistStopChan = make(chan struct{}) + // stop any goroutines that were relisting before + // as we may no longer be interested in those gvks + close(c.relistStopChan) - go c.replayGVKs(ctx, gvksToRelist) - } + // assume all gvks need to be relisted + // and while under lock, make a copy of + // all gvks so we can pass it in the goroutine + // without needing to read lock this data + gvksToRelist := c.gvksToSync.GVKs() + + // clean state + c.needToList = false + c.relistStopChan = make(chan struct{}) + + go c.replayGVKs(ctx, gvksToRelist) }() } } @@ -376,10 +392,11 @@ func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { if c.gvksToDeleteFromCache.Size() > 0 || c.excluderChanged { if err := c.wipeData(ctx); err != nil { log.Error(err, "internal: error wiping cache") - } else { - c.gvksToDeleteFromCache = watch.NewSet() - c.excluderChanged = false - c.needToList = true + return } + + c.gvksToDeleteFromCache = watch.NewSet() + c.excluderChanged = false + c.needToList = true } } diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 2fce80c50ea..b75ccd483f0 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -292,8 +292,8 @@ func TestCacheManager_AddSource(t *testing.T) { sourceB := aggregator.Key{Source: "b", ID: "source"} // given two sources with overlapping gvks ... - require.NoError(t, cacheManager.AddSource(ctx, sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) // ... expect the aggregator to dedup require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) @@ -301,13 +301,13 @@ func TestCacheManager_AddSource(t *testing.T) { require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK, configMapGVK}) // adding a source without a previously added gvk ... - require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, sourceB, []schema.GroupVersionKind{configMapGVK})) // ... should not remove any gvks that are still referenced by other sources require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) // adding a source that modifies the only reference to a gvk ... - require.NoError(t, cacheManager.AddSource(ctx, sourceB, []schema.GroupVersionKind{nsGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, sourceB, []schema.GroupVersionKind{nsGVK})) // ... will effectively remove the gvk from the aggregator require.False(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index a079b8d23df..d050f4f12a1 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -75,7 +75,7 @@ func TestCacheManager_replay_retries(t *testing.T) { require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -89,7 +89,7 @@ func TestCacheManager_replay_retries(t *testing.T) { failPlease <- "ConfigMapList" // this call should schedule a cache wipe and a replay for the configMapGVK - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected2 := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -125,7 +125,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -145,7 +145,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.True(t, foundPod) // now remove the podgvk and make sure we don't have pods in the cache anymore - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected = map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -163,14 +163,14 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { // now make sure that adding another sync source with the same gvk has no side effects syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) agg.IsPresent(configMapGVK) gvks = agg.List(syncSourceTwo) require.Len(t, gvks, 1) _, foundConfigMap = gvks[configMapGVK] require.True(t, foundConfigMap) - require.NoError(t, cacheManager.AddSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) expected2 := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, @@ -179,7 +179,7 @@ func TestCacheManager_AddSourceRemoveSource(t *testing.T) { require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed - require.NoError(t, cacheManager.AddSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) expected3 := map[cachemanager.CfDataKey]interface{}{ // config maps no longer required by any sync source // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, @@ -221,7 +221,7 @@ func TestCacheManager_ExcludeProcesses(t *testing.T) { } syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.AddSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) // check that everything is correctly added at first require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index e10f2277250..f8145376875 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -228,14 +228,8 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque r.cacheManager.ExcludeProcesses(newExcluder) configSourceKey := aggregator.Key{Source: "config", ID: request.NamespacedName.String()} - if len(gvksToSync) > 0 { - if err := r.cacheManager.AddSource(ctx, configSourceKey, gvksToSync); err != nil { - return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) - } - } else { - if err := r.cacheManager.RemoveSource(ctx, configSourceKey); err != nil { - return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error removing syncOny gvks from sync process: %w", err) - } + if err := r.cacheManager.UpsertSource(ctx, configSourceKey, gvksToSync); err != nil { + return reconcile.Result{Requeue: true}, fmt.Errorf("config-controller: error establishing watches for new syncOny: %w", err) } return reconcile.Result{}, nil From 848bc87d26812d1dc8bfa6dc29a1cea2cf113c40 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 31 Jul 2023 20:50:21 +0000 Subject: [PATCH 37/58] review: pass in stop ch Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index c9ef6adb894..25c291912c0 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -58,9 +58,6 @@ type CacheManager struct { registrar *watch.Registrar backgroundManagementTicker time.Ticker reader client.Reader - - // relistStopChan is used to stop any list operations still in progress - relistStopChan chan struct{} } // CFDataClient is an interface for caching data. @@ -101,7 +98,6 @@ func NewCacheManager(config *Config) (*CacheManager, error) { gvksToSync: config.GVKAggregator, backgroundManagementTicker: *time.NewTicker(3 * time.Second), gvksToDeleteFromCache: watch.NewSet(), - relistStopChan: make(chan struct{}), } return cm, nil @@ -309,6 +305,9 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) } func (c *CacheManager) manageCache(ctx context.Context) { + // relistStopChan is used to stop any list operations still in progress + relistStopChan := make(chan struct{}) + for { select { case <-ctx.Done(): @@ -331,7 +330,7 @@ func (c *CacheManager) manageCache(ctx context.Context) { // stop any goroutines that were relisting before // as we may no longer be interested in those gvks - close(c.relistStopChan) + close(relistStopChan) // assume all gvks need to be relisted // and while under lock, make a copy of @@ -341,15 +340,15 @@ func (c *CacheManager) manageCache(ctx context.Context) { // clean state c.needToList = false - c.relistStopChan = make(chan struct{}) + relistStopChan = make(chan struct{}) - go c.replayGVKs(ctx, gvksToRelist) + go c.replayGVKs(ctx, gvksToRelist, relistStopChan) }() } } } -func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.GroupVersionKind) { +func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.GroupVersionKind, stopCh <-chan struct{}) { gvksSet := watch.NewSet() gvksSet.Add(gvksToRelist...) @@ -360,7 +359,7 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro select { case <-ctx.Done(): return - case <-c.relistStopChan: + case <-stopCh: return default: operation := func() (bool, error) { From fe408f38ec6cc55dbaf3b2763d5e51916e1ce254 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 3 Aug 2023 17:06:17 +0000 Subject: [PATCH 38/58] add accessors to metrics cache Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/syncutil/stats_reporter.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/syncutil/stats_reporter.go b/pkg/syncutil/stats_reporter.go index adb5cf27cae..2e2320c0d0b 100644 --- a/pkg/syncutil/stats_reporter.go +++ b/pkg/syncutil/stats_reporter.go @@ -108,6 +108,28 @@ func (c *MetricsCache) DeleteObject(key string) { delete(c.Cache, key) } +func (c *MetricsCache) GetTags(key string) *Tags { + c.mux.Lock() + defer c.mux.Unlock() + + cpy := &Tags{} + v, ok := c.Cache[key] + if ok { + cpy.Kind = v.Kind + cpy.Status = v.Status + } + + return cpy +} + +func (c *MetricsCache) HasObject(key string) bool { + c.mux.Lock() + defer c.mux.Unlock() + + _, ok := c.Cache[key] + return ok +} + func (c *MetricsCache) ReportSync() { c.mux.RLock() defer c.mux.RUnlock() From f010d3b13326ecdc9ec9ecbf4267a5c5ace606a0 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 3 Aug 2023 17:07:17 +0000 Subject: [PATCH 39/58] review: table tests Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager_test.go | 596 +++++++++++------- .../cachemanager_integration_test.go | 164 +---- .../config/config_controller_test.go | 1 - 3 files changed, 383 insertions(+), 378 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index b75ccd483f0..355c2d8126c 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -8,6 +8,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" + "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" @@ -19,7 +20,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" ) @@ -29,12 +29,7 @@ func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 2) } -func unitCacheManagerForTest(t *testing.T) (*CacheManager, context.Context) { - cm, _, ctx := makeCacheManager(t) - return cm, ctx -} - -func makeCacheManager(t *testing.T) (*CacheManager, client.Client, context.Context) { +func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) @@ -68,149 +63,81 @@ func makeCacheManager(t *testing.T) (*CacheManager, client.Client, context.Conte }) require.NoError(t, err) - t.Cleanup(func() { - ctx.Done() - }) - t.Cleanup(func() { cancelFunc() + ctx.Done() }) testutils.StartManager(ctx, t, mgr) - return cacheManager, c, ctx + return cacheManager, ctx } -// TestCacheManager_wipeCacheIfNeeded. func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { - cacheManager, ctx := unitCacheManagerForTest(t) - cfClient, ok := cacheManager.cfClient.(*FakeCfClient) - require.True(t, ok) - - // seed one gvk configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - cm := unstructuredFor(configMapGVK, "config-test-1") - _, err := cfClient.AddData(ctx, cm) - require.NoError(t, err, "adding ConfigMap config-test-1 in cfClient") - - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - require.NoError(t, cacheManager.gvksToSync.Upsert(aggregator.Key{Source: "foo", ID: "bar"}, []schema.GroupVersionKind{podGVK})) - - cacheManager.gvksToDeleteFromCache.Add(configMapGVK) - cacheManager.wipeCacheIfNeeded(ctx) + dataClientForTest := func() CFDataClient { + cfdc := &FakeCfClient{} - require.False(t, cfClient.HasGVK(configMapGVK)) - require.ElementsMatch(t, cacheManager.gvksToSync.GVKs(), []schema.GroupVersionKind{podGVK}) -} + cm := unstructuredFor(configMapGVK, "config-test-1") + _, err := cfdc.AddData(context.Background(), cm) -// TestCacheManager_syncGVKInstances tests that GVK instances can be listed and added to the cfClient client. -func TestCacheManager_syncGVKInstances(t *testing.T) { - cacheManager, c, ctx := makeCacheManager(t) + require.NoError(t, err, "adding ConfigMap config-test-1 in cfClient") - configMapGVK := schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "ConfigMap", + return cfdc } - // Create configMaps to test for - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - cm2 := unstructuredFor(configMapGVK, "config-test-2") - require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") - cacheManager.watchedSet.Add(configMapGVK) - require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) - - cfClient, ok := cacheManager.cfClient.(*FakeCfClient) - require.True(t, ok) - expected := map[CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + tcs := []struct { + name string + cm *CacheManager + expectedData map[CfDataKey]interface{} + }{ + { + name: "wipe cache if there are gvks to remove", + cm: &CacheManager{ + cfClient: dataClientForTest(), + gvksToDeleteFromCache: func() *watch.Set { + gvksToDelete := watch.NewSet() + gvksToDelete.Add(configMapGVK) + return gvksToDelete + }(), + syncMetricsCache: syncutil.NewMetricsCache(), + }, + expectedData: map[CfDataKey]interface{}{}, + }, + { + name: "wipe cache if there are excluder changes", + cm: &CacheManager{ + cfClient: dataClientForTest(), + excluderChanged: true, + syncMetricsCache: syncutil.NewMetricsCache(), + gvksToDeleteFromCache: watch.NewSet(), + }, + expectedData: map[CfDataKey]interface{}{}, + }, + { + name: "don't wipe cache if no excluder changes or no gvks to delete", + cm: &CacheManager{ + cfClient: dataClientForTest(), + syncMetricsCache: syncutil.NewMetricsCache(), + gvksToDeleteFromCache: watch.NewSet(), + }, + expectedData: map[CfDataKey]interface{}{{Gvk: configMapGVK, Key: "default/config-test-1"}: nil}, + }, } - require.Equal(t, 2, cfClient.Len()) - require.True(t, cfClient.Contains(expected)) - - // wipe cache - require.NoError(t, cacheManager.wipeData(ctx)) - require.False(t, cfClient.Contains(expected)) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + cfClient, ok := tc.cm.cfClient.(*FakeCfClient) + require.True(t, ok) - // create a second GVK - podGVK := schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", + tc.cm.wipeCacheIfNeeded(context.Background()) + require.True(t, cfClient.Contains(tc.expectedData)) + }) } - // Create pods to test for - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - - pod2 := unstructuredFor(podGVK, "pod-2") - require.NoError(t, c.Create(ctx, pod2), "creating Pod pod-2") - - pod3 := unstructuredFor(podGVK, "pod-3") - require.NoError(t, c.Create(ctx, pod3), "creating Pod pod-3") - - cacheManager.watchedSet.Add(podGVK) - require.NoError(t, cacheManager.syncGVK(ctx, configMapGVK)) - require.NoError(t, cacheManager.syncGVK(ctx, podGVK)) - - expected = map[CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - {Gvk: podGVK, Key: "default/pod-2"}: nil, - {Gvk: podGVK, Key: "default/pod-3"}: nil, - } - - require.Equal(t, 5, cfClient.Len()) - require.True(t, cfClient.Contains(expected)) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") - require.NoError(t, c.Delete(ctx, pod3), "deleting Pod pod-3") - require.NoError(t, c.Delete(ctx, pod2), "deleting Pod pod-2") -} - -// TestCacheManager_wipeCacheIfNeeded_excluderChanges tests that we can remove gvks that were not previously process excluded but are now. -func TestCacheManager_wipeCacheIfNeeded_excluderChanges(t *testing.T) { - cacheManager, ctx := unitCacheManagerForTest(t) - cfClient, ok := cacheManager.cfClient.(*FakeCfClient) - require.True(t, ok) - - // seed gvks - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - cm := unstructuredFor(configMapGVK, "config-test-1") - cm.SetNamespace("excluded-ns") - _, err := cfClient.AddData(ctx, cm) - require.NoError(t, err, "adding ConfigMap config-test-1 in cfClient") - cacheManager.watchedSet.Add(configMapGVK) - - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - pod := unstructuredFor(configMapGVK, "pod-test-1") - pod.SetNamespace("excluded-ns") - _, err = cfClient.AddData(ctx, pod) - require.NoError(t, err, "adding Pod pod-test-1 in cfClient") - cacheManager.watchedSet.Add(podGVK) - - cacheManager.ExcludeProcesses(newSyncExcluderFor("excluded-ns")) - cacheManager.wipeCacheIfNeeded(ctx) - - // the cache manager should not be watching any of the gvks that are now excluded - require.False(t, cfClient.HasGVK(configMapGVK)) - require.False(t, cfClient.HasGVK(podGVK)) - require.False(t, cacheManager.excluderChanged) } -// TestCacheManager_AddObject_RemoveObject tests that we can add/ remove objects in the cache. -func TestCacheManager_AddObject_RemoveObject(t *testing.T) { - cm, ctx := unitCacheManagerForTest(t) - - cfClient, ok := cm.cfClient.(*FakeCfClient) - require.True(t, ok) - +// TestCacheManager_AddObject tests that we can add objects in the cache. +func TestCacheManager_AddObject(t *testing.T) { pod := fakes.Pod( fakes.WithNamespace("test-ns"), fakes.WithName("test-name"), @@ -218,111 +145,369 @@ func TestCacheManager_AddObject_RemoveObject(t *testing.T) { unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) require.NoError(t, err) - // when gvk is watched, we expect Add, Remove to work - cm.watchedSet.Add(pod.GroupVersionKind()) - - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.True(t, cfClient.HasGVK(pod.GroupVersionKind())) + mgr, _ := testutils.SetupManager(t, cfg) - // now remove the object and verify it's removed - require.NoError(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.False(t, cfClient.HasGVK(pod.GroupVersionKind())) - - cm.watchedSet.Remove(pod.GroupVersionKind()) - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) - require.False(t, cfClient.HasGVK(pod.GroupVersionKind())) // we drop calls for gvks that are not watched -} - -// TestCacheManager_AddObject_processExclusion makes sure that we don't add objects that are process excluded. -func TestCacheManager_AddObject_processExclusion(t *testing.T) { - cm, ctx := unitCacheManagerForTest(t) - processExcluder := process.Get() - processExcluder.Add([]configv1alpha1.MatchEntry{ + tcs := []struct { + name string + cm *CacheManager + expectSyncMetric bool + expectedMetricStatus metrics.Status + expectedData map[CfDataKey]interface{} + }{ { - ExcludedNamespaces: []wildcard.Wildcard{"test-ns-excluded"}, - Processes: []string{"sync"}, + name: "AddObject happy path", + cm: &CacheManager{ + cfClient: &FakeCfClient{}, + watchedSet: func() *watch.Set { + ws := watch.NewSet() + ws.Add(pod.GroupVersionKind()) + + return ws + }(), + tracker: readiness.NewTracker(mgr.GetAPIReader(), false, false, false), + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: process.Get(), + }, + expectedData: map[CfDataKey]interface{}{{Gvk: pod.GroupVersionKind(), Key: "test-ns/test-name"}: nil}, + expectSyncMetric: true, + expectedMetricStatus: metrics.ActiveStatus, }, - }) - cm.processExcluder.Replace(processExcluder) - - pod := fakes.Pod( - fakes.WithNamespace("test-ns-excluded"), - fakes.WithName("test-name"), - ) - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + { + name: "AddObject has no effect if GVK is not watched", + cm: &CacheManager{ + cfClient: &FakeCfClient{}, + watchedSet: watch.NewSet(), + tracker: readiness.NewTracker(mgr.GetAPIReader(), false, false, false), + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: process.Get(), + }, + expectedData: map[CfDataKey]interface{}{}, + expectSyncMetric: false, + }, + { + name: "AddObject has no effect if GVK is process excluded", + cm: &CacheManager{ + cfClient: &FakeCfClient{}, + watchedSet: func() *watch.Set { + ws := watch.NewSet() + ws.Add(pod.GroupVersionKind()) + + return ws + }(), + tracker: readiness.NewTracker(mgr.GetAPIReader(), false, false, false), + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: func() *process.Excluder { + processExcluder := process.New() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"test-ns"}, + Processes: []string{"sync"}, + }, + }) + return processExcluder + }(), + }, + expectedData: map[CfDataKey]interface{}{}, + expectSyncMetric: false, + }, + { + name: "AddObject sets metrics on error from cfdataclient", + cm: &CacheManager{ + cfClient: func() CFDataClient { + c := &FakeCfClient{} + c.SetErroring(true) + return c + }(), + watchedSet: func() *watch.Set { + ws := watch.NewSet() + ws.Add(pod.GroupVersionKind()) + + return ws + }(), + tracker: readiness.NewTracker(mgr.GetAPIReader(), false, false, false), + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: process.Get(), + }, + expectedData: map[CfDataKey]interface{}{}, + expectSyncMetric: true, + expectedMetricStatus: metrics.ErrorStatus, + }, + } - unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) - require.NoError(t, err) - require.NoError(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod})) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + err := tc.cm.AddObject(context.Background(), &unstructured.Unstructured{Object: unstructuredPod}) + if tc.expectedMetricStatus == metrics.ActiveStatus { + require.NoError(t, err) + } - // test that pod from excluded namespace is not added to the cache - cfClient, ok := cm.cfClient.(*FakeCfClient) - require.True(t, ok) - require.False(t, cfClient.HasGVK(pod.GroupVersionKind())) - require.False(t, cfClient.Contains(map[CfDataKey]interface{}{{Gvk: podGVK, Key: "default/config-test-1"}: nil})) + assertExpecations(t, tc.cm, &unstructured.Unstructured{Object: unstructuredPod}, tc.expectedData, tc.expectSyncMetric, &tc.expectedMetricStatus) + }) + } } -// TestCacheManager_cfClient_errors tests that the cache manager responds to errors from the cfClient client. -func TestCacheManager_cfClient_errors(t *testing.T) { - cm, ctx := unitCacheManagerForTest(t) +func assertExpecations(t *testing.T, cm *CacheManager, instance *unstructured.Unstructured, expectedData map[CfDataKey]interface{}, expectSyncMetric bool, expectedMetricStatus *metrics.Status) { + t.Helper() + cfClient, ok := cm.cfClient.(*FakeCfClient) require.True(t, ok) - cfClient.SetErroring(true) // This will cause AddObject, RemoveObject to err + require.True(t, cfClient.Contains(expectedData)) + + syncKey := syncutil.GetKeyForSyncMetrics(instance.GetNamespace(), instance.GetName()) + + require.Equal(t, expectSyncMetric, cm.syncMetricsCache.HasObject(syncKey)) + + if expectSyncMetric { + require.Equal(t, *expectedMetricStatus, cm.syncMetricsCache.GetTags(syncKey).Status) + } +} + +// TestCacheManager_RemoveObject tests that we can remove objects from the cache. +func TestCacheManager_RemoveObject(t *testing.T) { pod := fakes.Pod( fakes.WithNamespace("test-ns"), fakes.WithName("test-name"), ) unstructuredPod, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pod) require.NoError(t, err) - cm.watchedSet.Add(pod.GroupVersionKind()) - // test that cm bubbles up the errors - require.ErrorContains(t, cm.AddObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") - require.ErrorContains(t, cm.RemoveObject(ctx, &unstructured.Unstructured{Object: unstructuredPod}), "test error") + mgr, _ := testutils.SetupManager(t, cfg) + tracker := readiness.NewTracker(mgr.GetAPIReader(), false, false, false) + makeDataClient := func() *FakeCfClient { + c := &FakeCfClient{} + _, err := c.AddData(context.Background(), &unstructured.Unstructured{Object: unstructuredPod}) + require.NoError(t, err) + + return c + } + + tcs := []struct { + name string + cm *CacheManager + expectSyncMetric bool + expectedData map[CfDataKey]interface{} + }{ + { + name: "RemoveObject happy path", + cm: &CacheManager{ + cfClient: makeDataClient(), + watchedSet: func() *watch.Set { + ws := watch.NewSet() + ws.Add(pod.GroupVersionKind()) + + return ws + }(), + tracker: tracker, + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: process.Get(), + }, + expectedData: map[CfDataKey]interface{}{}, + expectSyncMetric: false, + }, + { + name: "RemoveObject has no effect if GVK is not watched", + cm: &CacheManager{ + cfClient: makeDataClient(), + watchedSet: watch.NewSet(), + tracker: tracker, + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: process.Get(), + }, + expectedData: map[CfDataKey]interface{}{{Gvk: pod.GroupVersionKind(), Key: "test-ns/test-name"}: nil}, + expectSyncMetric: false, + }, + { + name: "RemoveObject succeeds even if process excluded", + cm: &CacheManager{ + cfClient: makeDataClient(), + watchedSet: func() *watch.Set { + ws := watch.NewSet() + ws.Add(pod.GroupVersionKind()) + + return ws + }(), + tracker: tracker, + syncMetricsCache: syncutil.NewMetricsCache(), + processExcluder: func() *process.Excluder { + processExcluder := process.New() + processExcluder.Add([]configv1alpha1.MatchEntry{ + { + ExcludedNamespaces: []wildcard.Wildcard{"test-ns"}, + Processes: []string{"sync"}, + }, + }) + return processExcluder + }(), + }, + expectedData: map[CfDataKey]interface{}{}, + expectSyncMetric: false, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, tc.cm.RemoveObject(context.Background(), &unstructured.Unstructured{Object: unstructuredPod})) + + assertExpecations(t, tc.cm, &unstructured.Unstructured{Object: unstructuredPod}, tc.expectedData, tc.expectSyncMetric, nil) + }) + } } -// TestCacheManager_AddSource tests that we can modify the gvk aggregator and watched set when adding a new source. -func TestCacheManager_AddSource(t *testing.T) { - cacheManager, ctx := unitCacheManagerForTest(t) +// TestCacheManager_UpsertSource tests that we can modify the gvk aggregator and watched set when adding a new source. +func TestCacheManager_UpsertSource(t *testing.T) { configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - nsGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} sourceA := aggregator.Key{Source: "a", ID: "source"} sourceB := aggregator.Key{Source: "b", ID: "source"} - // given two sources with overlapping gvks ... - require.NoError(t, cacheManager.UpsertSource(ctx, sourceA, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.UpsertSource(ctx, sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) + type sourcesAndGvk struct { + source aggregator.Key + gvks []schema.GroupVersionKind + } - // ... expect the aggregator to dedup - require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) - require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) - require.ElementsMatch(t, cacheManager.watchedSet.Items(), []schema.GroupVersionKind{podGVK, configMapGVK}) + tcs := []struct { + name string + sourcesAndGvks []sourcesAndGvk + expectedGVKs []schema.GroupVersionKind + }{ + { + name: "add one source", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK}, + }, + { + name: "overwrite source", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, + }, + { + source: sourceA, + gvks: []schema.GroupVersionKind{podGVK}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{podGVK}, + }, + { + name: "remove source by not specifying any gvk", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, + }, + { + source: sourceA, + gvks: []schema.GroupVersionKind{}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{}, + }, + { + name: "add two disjoing sources", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK}, + }, + { + source: sourceB, + gvks: []schema.GroupVersionKind{podGVK}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, + }, + { + name: "add two sources with overlapping gvks", + sourcesAndGvks: []sourcesAndGvk{ + { + source: sourceA, + gvks: []schema.GroupVersionKind{configMapGVK, podGVK}, + }, + { + source: sourceB, + gvks: []schema.GroupVersionKind{podGVK}, + }, + }, + expectedGVKs: []schema.GroupVersionKind{configMapGVK, podGVK}, + }, + } - // adding a source without a previously added gvk ... - require.NoError(t, cacheManager.UpsertSource(ctx, sourceB, []schema.GroupVersionKind{configMapGVK})) - // ... should not remove any gvks that are still referenced by other sources - require.True(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) - require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + cacheManager, ctx := makeCacheManager(t) - // adding a source that modifies the only reference to a gvk ... - require.NoError(t, cacheManager.UpsertSource(ctx, sourceB, []schema.GroupVersionKind{nsGVK})) + for _, sourceAndGVK := range tc.sourcesAndGvks { + require.NoError(t, cacheManager.UpsertSource(ctx, sourceAndGVK.source, sourceAndGVK.gvks)) + } - // ... will effectively remove the gvk from the aggregator - require.False(t, cacheManager.gvksToSync.IsPresent(configMapGVK)) - require.True(t, cacheManager.gvksToSync.IsPresent(podGVK)) - require.True(t, cacheManager.gvksToSync.IsPresent(nsGVK)) + require.ElementsMatch(t, cacheManager.watchedSet.Items(), tc.expectedGVKs) + require.ElementsMatch(t, cacheManager.gvksToSync.GVKs(), tc.expectedGVKs) + }) + } } // TestCacheManager_RemoveSource tests that we can modify the gvk aggregator when removing a source. func TestCacheManager_RemoveSource(t *testing.T) { - cacheManager, ctx := unitCacheManagerForTest(t) configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} sourceA := aggregator.Key{Source: "a", ID: "source"} sourceB := aggregator.Key{Source: "b", ID: "source"} + tcs := []struct { + name string + seed func(c *CacheManager) + sourcesToRemove []aggregator.Key + expectedGVKs []schema.GroupVersionKind + }{ + { + name: "remove disjoint source", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{configMapGVK})) + }, + sourcesToRemove: []aggregator.Key{sourceB}, + expectedGVKs: []schema.GroupVersionKind{podGVK}, + }, + { + name: "remove overlapping source", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + require.NoError(t, c.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK})) + }, + sourcesToRemove: []aggregator.Key{sourceB}, + expectedGVKs: []schema.GroupVersionKind{podGVK}, + }, + { + name: "remove non existing source", + seed: func(c *CacheManager) { + require.NoError(t, c.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) + }, + sourcesToRemove: []aggregator.Key{sourceB}, + expectedGVKs: []schema.GroupVersionKind{podGVK}, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + cm, ctx := makeCacheManager(t) + tc.seed(cm) + + for _, source := range tc.sourcesToRemove { + require.NoError(t, cm.RemoveSource(ctx, source)) + } + + require.ElementsMatch(t, cm.gvksToSync.GVKs(), tc.expectedGVKs) + }) + } + cacheManager, ctx := makeCacheManager(t) + // seed the gvk aggregator require.NoError(t, cacheManager.gvksToSync.Upsert(sourceA, []schema.GroupVersionKind{podGVK})) require.NoError(t, cacheManager.gvksToSync.Upsert(sourceB, []schema.GroupVersionKind{podGVK, configMapGVK})) @@ -337,23 +522,6 @@ func TestCacheManager_RemoveSource(t *testing.T) { require.False(t, cacheManager.gvksToSync.IsPresent(podGVK)) } -func newSyncExcluderFor(nsToExclude string) *process.Excluder { - excluder := process.New() - excluder.Add([]configv1alpha1.MatchEntry{ - { - ExcludedNamespaces: []wildcard.Wildcard{wildcard.Wildcard(nsToExclude)}, - Processes: []string{"sync"}, - }, - // exclude kube-system by default to prevent noise - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - - return excluder -} - func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Unstructured { u := &unstructured.Unstructured{} u.SetGroupVersionKind(gvk) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index d050f4f12a1..eaea41a757a 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -101,165 +101,6 @@ func TestCacheManager_replay_retries(t *testing.T) { require.NoError(t, c.Delete(ctx, pod), "creating ConfigMap pod-1") } -// TestCacheManager_AddSourceRemoveSource makes sure that we can add and remove multiple sources -// and changes to the underlying cache are reflected. -func TestCacheManager_AddSourceRemoveSource(t *testing.T) { - mgr, wm := testutils.SetupManager(t, cfg) - c := testclient.NewRetryClient(mgr.GetClient()) - cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) - - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - - // Create configMaps to test for - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - - cm2 := unstructuredFor(configMapGVK, "config-test-2") - require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") - - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") - - cfClient, ok := dataStore.(*cachemanager.FakeCfClient) - require.True(t, ok) - - syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - - expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - - require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - - // now assert that the gvkAggregator looks as expected - agg.IsPresent(configMapGVK) - gvks := agg.List(syncSourceOne) - require.Len(t, gvks, 2) - _, foundConfigMap := gvks[configMapGVK] - require.True(t, foundConfigMap) - _, foundPod := gvks[podGVK] - require.True(t, foundPod) - - // now remove the podgvk and make sure we don't have pods in the cache anymore - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - - expected = map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - } - require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - // now assert that the gvkAggregator looks as expected - agg.IsPresent(configMapGVK) - gvks = agg.List(syncSourceOne) - require.Len(t, gvks, 1) - _, foundConfigMap = gvks[configMapGVK] - require.True(t, foundConfigMap) - _, foundPod = gvks[podGVK] - require.False(t, foundPod) - - // now make sure that adding another sync source with the same gvk has no side effects - syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - agg.IsPresent(configMapGVK) - gvks = agg.List(syncSourceTwo) - require.Len(t, gvks, 1) - _, foundConfigMap = gvks[configMapGVK] - require.True(t, foundConfigMap) - - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - expected2 := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) - - // now go on and unreference sourceTwo's gvks; this should schedule the config maps to be removed - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{})) - expected3 := map[cachemanager.CfDataKey]interface{}{ - // config maps no longer required by any sync source - // {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - // {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, - } - require.Eventually(t, expectedCheck(cfClient, expected3), eventuallyTimeout, eventuallyTicker) - - // now remove all the sources - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) - - // and expect an empty cache and empty aggregator - require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) - require.True(t, len(agg.GVKs()) == 0) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") -} - -// TestCacheManager_ExcludeProcesses makes sure that changing the process excluder -// in the cache manager triggers a re-evaluation of GVKs. -func TestCacheManager_ExcludeProcesses(t *testing.T) { - mgr, wm := testutils.SetupManager(t, cfg) - c := testclient.NewRetryClient(mgr.GetClient()) - cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) - - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") - - cfClient, ok := dataStore.(*cachemanager.FakeCfClient) - require.True(t, ok) - - expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - } - - syncSource := aggregator.Key{Source: "source_b", ID: "ID_b"} - require.NoError(t, cacheManager.UpsertSource(ctx, syncSource, []schema.GroupVersionKind{configMapGVK})) - // check that everything is correctly added at first - require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - - // make sure that replacing w same process excluder is a no op - sameExcluder := process.New() - sameExcluder.Add([]configv1alpha1.MatchEntry{ - // same excluder as the one in makeCacheManagerForTest - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - cacheManager.ExcludeProcesses(sameExcluder) - - // now process exclude the remaining gvk, it should get removed by the background process. - excluder := process.New() - excluder.Add([]configv1alpha1.MatchEntry{ - // exclude the "default" namespace - { - ExcludedNamespaces: []wildcard.Wildcard{"default"}, - Processes: []string{"sync"}, - }, - { - ExcludedNamespaces: []wildcard.Wildcard{"kube-system"}, - Processes: []string{"sync"}, - }, - }) - cacheManager.ExcludeProcesses(excluder) - - require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) - // make sure the gvk is still in gvkAggregator - require.True(t, len(agg.GVKs()) == 1) - require.True(t, agg.IsPresent(configMapGVK)) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") -} - func expectedCheck(cfClient *cachemanager.FakeCfClient, expected map[cachemanager.CfDataKey]interface{}) func() bool { return func() bool { if cfClient.Len() != len(expected) { @@ -333,14 +174,11 @@ func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, r require.NoError(t, cacheManager.Start(ctx)) }() - t.Cleanup(func() { - ctx.Done() - }) - testutils.StartManager(ctx, t, mgr) t.Cleanup(func() { cancelFunc() + ctx.Done() }) return cacheManager, cfClient, aggregator, ctx } diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 2067c9aed52..1d5ad4c1a09 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -534,7 +534,6 @@ func TestConfig_CacheContents(t *testing.T) { // Reconfigure to drop the namespace watches config = configFor([]schema.GroupVersionKind{configMapGVK}) configUpdate := config.DeepCopy() - // configUpdate.SetResourceVersion() require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(configUpdate), configUpdate)) configUpdate.Spec = config.Spec From 86425fc01d06e91f5be02cdf93251a18ce4986b2 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:58:21 +0000 Subject: [PATCH 40/58] add a concurrent test Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../cachemanager_integration_test.go | 153 +++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index eaea41a757a..ebc993421a4 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -3,6 +3,7 @@ package cachemanager_test import ( "context" "fmt" + "sync" "testing" "time" @@ -10,7 +11,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager/aggregator" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" - "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" + syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/fakes" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" @@ -101,6 +102,154 @@ func TestCacheManager_replay_retries(t *testing.T) { require.NoError(t, c.Delete(ctx, pod), "creating ConfigMap pod-1") } +// TestCacheManager_concurrent makes sure that we can add and remove multiple sources +// from separate go routines and changes to the underlying cache are reflected. +func TestCacheManager_concurrent(t *testing.T) { + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) + + configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + + // Create configMaps to test for + cm := unstructuredFor(configMapGVK, "config-test-1") + require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + + cm2 := unstructuredFor(configMapGVK, "config-test-2") + require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + + pod := unstructuredFor(podGVK, "pod-1") + require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + + cfClient, ok := dataStore.(*cachemanager.FakeCfClient) + require.True(t, ok) + + syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} + syncSourceTwo := aggregator.Key{Source: "source_b", ID: "ID_b"} + + wg := &sync.WaitGroup{} + + wg.Add(2) + go func() { + defer wg.Done() + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + }() + go func() { + defer wg.Done() + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) + }() + + wg.Wait() + + expected := map[cachemanager.CfDataKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) + // now assert that the gvkAggregator looks as expected + agg.IsPresent(configMapGVK) + gvks := agg.List(syncSourceOne) + require.Len(t, gvks, 1) + _, foundConfigMap := gvks[configMapGVK] + require.True(t, foundConfigMap) + gvks = agg.List(syncSourceTwo) + require.Len(t, gvks, 1) + _, foundPod := gvks[podGVK] + require.True(t, foundPod) + + // now remove the podgvk for sync source two and make sure we don't have pods in the cache anymore + wg.Add(1) + go func() { + defer wg.Done() + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) + }() + + wg.Wait() + + // expecte the config map instances to be repopulated eventually + expected = map[cachemanager.CfDataKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + } + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) + // now assert that the gvkAggregator looks as expected + agg.IsPresent(configMapGVK) + gvks = agg.List(syncSourceOne) + require.Len(t, gvks, 1) + _, foundConfigMap = gvks[configMapGVK] + require.True(t, foundConfigMap) + _, foundPod = gvks[podGVK] + require.False(t, foundPod) + + // now swap the gvks for each source and do so repeatedly to generate some churn + wg.Add(1) + go func() { + defer wg.Done() + + order := true + for i := 1; i <= 10; i++ { + if order { + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) + } else { + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) + } + + order = !order + } + + // final upsert for determinism + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) + }() + + wg.Wait() + + expected = map[cachemanager.CfDataKey]interface{}{ + {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: podGVK, Key: "default/pod-1"}: nil, + } + + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) + // now assert that the gvkAggregator looks as expected + agg.IsPresent(configMapGVK) + gvks = agg.List(syncSourceOne) + require.Len(t, gvks, 1) + _, foundConfigMap = gvks[configMapGVK] + require.True(t, foundConfigMap) + gvks = agg.List(syncSourceTwo) + require.Len(t, gvks, 1) + _, foundPod = gvks[podGVK] + require.True(t, foundPod) + + // now remove the sources + wg.Add(2) + go func() { + defer wg.Done() + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) + }() + go func() { + defer wg.Done() + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) + }() + + wg.Wait() + + // and expect an empty cache and empty aggregator + require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.True(t, len(agg.GVKs()) == 0) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") + require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") + require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") +} + func expectedCheck(cfClient *cachemanager.FakeCfClient, expected map[cachemanager.CfDataKey]interface{}) func() bool { return func() bool { if cfClient.Len() != len(expected) { @@ -165,7 +314,7 @@ func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, r cacheManager, err := cachemanager.NewCacheManager(cfg) require.NoError(t, err) - syncAdder := sync.Adder{ + syncAdder := syncc.Adder{ Events: events, CacheManager: cacheManager, } From a563e9eee593d46f37c2b19200bb2ef128d26ab4 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Sat, 5 Aug 2023 00:11:12 +0000 Subject: [PATCH 41/58] review: add a test for instance updates also vars, comments from review Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../cachemanager_integration_test.go | 164 +++++++++++------- pkg/cachemanager/fakecfdataclient.go | 10 ++ .../config/config_controller_test.go | 4 +- pkg/fakes/reader.go | 4 +- 4 files changed, 117 insertions(+), 65 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index ebc993421a4..fb0c9ba74ee 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -35,6 +35,19 @@ const ( var cfg *rest.Config +var ( + configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} + podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} + + configInstancOne = "config-test-1" + configInstancTwo = "config-test-2" + nsedConfigInstanceOne = "default/config-test-1" + nsedConfigInstanceTwo = "default/config-test-2" + + podInstanceOne = "pod-test-1" + nsedPodInstanceOne = "default/pod-test-1" +) + func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 3) } @@ -45,7 +58,7 @@ func TestCacheManager_replay_retries(t *testing.T) { c := testclient.NewRetryClient(mgr.GetClient()) failPlease := make(chan string, 2) - reader := fakes.HookReader{ + reader := fakes.SpyReader{ Reader: c, ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // Return an error the first go-around. @@ -67,20 +80,18 @@ func TestCacheManager_replay_retries(t *testing.T) { require.True(t, ok) // seed one gvk - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + cm := unstructuredFor(configMapGVK, configInstancOne) + require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + pod := unstructuredFor(podGVK, podInstanceOne) + require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", podInstanceOne)) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -93,13 +104,13 @@ func TestCacheManager_replay_retries(t *testing.T) { require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected2 := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, } require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) // cleanup - require.NoError(t, c.Delete(ctx, cm), "creating ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, pod), "creating ConfigMap pod-1") + require.NoError(t, c.Delete(ctx, cm), fmt.Sprintf("deleting ConfigMap %s", configInstancOne)) + require.NoError(t, c.Delete(ctx, pod), fmt.Sprintf("deleting Pod %s", configInstancOne)) } // TestCacheManager_concurrent makes sure that we can add and remove multiple sources @@ -109,18 +120,15 @@ func TestCacheManager_concurrent(t *testing.T) { c := testclient.NewRetryClient(mgr.GetClient()) cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) - configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} - podGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - // Create configMaps to test for - cm := unstructuredFor(configMapGVK, "config-test-1") - require.NoError(t, c.Create(ctx, cm), "creating ConfigMap config-test-1") + cm := unstructuredFor(configMapGVK, configInstancOne) + require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) - cm2 := unstructuredFor(configMapGVK, "config-test-2") - require.NoError(t, c.Create(ctx, cm2), "creating ConfigMap config-test-2") + cm2 := unstructuredFor(configMapGVK, configInstancTwo) + require.NoError(t, c.Create(ctx, cm2), fmt.Sprintf("creating ConfigMap %s", configInstancTwo)) - pod := unstructuredFor(podGVK, "pod-1") - require.NoError(t, c.Create(ctx, pod), "creating Pod pod-1") + pod := unstructuredFor(podGVK, podInstanceOne) + require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", podInstanceOne)) cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) @@ -143,9 +151,9 @@ func TestCacheManager_concurrent(t *testing.T) { wg.Wait() expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, + {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -161,18 +169,12 @@ func TestCacheManager_concurrent(t *testing.T) { require.True(t, foundPod) // now remove the podgvk for sync source two and make sure we don't have pods in the cache anymore - wg.Add(1) - go func() { - defer wg.Done() - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - }() - - wg.Wait() + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - // expecte the config map instances to be repopulated eventually + // expect the config map instances to be repopulated eventually expected = map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected @@ -185,34 +187,32 @@ func TestCacheManager_concurrent(t *testing.T) { require.False(t, foundPod) // now swap the gvks for each source and do so repeatedly to generate some churn - wg.Add(1) - go func() { - defer wg.Done() - - order := true - for i := 1; i <= 10; i++ { - if order { - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) - } else { - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - } - - order = !order - } - - // final upsert for determinism - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) - }() + for i := 1; i < 100; i++ { + wg.Add(2) + go func() { + defer wg.Done() + + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) + }() + go func() { + defer wg.Done() + + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) + }() + } wg.Wait() + // final upsert for determinism + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) + expected = map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-2"}: nil, - {Gvk: podGVK, Key: "default/pod-1"}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, + {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -245,9 +245,51 @@ func TestCacheManager_concurrent(t *testing.T) { require.True(t, len(agg.GVKs()) == 0) // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting ConfigMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting ConfigMap config-test-2") - require.NoError(t, c.Delete(ctx, pod), "deleting Pod pod-1") + require.NoError(t, c.Delete(ctx, cm), fmt.Sprintf("deleting ConfigMap %s", configInstancOne)) + require.NoError(t, c.Delete(ctx, cm2), fmt.Sprintf("deleting ConfigMap %s", configInstancTwo)) + require.NoError(t, c.Delete(ctx, pod), fmt.Sprintf("deleting Pod %s", configInstancOne)) +} + +// TestCacheManager_instance_updates tests that cache manager wires up dependencies correctly +// such that updates to an instance of a watched gvks are reconciled in the sync_controller. +func TestCacheManager_instance_updates(t *testing.T) { + mgr, wm := testutils.SetupManager(t, cfg) + c := testclient.NewRetryClient(mgr.GetClient()) + + cacheManager, dataStore, _, ctx := cacheManagerForTest(t, mgr, wm, c) + + cfClient, ok := dataStore.(*cachemanager.FakeCfClient) + require.True(t, ok) + + cm := unstructuredFor(configMapGVK, configInstancOne) + require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + + syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + + expected := map[cachemanager.CfDataKey]interface{}{ + {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + } + + require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) + + cm2 := unstructuredFor(configMapGVK, configInstancOne) + cm2.SetLabels(map[string]string{"testlabel": "test"}) // trigger an instance update + require.NoError(t, c.Update(ctx, cm2)) + + require.Eventually(t, func() bool { + instance := cfClient.GetData(cachemanager.CfDataKey{Gvk: configMapGVK, Key: nsedConfigInstanceOne}) + unInstance, ok := instance.(*unstructured.Unstructured) + require.True(t, ok) + + value, found, err := unstructured.NestedString(unInstance.Object, "metadata", "labels", "testlabel") + require.NoError(t, err) + + return found && "test" == value + }, eventuallyTimeout, eventuallyTicker) + + // cleanup + require.NoError(t, c.Delete(ctx, cm), fmt.Sprintf("deleting ConfigMap %s", configInstancOne)) } func expectedCheck(cfClient *cachemanager.FakeCfClient, expected map[cachemanager.CfDataKey]interface{}) func() bool { diff --git a/pkg/cachemanager/fakecfdataclient.go b/pkg/cachemanager/fakecfdataclient.go index e903deea117..d0bf42428b9 100644 --- a/pkg/cachemanager/fakecfdataclient.go +++ b/pkg/cachemanager/fakecfdataclient.go @@ -97,6 +97,16 @@ func (f *FakeCfClient) RemoveData(ctx context.Context, data interface{}) (*const return &constraintTypes.Responses{}, nil } +// GetData returns data for a CfDataKey. It assumes that the +// key is present in the FakeCfClient. Also the data returned is not copied +// and it's meant only for assertions not modifications. +func (f *FakeCfClient) GetData(key CfDataKey) interface{} { + f.mu.Lock() + defer f.mu.Unlock() + + return f.data[key] +} + // Contains returns true if all expected resources are in the cache. func (f *FakeCfClient) Contains(expected map[CfDataKey]interface{}) bool { f.mu.Lock() diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 1d5ad4c1a09..7fc486be0ff 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -642,9 +642,9 @@ func TestConfig_Retries(t *testing.T) { } require.NoError(t, syncAdder.Add(mgr), "registering sync controller") - // Use our special hookReader to inject controlled failures + // Use our special reader interceptor to inject controlled failures failPlease := make(chan string, 1) - rec.reader = fakes.HookReader{ + rec.reader = fakes.SpyReader{ Reader: mgr.GetCache(), ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // Return an error the first go-around. diff --git a/pkg/fakes/reader.go b/pkg/fakes/reader.go index 0d930404413..0bfd88b285e 100644 --- a/pkg/fakes/reader.go +++ b/pkg/fakes/reader.go @@ -6,12 +6,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type HookReader struct { +type SpyReader struct { client.Reader ListFunc func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error } -func (r HookReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { +func (r SpyReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { if r.ListFunc != nil { return r.ListFunc(ctx, list, opts...) } From 38631841285917093c7765bccc6cad95bbb3973b Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 7 Aug 2023 19:09:37 +0000 Subject: [PATCH 42/58] review, test: use struct for test resources, use maps for failures also: - delete/ reword some comments Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../cachemanager_integration_test.go | 83 +++++++++++++++---- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index fb0c9ba74ee..6c9c78fad4d 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -52,34 +52,69 @@ func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 3) } -// TestCacheManager_replay_retries tests that we can retry GVKs that error out in the reply goroutine. +type failureInjector struct { + mu sync.Mutex + failures map[string]int // registers GVK.Kind and how many times to fail +} + +func (f *failureInjector) setFailures(kind string, failures int) { + f.mu.Lock() + defer f.mu.Unlock() + + f.failures[kind] = failures +} + +// checkFailures looks at the count of failures and returns true +// if there are still failures for the kind to consume, false otherwise. +func (f *failureInjector) checkFailures(kind string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + v, ok := f.failures[kind] + if !ok { + return false + } + + if v == 0 { + return false + } + + f.failures[kind] = v - 1 + + return true +} + +func newFailureInjector() *failureInjector { + return &failureInjector{ + failures: make(map[string]int), + } +} + +// TestCacheManager_replay_retries tests that we can retry GVKs that error out in the replay goroutine. func TestCacheManager_replay_retries(t *testing.T) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - failPlease := make(chan string, 2) + fi := newFailureInjector() reader := fakes.SpyReader{ Reader: c, ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - // Return an error the first go-around. - var failKind string - select { - case failKind = <-failPlease: - default: - } - if failKind != "" && list.GetObjectKind().GroupVersionKind().Kind == failKind { + // return as many syntenthic failures as there are registered for this kind + if fi.checkFailures(list.GetObjectKind().GroupVersionKind().Kind) { return fmt.Errorf("synthetic failure") } + return c.List(ctx, list, opts...) }, } - cacheManager, dataStore, _, ctx := cacheManagerForTest(t, mgr, wm, reader) + testResources, ctx := makeTestResources(t, mgr, wm, reader) + cacheManager := testResources.CacheManager + dataStore := testResources.CFDataClient cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) - // seed one gvk cm := unstructuredFor(configMapGVK, configInstancOne) require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) @@ -96,9 +131,7 @@ func TestCacheManager_replay_retries(t *testing.T) { require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - // set up a scenario where the list from replay will fail a few times - failPlease <- "ConfigMapList" - failPlease <- "ConfigMapList" + fi.setFailures("ConfigMapList", 5) // this call should schedule a cache wipe and a replay for the configMapGVK require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) @@ -118,7 +151,11 @@ func TestCacheManager_replay_retries(t *testing.T) { func TestCacheManager_concurrent(t *testing.T) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - cacheManager, dataStore, agg, ctx := cacheManagerForTest(t, mgr, wm, c) + testResources, ctx := makeTestResources(t, mgr, wm, c) + + cacheManager := testResources.CacheManager + dataStore := testResources.CFDataClient + agg := testResources.GVKAgreggator // Create configMaps to test for cm := unstructuredFor(configMapGVK, configInstancOne) @@ -256,7 +293,10 @@ func TestCacheManager_instance_updates(t *testing.T) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - cacheManager, dataStore, _, ctx := cacheManagerForTest(t, mgr, wm, c) + testResources, ctx := makeTestResources(t, mgr, wm, c) + + cacheManager := testResources.CacheManager + dataStore := testResources.CFDataClient cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) @@ -323,7 +363,13 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns return u } -func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, reader client.Reader) (*cachemanager.CacheManager, cachemanager.CFDataClient, *aggregator.GVKAgreggator, context.Context) { +type testResources struct { + *cachemanager.CacheManager + cachemanager.CFDataClient + *aggregator.GVKAgreggator +} + +func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, reader client.Reader) (testResources, context.Context) { ctx, cancelFunc := context.WithCancel(context.Background()) cfClient := &cachemanager.FakeCfClient{} @@ -371,5 +417,6 @@ func cacheManagerForTest(t *testing.T, mgr manager.Manager, wm *watch.Manager, r cancelFunc() ctx.Done() }) - return cacheManager, cfClient, aggregator, ctx + + return testResources{cacheManager, cfClient, aggregator}, ctx } From db5a0cf9aec3f824eed8ff41306548031e769ced Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Mon, 7 Aug 2023 23:48:02 +0000 Subject: [PATCH 43/58] review: better test cleanup Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager_test.go | 1 - .../cachemanager_integration_test.go | 41 +++++++++++++------ .../config/config_controller_test.go | 27 +++++++++--- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 355c2d8126c..fc1f7ae7805 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -65,7 +65,6 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { t.Cleanup(func() { cancelFunc() - ctx.Done() }) testutils.StartManager(ctx, t, mgr) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index 6c9c78fad4d..d44f9663d2d 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -19,7 +19,9 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/wildcard" testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" @@ -117,9 +119,15 @@ func TestCacheManager_replay_retries(t *testing.T) { cm := unstructuredFor(configMapGVK, configInstancOne) require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", configInstancOne)) + }) pod := unstructuredFor(podGVK, podInstanceOne) require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", podInstanceOne)) + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", podInstanceOne)) + }) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) @@ -140,10 +148,6 @@ func TestCacheManager_replay_retries(t *testing.T) { {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, } require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), fmt.Sprintf("deleting ConfigMap %s", configInstancOne)) - require.NoError(t, c.Delete(ctx, pod), fmt.Sprintf("deleting Pod %s", configInstancOne)) } // TestCacheManager_concurrent makes sure that we can add and remove multiple sources @@ -160,12 +164,21 @@ func TestCacheManager_concurrent(t *testing.T) { // Create configMaps to test for cm := unstructuredFor(configMapGVK, configInstancOne) require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", configInstancOne)) + }) cm2 := unstructuredFor(configMapGVK, configInstancTwo) require.NoError(t, c.Create(ctx, cm2), fmt.Sprintf("creating ConfigMap %s", configInstancTwo)) + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, cm2), fmt.Sprintf("deleting resource %s", configInstancTwo)) + }) pod := unstructuredFor(podGVK, podInstanceOne) require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", podInstanceOne)) + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", podInstanceOne)) + }) cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) @@ -280,11 +293,6 @@ func TestCacheManager_concurrent(t *testing.T) { // and expect an empty cache and empty aggregator require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) require.True(t, len(agg.GVKs()) == 0) - - // cleanup - require.NoError(t, c.Delete(ctx, cm), fmt.Sprintf("deleting ConfigMap %s", configInstancOne)) - require.NoError(t, c.Delete(ctx, cm2), fmt.Sprintf("deleting ConfigMap %s", configInstancTwo)) - require.NoError(t, c.Delete(ctx, pod), fmt.Sprintf("deleting Pod %s", configInstancOne)) } // TestCacheManager_instance_updates tests that cache manager wires up dependencies correctly @@ -303,6 +311,9 @@ func TestCacheManager_instance_updates(t *testing.T) { cm := unstructuredFor(configMapGVK, configInstancOne) require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", configInstancOne)) + }) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) @@ -327,9 +338,16 @@ func TestCacheManager_instance_updates(t *testing.T) { return found && "test" == value }, eventuallyTimeout, eventuallyTicker) +} + +func deleteResource(ctx context.Context, c client.Client, resounce *unstructured.Unstructured) error { + err := c.Delete(ctx, resounce) + if apierrors.IsNotFound(err) { + // resource does not exist, this is good + return nil + } - // cleanup - require.NoError(t, c.Delete(ctx, cm), fmt.Sprintf("deleting ConfigMap %s", configInstancOne)) + return err } func expectedCheck(cfClient *cachemanager.FakeCfClient, expected map[cachemanager.CfDataKey]interface{}) func() bool { @@ -415,7 +433,6 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea t.Cleanup(func() { cancelFunc() - ctx.Done() }) return testResources{cacheManager, cfClient, aggregator}, ctx diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 7fc486be0ff..71411f8af02 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -38,10 +38,12 @@ import ( testclient "github.com/open-policy-agent/gatekeeper/v3/test/clients" "github.com/open-policy-agent/gatekeeper/v3/test/testutils" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -493,10 +495,16 @@ func TestConfig_CacheContents(t *testing.T) { cm := unstructuredFor(configMapGVK, "config-test-1") cm.SetNamespace("default") require.NoError(t, c.Create(ctx, cm), "creating configMap config-test-1") + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, cm), "deleting configMap config-test-1") + }) cm2 := unstructuredFor(configMapGVK, "config-test-2") cm2.SetNamespace("kube-system") - require.NoError(t, c.Create(ctx, cm2), "creating configMap config-test21") + require.NoError(t, c.Create(ctx, cm2), "creating configMap config-test-2") + t.Cleanup(func() { + assert.NoError(t, deleteResource(ctx, c, cm2), "deleting configMap config-test-2") + }) tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) @@ -576,10 +584,6 @@ func TestConfig_CacheContents(t *testing.T) { g.Eventually(func() int { return opaClient.Len() }, 10*time.Second).Should(gomega.BeZero(), "waiting for cache to empty") - - // cleanup - require.NoError(t, c.Delete(ctx, cm), "deleting configMap config-test-1") - require.NoError(t, c.Delete(ctx, cm2), "deleting configMap config-test-2") } func TestConfig_Retries(t *testing.T) { @@ -769,3 +773,16 @@ func unstructuredFor(gvk schema.GroupVersionKind, name string) *unstructured.Uns type testExpectations interface { IsExpecting(gvk schema.GroupVersionKind, nsName types.NamespacedName) bool } + +func deleteResource(ctx context.Context, c client.Client, resounce *unstructured.Unstructured) error { + if ctx.Err() != nil { + ctx = context.Background() + } + err := c.Delete(ctx, resounce) + if apierrors.IsNotFound(err) { + // resource does not exist, this is good + return nil + } + + return err +} From a799d605da6338d04f2b324a43859886434cd521 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 8 Aug 2023 00:02:33 +0000 Subject: [PATCH 44/58] review, test: add jiiter, remove sources, shorten test Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../cachemanager_integration_test.go | 79 ++++++------------- 1 file changed, 25 insertions(+), 54 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index d44f9663d2d..15e3166c995 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -3,6 +3,7 @@ package cachemanager_test import ( "context" "fmt" + "math/rand" "sync" "testing" "time" @@ -153,6 +154,8 @@ func TestCacheManager_replay_retries(t *testing.T) { // TestCacheManager_concurrent makes sure that we can add and remove multiple sources // from separate go routines and changes to the underlying cache are reflected. func TestCacheManager_concurrent(t *testing.T) { + r := rand.New(rand.NewSource(12345)) // #nosec G404: Using weak random number generator for determinism between calls + mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) testResources, ctx := makeTestResources(t, mgr, wm, c) @@ -188,69 +191,37 @@ func TestCacheManager_concurrent(t *testing.T) { wg := &sync.WaitGroup{} - wg.Add(2) - go func() { - defer wg.Done() - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - }() - go func() { - defer wg.Done() - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) - }() - - wg.Wait() - - expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, - {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, - {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, - } - - require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - // now assert that the gvkAggregator looks as expected - agg.IsPresent(configMapGVK) - gvks := agg.List(syncSourceOne) - require.Len(t, gvks, 1) - _, foundConfigMap := gvks[configMapGVK] - require.True(t, foundConfigMap) - gvks = agg.List(syncSourceTwo) - require.Len(t, gvks, 1) - _, foundPod := gvks[podGVK] - require.True(t, foundPod) - - // now remove the podgvk for sync source two and make sure we don't have pods in the cache anymore - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) - - // expect the config map instances to be repopulated eventually - expected = map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, - {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, - } - require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - // now assert that the gvkAggregator looks as expected - agg.IsPresent(configMapGVK) - gvks = agg.List(syncSourceOne) - require.Len(t, gvks, 1) - _, foundConfigMap = gvks[configMapGVK] - require.True(t, foundConfigMap) - _, foundPod = gvks[podGVK] - require.False(t, foundPod) - - // now swap the gvks for each source and do so repeatedly to generate some churn + // simulate a churn-y concurrent access by swapping the gvks for the sync sources repeatedly + // and removing sync sources, all from different go routines. for i := 1; i < 100; i++ { - wg.Add(2) + wg.Add(3) go func() { defer wg.Done() + // add some jitter + time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) }() go func() { defer wg.Done() + // add some jitter + time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) + require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) }() + go func() { + defer wg.Done() + + time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) + + time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) + }() } wg.Wait() @@ -259,7 +230,7 @@ func TestCacheManager_concurrent(t *testing.T) { require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) - expected = map[cachemanager.CfDataKey]interface{}{ + expected := map[cachemanager.CfDataKey]interface{}{ {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, @@ -268,13 +239,13 @@ func TestCacheManager_concurrent(t *testing.T) { require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) // now assert that the gvkAggregator looks as expected agg.IsPresent(configMapGVK) - gvks = agg.List(syncSourceOne) + gvks := agg.List(syncSourceOne) require.Len(t, gvks, 1) - _, foundConfigMap = gvks[configMapGVK] + _, foundConfigMap := gvks[configMapGVK] require.True(t, foundConfigMap) gvks = agg.List(syncSourceTwo) require.Len(t, gvks, 1) - _, foundPod = gvks[podGVK] + _, foundPod := gvks[podGVK] require.True(t, foundPod) // now remove the sources From 734c0efb0694f4f4ff07dd8c9a1f33ac8de6c4f6 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 8 Aug 2023 18:22:04 +0000 Subject: [PATCH 45/58] review: move fi, var names Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../cachemanager_integration_test.go | 152 ++++++------------ pkg/fakes/reader.go | 39 +++++ 2 files changed, 91 insertions(+), 100 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index 15e3166c995..eacb704377f 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -34,6 +34,8 @@ import ( const ( eventuallyTimeout = 10 * time.Second eventuallyTicker = 2 * time.Second + + jitterUpperBound = 100 ) var cfg *rest.Config @@ -42,68 +44,30 @@ var ( configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - configInstancOne = "config-test-1" - configInstancTwo = "config-test-2" - nsedConfigInstanceOne = "default/config-test-1" - nsedConfigInstanceTwo = "default/config-test-2" + cm1Name = "config-test-1" + cm2Name = "config-test-2" + namespacedCm1Name = "default/config-test-1" + namespacedCm2Name = "default/config-test-2" - podInstanceOne = "pod-test-1" - nsedPodInstanceOne = "default/pod-test-1" + pod1Name = "pod-test-1" + namespacedPod1Name = "default/pod-test-1" ) func TestMain(m *testing.M) { testutils.StartControlPlane(m, &cfg, 3) } -type failureInjector struct { - mu sync.Mutex - failures map[string]int // registers GVK.Kind and how many times to fail -} - -func (f *failureInjector) setFailures(kind string, failures int) { - f.mu.Lock() - defer f.mu.Unlock() - - f.failures[kind] = failures -} - -// checkFailures looks at the count of failures and returns true -// if there are still failures for the kind to consume, false otherwise. -func (f *failureInjector) checkFailures(kind string) bool { - f.mu.Lock() - defer f.mu.Unlock() - - v, ok := f.failures[kind] - if !ok { - return false - } - - if v == 0 { - return false - } - - f.failures[kind] = v - 1 - - return true -} - -func newFailureInjector() *failureInjector { - return &failureInjector{ - failures: make(map[string]int), - } -} - // TestCacheManager_replay_retries tests that we can retry GVKs that error out in the replay goroutine. func TestCacheManager_replay_retries(t *testing.T) { mgr, wm := testutils.SetupManager(t, cfg) c := testclient.NewRetryClient(mgr.GetClient()) - fi := newFailureInjector() + fi := fakes.NewFailureInjector() reader := fakes.SpyReader{ Reader: c, ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { // return as many syntenthic failures as there are registered for this kind - if fi.checkFailures(list.GetObjectKind().GroupVersionKind().Kind) { + if fi.CheckFailures(list.GetObjectKind().GroupVersionKind().Kind) { return fmt.Errorf("synthetic failure") } @@ -118,35 +82,35 @@ func TestCacheManager_replay_retries(t *testing.T) { cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) - cm := unstructuredFor(configMapGVK, configInstancOne) - require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + cm := unstructuredFor(configMapGVK, cm1Name) + require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", cm1Name)) t.Cleanup(func() { - assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", configInstancOne)) + assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) - pod := unstructuredFor(podGVK, podInstanceOne) - require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", podInstanceOne)) + pod := unstructuredFor(podGVK, pod1Name) + require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", pod1Name)) t.Cleanup(func() { - assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", podInstanceOne)) + assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", pod1Name)) }) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, - {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, + {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, + {Gvk: podGVK, Key: namespacedPod1Name}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - fi.setFailures("ConfigMapList", 5) + fi.SetFailures("ConfigMapList", 5) // this call should schedule a cache wipe and a replay for the configMapGVK require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected2 := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, } require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) } @@ -165,22 +129,22 @@ func TestCacheManager_concurrent(t *testing.T) { agg := testResources.GVKAgreggator // Create configMaps to test for - cm := unstructuredFor(configMapGVK, configInstancOne) - require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + cm := unstructuredFor(configMapGVK, cm1Name) + require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", cm1Name)) t.Cleanup(func() { - assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", configInstancOne)) + assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) - cm2 := unstructuredFor(configMapGVK, configInstancTwo) - require.NoError(t, c.Create(ctx, cm2), fmt.Sprintf("creating ConfigMap %s", configInstancTwo)) + cm2 := unstructuredFor(configMapGVK, cm2Name) + require.NoError(t, c.Create(ctx, cm2), fmt.Sprintf("creating ConfigMap %s", cm2Name)) t.Cleanup(func() { - assert.NoError(t, deleteResource(ctx, c, cm2), fmt.Sprintf("deleting resource %s", configInstancTwo)) + assert.NoError(t, deleteResource(ctx, c, cm2), fmt.Sprintf("deleting resource %s", cm2Name)) }) - pod := unstructuredFor(podGVK, podInstanceOne) - require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", podInstanceOne)) + pod := unstructuredFor(podGVK, pod1Name) + require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", pod1Name)) t.Cleanup(func() { - assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", podInstanceOne)) + assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", pod1Name)) }) cfClient, ok := dataStore.(*cachemanager.FakeCfClient) @@ -195,31 +159,29 @@ func TestCacheManager_concurrent(t *testing.T) { // and removing sync sources, all from different go routines. for i := 1; i < 100; i++ { wg.Add(3) + + // add some jitter between go func calls + time.Sleep(time.Duration(r.Intn(jitterUpperBound)) * time.Millisecond) go func() { defer wg.Done() - // add some jitter - time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) }() + + time.Sleep(time.Duration(r.Intn(jitterUpperBound)) * time.Millisecond) go func() { defer wg.Done() - // add some jitter - time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) }() + + time.Sleep(time.Duration(r.Intn(jitterUpperBound)) * time.Millisecond) go func() { defer wg.Done() - time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - - time.Sleep(time.Duration(r.Intn(1000)) * time.Millisecond) require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) }() } @@ -231,9 +193,9 @@ func TestCacheManager_concurrent(t *testing.T) { require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, - {Gvk: configMapGVK, Key: nsedConfigInstanceTwo}: nil, - {Gvk: podGVK, Key: nsedPodInstanceOne}: nil, + {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, + {Gvk: configMapGVK, Key: namespacedCm2Name}: nil, + {Gvk: podGVK, Key: namespacedPod1Name}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -248,20 +210,10 @@ func TestCacheManager_concurrent(t *testing.T) { _, foundPod := gvks[podGVK] require.True(t, foundPod) - // now remove the sources - wg.Add(2) - go func() { - defer wg.Done() - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) - }() - go func() { - defer wg.Done() - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - }() - - wg.Wait() + // do a final remove and expect the cache to clear + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) + require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - // and expect an empty cache and empty aggregator require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) require.True(t, len(agg.GVKs()) == 0) } @@ -280,27 +232,27 @@ func TestCacheManager_instance_updates(t *testing.T) { cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) - cm := unstructuredFor(configMapGVK, configInstancOne) - require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", configInstancOne)) + cm := unstructuredFor(configMapGVK, cm1Name) + require.NoError(t, c.Create(ctx, cm), fmt.Sprintf("creating ConfigMap %s", cm1Name)) t.Cleanup(func() { - assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", configInstancOne)) + assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: nsedConfigInstanceOne}: nil, + {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) - cm2 := unstructuredFor(configMapGVK, configInstancOne) - cm2.SetLabels(map[string]string{"testlabel": "test"}) // trigger an instance update - require.NoError(t, c.Update(ctx, cm2)) + cmUpdate := unstructuredFor(configMapGVK, cm1Name) + cmUpdate.SetLabels(map[string]string{"testlabel": "test"}) // trigger an instance update + require.NoError(t, c.Update(ctx, cmUpdate)) require.Eventually(t, func() bool { - instance := cfClient.GetData(cachemanager.CfDataKey{Gvk: configMapGVK, Key: nsedConfigInstanceOne}) + instance := cfClient.GetData(cachemanager.CfDataKey{Gvk: configMapGVK, Key: namespacedCm1Name}) unInstance, ok := instance.(*unstructured.Unstructured) require.True(t, ok) @@ -378,7 +330,7 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea require.NoError(t, err) aggregator := aggregator.NewGVKAggregator() - cfg := &cachemanager.Config{ + config := &cachemanager.Config{ CfClient: cfClient, SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, @@ -388,7 +340,7 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea Reader: reader, GVKAggregator: aggregator, } - cacheManager, err := cachemanager.NewCacheManager(cfg) + cacheManager, err := cachemanager.NewCacheManager(config) require.NoError(t, err) syncAdder := syncc.Adder{ diff --git a/pkg/fakes/reader.go b/pkg/fakes/reader.go index 0bfd88b285e..2b52c21da01 100644 --- a/pkg/fakes/reader.go +++ b/pkg/fakes/reader.go @@ -2,6 +2,7 @@ package fakes import ( "context" + "sync" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -17,3 +18,41 @@ func (r SpyReader) List(ctx context.Context, list client.ObjectList, opts ...cli } return r.Reader.List(ctx, list, opts...) } + +type FailureInjector struct { + mu sync.Mutex + failures map[string]int // registers GVK.Kind and how many times to fail +} + +func (f *FailureInjector) SetFailures(kind string, failures int) { + f.mu.Lock() + defer f.mu.Unlock() + + f.failures[kind] = failures +} + +// CheckFailures looks at the count of failures and returns true +// if there are still failures for the kind to consume, false otherwise. +func (f *FailureInjector) CheckFailures(kind string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + v, ok := f.failures[kind] + if !ok { + return false + } + + if v == 0 { + return false + } + + f.failures[kind] = v - 1 + + return true +} + +func NewFailureInjector() *FailureInjector { + return &FailureInjector{ + failures: make(map[string]int), + } +} From fbeb772691024177dbbdbeab72398232e769269e Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 9 Aug 2023 18:56:12 +0000 Subject: [PATCH 46/58] review: export KeyFor Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../cachemanager_integration_test.go | 37 ++++++++++++------- pkg/cachemanager/fakecfdataclient.go | 8 ++-- .../config/config_controller_test.go | 22 +++++------ 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index eacb704377f..b38f67d44a8 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -44,13 +44,10 @@ var ( configMapGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} podGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - cm1Name = "config-test-1" - cm2Name = "config-test-2" - namespacedCm1Name = "default/config-test-1" - namespacedCm2Name = "default/config-test-2" + cm1Name = "config-test-1" + cm2Name = "config-test-2" - pod1Name = "pod-test-1" - namespacedPod1Name = "default/pod-test-1" + pod1Name = "pod-test-1" ) func TestMain(m *testing.M) { @@ -87,19 +84,23 @@ func TestCacheManager_replay_retries(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) + cmKey, err := cachemanager.KeyFor(cm) + require.NoError(t, err) pod := unstructuredFor(podGVK, pod1Name) require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", pod1Name)) t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", pod1Name)) }) + podKey, err := cachemanager.KeyFor(pod) + require.NoError(t, err) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, - {Gvk: podGVK, Key: namespacedPod1Name}: nil, + cmKey: nil, + podKey: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -110,7 +111,7 @@ func TestCacheManager_replay_retries(t *testing.T) { require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected2 := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, + cmKey: nil, } require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) } @@ -134,18 +135,24 @@ func TestCacheManager_concurrent(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) + cmKey, err := cachemanager.KeyFor(cm) + require.NoError(t, err) cm2 := unstructuredFor(configMapGVK, cm2Name) require.NoError(t, c.Create(ctx, cm2), fmt.Sprintf("creating ConfigMap %s", cm2Name)) t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm2), fmt.Sprintf("deleting resource %s", cm2Name)) }) + cm2Key, err := cachemanager.KeyFor(cm2) + require.NoError(t, err) pod := unstructuredFor(podGVK, pod1Name) require.NoError(t, c.Create(ctx, pod), fmt.Sprintf("creating Pod %s", pod1Name)) t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", pod1Name)) }) + podKey, err := cachemanager.KeyFor(pod) + require.NoError(t, err) cfClient, ok := dataStore.(*cachemanager.FakeCfClient) require.True(t, ok) @@ -193,9 +200,9 @@ func TestCacheManager_concurrent(t *testing.T) { require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, - {Gvk: configMapGVK, Key: namespacedCm2Name}: nil, - {Gvk: podGVK, Key: namespacedPod1Name}: nil, + cmKey: nil, + cm2Key: nil, + podKey: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -237,12 +244,14 @@ func TestCacheManager_instance_updates(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) + cmKey, err := cachemanager.KeyFor(cm) + require.NoError(t, err) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: namespacedCm1Name}: nil, + cmKey: nil, } require.Eventually(t, expectedCheck(cfClient, expected), eventuallyTimeout, eventuallyTicker) @@ -252,7 +261,7 @@ func TestCacheManager_instance_updates(t *testing.T) { require.NoError(t, c.Update(ctx, cmUpdate)) require.Eventually(t, func() bool { - instance := cfClient.GetData(cachemanager.CfDataKey{Gvk: configMapGVK, Key: namespacedCm1Name}) + instance := cfClient.GetData(cmKey) unInstance, ok := instance.(*unstructured.Unstructured) require.True(t, ok) diff --git a/pkg/cachemanager/fakecfdataclient.go b/pkg/cachemanager/fakecfdataclient.go index d0bf42428b9..049a4c119a4 100644 --- a/pkg/cachemanager/fakecfdataclient.go +++ b/pkg/cachemanager/fakecfdataclient.go @@ -38,9 +38,9 @@ type FakeCfClient struct { var _ CFDataClient = &FakeCfClient{} -// keyFor returns a cfDataKey for the provided resource. +// KeyFor returns a CfDataKey for the provided resource. // Returns error if the resource is not a runtime.Object w/ metadata. -func (f *FakeCfClient) keyFor(obj interface{}) (CfDataKey, error) { +func KeyFor(obj interface{}) (CfDataKey, error) { o, ok := obj.(client.Object) if !ok { return CfDataKey{}, fmt.Errorf("expected runtime.Object, got: %T", obj) @@ -62,7 +62,7 @@ func (f *FakeCfClient) AddData(ctx context.Context, data interface{}) (*constrai return nil, fmt.Errorf("test error") } - key, err := f.keyFor(data) + key, err := KeyFor(data) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func (f *FakeCfClient) RemoveData(ctx context.Context, data interface{}) (*const return &constraintTypes.Responses{}, nil } - key, err := f.keyFor(data) + key, err := KeyFor(data) if err != nil { return nil, err } diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 71411f8af02..02305bc3242 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -498,6 +498,8 @@ func TestConfig_CacheContents(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), "deleting configMap config-test-1") }) + cmKey, err := cachemanager.KeyFor(cm) + require.NoError(t, err) cm2 := unstructuredFor(configMapGVK, "config-test-2") cm2.SetNamespace("kube-system") @@ -505,6 +507,8 @@ func TestConfig_CacheContents(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm2), "deleting configMap config-test-2") }) + cm2Key, err := cachemanager.KeyFor(cm2) + require.NoError(t, err) tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) @@ -530,8 +534,8 @@ func TestConfig_CacheContents(t *testing.T) { require.NoError(t, c.Create(ctx, config), "creating Config config") expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: nsGVK, Key: "default"}: nil, - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + {Gvk: nsGVK, Key: "default"}: nil, + cmKey: nil, // kube-system namespace is being excluded, it should not be in opa cache } g.Eventually(func() bool { @@ -555,20 +559,14 @@ func TestConfig_CacheContents(t *testing.T) { // Expect our configMap to return at some point // TODO: In the future it will remain instead of having to repopulate. expected = map[cachemanager.CfDataKey]interface{}{ - { - Gvk: configMapGVK, - Key: "default/config-test-1", - }: nil, + cmKey: nil, } g.Eventually(func() bool { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "waiting for ConfigMap to repopulate in cache") expected = map[cachemanager.CfDataKey]interface{}{ - { - Gvk: configMapGVK, - Key: "kube-system/config-test-2", - }: nil, + cm2Key: nil, } g.Eventually(func() bool { return !opaClient.Contains(expected) @@ -701,9 +699,11 @@ func TestConfig_Retries(t *testing.T) { t.Error(err) } }() + cmKey, err := cachemanager.KeyFor(cm) + require.NoError(t, err) expected := map[cachemanager.CfDataKey]interface{}{ - {Gvk: configMapGVK, Key: "default/config-test-1"}: nil, + cmKey: nil, } g.Eventually(func() bool { return opaClient.Contains(expected) From 2b0ff42557a577f1ecaf3acd31dc25045730fe7c Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 9 Aug 2023 19:04:18 +0000 Subject: [PATCH 47/58] review: docstring for FailureInjector Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/fakes/reader.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/fakes/reader.go b/pkg/fakes/reader.go index 2b52c21da01..2acd81d19da 100644 --- a/pkg/fakes/reader.go +++ b/pkg/fakes/reader.go @@ -19,6 +19,8 @@ func (r SpyReader) List(ctx context.Context, list client.ObjectList, opts ...cli return r.Reader.List(ctx, list, opts...) } +// FailureInjector can be used in combination with the SpyReader to simulate transient +// failures for network calls. type FailureInjector struct { mu sync.Mutex failures map[string]int // registers GVK.Kind and how many times to fail From f27e91ff606a41c537869ebdd10a408fa1da5014 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 10 Aug 2023 20:48:16 +0000 Subject: [PATCH 48/58] review: use assert in go funcs - comments - directly call AddObject - call cancelFunc next to creation Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 20 ++--- pkg/cachemanager/cachemanager_test.go | 4 +- .../cachemanager_integration_test.go | 21 +++-- .../config/config_controller_test.go | 78 ++++++++++--------- 4 files changed, 60 insertions(+), 63 deletions(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 25c291912c0..260096d6123 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -111,8 +111,7 @@ func (c *CacheManager) Start(ctx context.Context) error { } // UpsertSource adjusts the watched set of gvks according to the newGVKs passed in -// for a given sourceKey. -// Callers are responsible for retrying on error. +// for a given sourceKey. Callers are responsible for retrying on error. func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Key, newGVKs []schema.GroupVersionKind) error { c.mu.Lock() defer c.mu.Unlock() @@ -139,8 +138,7 @@ func (c *CacheManager) UpsertSource(ctx context.Context, sourceKey aggregator.Ke } // replaceWatchSet looks at the gvksToSync and makes changes to the registrar's watch set. -// assumes caller has lock. -// On error, actual watch state may not align with intended watch state. +// Assumes caller has lock. On error, actual watch state may not align with intended watch state. func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() newWatchSet.Add(c.gvksToSync.GVKs()...) @@ -159,8 +157,7 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } -// RemoveSource removes the watches of the GVKs for a given aggregator.Key. -// Callers are responsible for retrying on error. +// RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() defer c.mu.Unlock() @@ -241,12 +238,8 @@ func (c *CacheManager) AddObject(ctx context.Context, instance *unstructured.Uns } func (c *CacheManager) RemoveObject(ctx context.Context, instance *unstructured.Unstructured) error { - gvk := instance.GroupVersionKind() - - if c.watchesGVK(gvk) { - if _, err := c.cfClient.RemoveData(ctx, instance); err != nil { - return err - } + if _, err := c.cfClient.RemoveData(ctx, instance); err != nil { + return err } // only delete from metrics map if the data removal was successful @@ -383,8 +376,7 @@ func (c *CacheManager) replayGVKs(ctx context.Context, gvksToRelist []schema.Gro // wipeCacheIfNeeded performs a cache wipe if there are any gvks needing to be removed // from the cache or if the excluder has changed. It also marks which gvks need to be -// re listed again in the cf data cache after the wipe. -// assumes the caller has lock. +// re listed again in the cf data cache after the wipe. Assumes the caller has lock. func (c *CacheManager) wipeCacheIfNeeded(ctx context.Context) { // remove any gvks not needing to be synced anymore // or re evaluate all if the excluder changed. diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index fc1f7ae7805..3bd5e10e247 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -305,7 +305,7 @@ func TestCacheManager_RemoveObject(t *testing.T) { expectSyncMetric: false, }, { - name: "RemoveObject has no effect if GVK is not watched", + name: "RemoveObject succeeds even if GVK is not watched", cm: &CacheManager{ cfClient: makeDataClient(), watchedSet: watch.NewSet(), @@ -313,7 +313,7 @@ func TestCacheManager_RemoveObject(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), processExcluder: process.Get(), }, - expectedData: map[CfDataKey]interface{}{{Gvk: pod.GroupVersionKind(), Key: "test-ns/test-name"}: nil}, + expectedData: map[CfDataKey]interface{}{}, expectSyncMetric: false, }, { diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index b38f67d44a8..e451ed10b68 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -172,24 +172,24 @@ func TestCacheManager_concurrent(t *testing.T) { go func() { defer wg.Done() - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) + assert.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) + assert.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) }() time.Sleep(time.Duration(r.Intn(jitterUpperBound)) * time.Millisecond) go func() { defer wg.Done() - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) - require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) + assert.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{podGVK})) + assert.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{configMapGVK})) }() time.Sleep(time.Duration(r.Intn(jitterUpperBound)) * time.Millisecond) go func() { defer wg.Done() - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) + assert.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) + assert.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) }() } @@ -321,6 +321,9 @@ type testResources struct { func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, reader client.Reader) (testResources, context.Context) { ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancelFunc() + }) cfClient := &cachemanager.FakeCfClient{} tracker, err := readiness.SetupTracker(mgr, false, false, false) @@ -358,14 +361,10 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea } require.NoError(t, syncAdder.Add(mgr), "registering sync controller") go func() { - require.NoError(t, cacheManager.Start(ctx)) + assert.NoError(t, cacheManager.Start(ctx)) }() testutils.StartManager(ctx, t, mgr) - t.Cleanup(func() { - cancelFunc() - }) - return testResources{cacheManager, cfClient, aggregator}, ctx } diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 02305bc3242..2e1ef3e0fbe 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -97,6 +97,14 @@ func setupManager(t *testing.T) (manager.Manager, *watch.Manager) { func TestReconcile(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) g := gomega.NewGomegaWithT(t) + once := gosync.Once{} + testMgrStopped := func() { + once.Do(func() { + cancelFunc() + }) + } + + defer testMgrStopped() instance := &configv1alpha1.Config{ ObjectMeta: metav1.ObjectMeta{ @@ -157,7 +165,7 @@ func TestReconcile(t *testing.T) { // start the cache manager go func() { - require.NoError(t, cacheManager.Start(ctx)) + assert.NoError(t, cacheManager.Start(ctx)) }() rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) @@ -167,14 +175,6 @@ func TestReconcile(t *testing.T) { require.NoError(t, add(mgr, recFn)) testutils.StartManager(ctx, t, mgr) - once := gosync.Once{} - testMgrStopped := func() { - once.Do(func() { - cancelFunc() - }) - } - - defer testMgrStopped() // Create the Config object and expect the Reconcile to be created err = c.Create(ctx, instance) @@ -284,12 +284,14 @@ func TestReconcile(t *testing.T) { }, }, } - require.NoError(t, c.Create(ctx, fooPod)) - // fooPod should be namespace excluded, hence not synced - g.Eventually(opaClient.Contains(map[cachemanager.CfDataKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}}), timeout).ShouldNot(gomega.BeTrue()) - require.NoError(t, c.Delete(ctx, fooPod)) - testMgrStopped() + // directly call cacheManager to avoid any race condition + // between adding the pod and the sync_controller calling AddObject + require.NoError(t, cacheManager.AddObject(ctx, fooPod)) + + // fooPod should be namespace excluded, hence not added to the cache + require.False(t, opaClient.Contains(map[cachemanager.CfDataKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}})) + cs.Stop() } @@ -319,7 +321,15 @@ func TestConfig_DeleteSyncResources(t *testing.T) { }, }, } + ctx, cancelFunc := context.WithCancel(context.Background()) + once := gosync.Once{} + defer func() { + once.Do(func() { + cancelFunc() + }) + }() + err := c.Create(ctx, instance) if err != nil { t.Fatal(err) @@ -367,12 +377,7 @@ func TestConfig_DeleteSyncResources(t *testing.T) { // start manager that will start tracker and controller testutils.StartManager(ctx, t, mgr) - once := gosync.Once{} - defer func() { - once.Do(func() { - cancelFunc() - }) - }() + gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} // get the object tracker for the synconly pod resource @@ -477,6 +482,14 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager // Verify the Opa cache is populated based on the config resource. func TestConfig_CacheContents(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) + once := gosync.Once{} + testMgrStopped := func() { + once.Do(func() { + cancelFunc() + }) + } + defer testMgrStopped() + // Setup the Manager and Controller. mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) @@ -521,13 +534,6 @@ func TestConfig_CacheContents(t *testing.T) { require.True(t, ok) testutils.StartManager(ctx, t, mgr) - once := gosync.Once{} - testMgrStopped := func() { - once.Do(func() { - cancelFunc() - }) - } - defer testMgrStopped() // Create the Config object and expect the Reconcile to be created config := configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) @@ -586,6 +592,14 @@ func TestConfig_CacheContents(t *testing.T) { func TestConfig_Retries(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) + once := gosync.Once{} + testMgrStopped := func() { + once.Do(func() { + cancelFunc() + }) + } + defer testMgrStopped() + g := gomega.NewGomegaWithT(t) nsGVK := schema.GroupVersionKind{ Group: "", @@ -630,7 +644,7 @@ func TestConfig_Retries(t *testing.T) { }) require.NoError(t, err) go func() { - require.NoError(t, cacheManager.Start(ctx)) + assert.NoError(t, cacheManager.Start(ctx)) }() rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) @@ -663,14 +677,6 @@ func TestConfig_Retries(t *testing.T) { } testutils.StartManager(ctx, t, mgr) - once := gosync.Once{} - testMgrStopped := func() { - once.Do(func() { - cancelFunc() - }) - } - - defer testMgrStopped() // Create the Config object and expect the Reconcile to be created g.Eventually(func() error { From 605866f4050268f6ddde549cdd3a63a413d6bfef Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:28:45 +0000 Subject: [PATCH 49/58] review, refactor: clean up dep injection in config_c Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller.go | 28 ++++--------------- .../config/config_controller_test.go | 6 ++-- pkg/controller/controller.go | 14 ---------- 3 files changed, 9 insertions(+), 39 deletions(-) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index f8145376875..3a5243b6609 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -51,19 +51,15 @@ const ( var log = logf.Log.WithName("controller").WithValues("kind", "Config") type Adder struct { - Opa *constraintclient.Client - WatchManager *watch.Manager ControllerSwitch *watch.ControllerSwitch Tracker *readiness.Tracker - ProcessExcluder *process.Excluder - WatchSet *watch.Set CacheManager *cm.CacheManager } // Add creates a new ConfigController and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func (a *Adder) Add(mgr manager.Manager) error { - r, err := newReconciler(mgr, a.CacheManager, a.WatchManager, a.ControllerSwitch, a.Tracker, a.ProcessExcluder, a.WatchSet) + r, err := newReconciler(mgr, a.CacheManager, a.ControllerSwitch, a.Tracker) if err != nil { return err } @@ -71,13 +67,9 @@ func (a *Adder) Add(mgr manager.Manager) error { return add(mgr, r) } -func (a *Adder) InjectOpa(o *constraintclient.Client) { - a.Opa = o -} +func (a *Adder) InjectOpa(_ *constraintclient.Client) {} -func (a *Adder) InjectWatchManager(wm *watch.Manager) { - a.WatchManager = wm -} +func (a *Adder) InjectWatchManager(_ *watch.Manager) {} func (a *Adder) InjectControllerSwitch(cs *watch.ControllerSwitch) { a.ControllerSwitch = cs @@ -87,28 +79,20 @@ func (a *Adder) InjectTracker(t *readiness.Tracker) { a.Tracker = t } -func (a *Adder) InjectProcessExcluder(m *process.Excluder) { - a.ProcessExcluder = m -} - func (a *Adder) InjectMutationSystem(mutationSystem *mutation.System) {} func (a *Adder) InjectExpansionSystem(expansionSystem *expansion.System) {} func (a *Adder) InjectProviderCache(providerCache *externaldata.ProviderCache) {} -func (a *Adder) InjectWatchSet(watchSet *watch.Set) { - a.WatchSet = watchSet -} - func (a *Adder) InjectCacheManager(cm *cm.CacheManager) { a.CacheManager = cm } // newReconciler returns a new reconcile.Reconciler. -func newReconciler(mgr manager.Manager, cm *cm.CacheManager, wm *watch.Manager, cs *watch.ControllerSwitch, tracker *readiness.Tracker, processExcluder *process.Excluder, watchSet *watch.Set) (*ReconcileConfig, error) { - if watchSet == nil { - return nil, fmt.Errorf("watchSet must be non-nil") +func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.ControllerSwitch, tracker *readiness.Tracker) (*ReconcileConfig, error) { + if cm == nil { + return nil, fmt.Errorf("cacheManager must be non-nil") } return &ReconcileConfig{ diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 2e1ef3e0fbe..4932561ae40 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -168,7 +168,7 @@ func TestReconcile(t *testing.T) { assert.NoError(t, cacheManager.Start(ctx)) }() - rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) + rec, err := newReconciler(mgr, cacheManager, cs, tracker) require.NoError(t, err) recFn, requests := SetupTestReconcile(rec) @@ -459,7 +459,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager _ = cacheManager.Start(ctx) }() - rec, err := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) + rec, err := newReconciler(mgr, cacheManager, cs, tracker) if err != nil { return nil, fmt.Errorf("creating reconciler: %w", err) } @@ -647,7 +647,7 @@ func TestConfig_Retries(t *testing.T) { assert.NoError(t, cacheManager.Start(ctx)) }() - rec, _ := newReconciler(mgr, cacheManager, wm, cs, tracker, processExcluder, watchSet) + rec, _ := newReconciler(mgr, cacheManager, cs, tracker) err = add(mgr, rec) if err != nil { t.Fatal(err) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 889d97015e5..88c2a85c0d0 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -63,14 +63,6 @@ type GetPodInjector interface { InjectGetPod(func(context.Context) (*corev1.Pod, error)) } -type GetProcessExcluderInjector interface { - InjectProcessExcluder(processExcluder *process.Excluder) -} - -type WatchSetInjector interface { - InjectWatchSet(watchSet *watch.Set) -} - type PubsubInjector interface { InjectPubsubSystem(pubsubSystem *pubsub.System) } @@ -220,12 +212,6 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { if a2, ok := a.(GetPodInjector); ok { a2.InjectGetPod(deps.GetPod) } - if a2, ok := a.(GetProcessExcluderInjector); ok { - a2.InjectProcessExcluder(deps.ProcessExcluder) - } - if a2, ok := a.(WatchSetInjector); ok { - a2.InjectWatchSet(deps.WatchSet) - } if a2, ok := a.(PubsubInjector); ok { a2.InjectPubsubSystem(deps.PubsubSystem) } From 4c3cbf8b98964b4fa7fd004a5e98512db0f7bd9d Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:29:52 +0000 Subject: [PATCH 50/58] add logging to test controller mgr Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- test/testutils/manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/testutils/manager.go b/test/testutils/manager.go index 2e5641f3a5f..8bfb377ea06 100644 --- a/test/testutils/manager.go +++ b/test/testutils/manager.go @@ -8,7 +8,9 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -40,6 +42,7 @@ func StartManager(ctx context.Context, t *testing.T, mgr manager.Manager) { func SetupManager(t *testing.T, cfg *rest.Config) (manager.Manager, *watch.Manager) { t.Helper() + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) metrics.Registry = prometheus.NewRegistry() mgr, err := manager.New(cfg, manager.Options{ MetricsBindAddress: "0", From d1f146f09b4c04811bb189541ac037568c90d037 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 10 Aug 2023 23:08:07 +0000 Subject: [PATCH 51/58] review: move FakeCfClient to pkg/fakes Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager_test.go | 48 +++++++++---------- .../cachemanager_integration_test.go | 32 ++++++------- .../config/config_controller_test.go | 24 +++++----- .../fakecfdataclient.go | 4 +- 4 files changed, 53 insertions(+), 55 deletions(-) rename pkg/{cachemanager => fakes}/fakecfdataclient.go (98%) diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 3bd5e10e247..6bd4672ef84 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -35,7 +35,7 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { ctx, cancelFunc := context.WithCancel(context.Background()) - cfClient := &FakeCfClient{} + cfClient := &fakes.FakeCfClient{} tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) processExcluder := process.Get() @@ -75,7 +75,7 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { configMapGVK := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} dataClientForTest := func() CFDataClient { - cfdc := &FakeCfClient{} + cfdc := &fakes.FakeCfClient{} cm := unstructuredFor(configMapGVK, "config-test-1") _, err := cfdc.AddData(context.Background(), cm) @@ -88,7 +88,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { tcs := []struct { name string cm *CacheManager - expectedData map[CfDataKey]interface{} + expectedData map[fakes.CfDataKey]interface{} }{ { name: "wipe cache if there are gvks to remove", @@ -101,7 +101,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { }(), syncMetricsCache: syncutil.NewMetricsCache(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, }, { name: "wipe cache if there are excluder changes", @@ -111,7 +111,7 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), gvksToDeleteFromCache: watch.NewSet(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, }, { name: "don't wipe cache if no excluder changes or no gvks to delete", @@ -120,13 +120,13 @@ func TestCacheManager_wipeCacheIfNeeded(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), gvksToDeleteFromCache: watch.NewSet(), }, - expectedData: map[CfDataKey]interface{}{{Gvk: configMapGVK, Key: "default/config-test-1"}: nil}, + expectedData: map[fakes.CfDataKey]interface{}{{Gvk: configMapGVK, Key: "default/config-test-1"}: nil}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - cfClient, ok := tc.cm.cfClient.(*FakeCfClient) + cfClient, ok := tc.cm.cfClient.(*fakes.FakeCfClient) require.True(t, ok) tc.cm.wipeCacheIfNeeded(context.Background()) @@ -151,12 +151,12 @@ func TestCacheManager_AddObject(t *testing.T) { cm *CacheManager expectSyncMetric bool expectedMetricStatus metrics.Status - expectedData map[CfDataKey]interface{} + expectedData map[fakes.CfDataKey]interface{} }{ { name: "AddObject happy path", cm: &CacheManager{ - cfClient: &FakeCfClient{}, + cfClient: &fakes.FakeCfClient{}, watchedSet: func() *watch.Set { ws := watch.NewSet() ws.Add(pod.GroupVersionKind()) @@ -167,26 +167,26 @@ func TestCacheManager_AddObject(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), processExcluder: process.Get(), }, - expectedData: map[CfDataKey]interface{}{{Gvk: pod.GroupVersionKind(), Key: "test-ns/test-name"}: nil}, + expectedData: map[fakes.CfDataKey]interface{}{{Gvk: pod.GroupVersionKind(), Key: "test-ns/test-name"}: nil}, expectSyncMetric: true, expectedMetricStatus: metrics.ActiveStatus, }, { name: "AddObject has no effect if GVK is not watched", cm: &CacheManager{ - cfClient: &FakeCfClient{}, + cfClient: &fakes.FakeCfClient{}, watchedSet: watch.NewSet(), tracker: readiness.NewTracker(mgr.GetAPIReader(), false, false, false), syncMetricsCache: syncutil.NewMetricsCache(), processExcluder: process.Get(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, expectSyncMetric: false, }, { name: "AddObject has no effect if GVK is process excluded", cm: &CacheManager{ - cfClient: &FakeCfClient{}, + cfClient: &fakes.FakeCfClient{}, watchedSet: func() *watch.Set { ws := watch.NewSet() ws.Add(pod.GroupVersionKind()) @@ -206,14 +206,14 @@ func TestCacheManager_AddObject(t *testing.T) { return processExcluder }(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, expectSyncMetric: false, }, { name: "AddObject sets metrics on error from cfdataclient", cm: &CacheManager{ cfClient: func() CFDataClient { - c := &FakeCfClient{} + c := &fakes.FakeCfClient{} c.SetErroring(true) return c }(), @@ -227,7 +227,7 @@ func TestCacheManager_AddObject(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), processExcluder: process.Get(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, expectSyncMetric: true, expectedMetricStatus: metrics.ErrorStatus, }, @@ -245,10 +245,10 @@ func TestCacheManager_AddObject(t *testing.T) { } } -func assertExpecations(t *testing.T, cm *CacheManager, instance *unstructured.Unstructured, expectedData map[CfDataKey]interface{}, expectSyncMetric bool, expectedMetricStatus *metrics.Status) { +func assertExpecations(t *testing.T, cm *CacheManager, instance *unstructured.Unstructured, expectedData map[fakes.CfDataKey]interface{}, expectSyncMetric bool, expectedMetricStatus *metrics.Status) { t.Helper() - cfClient, ok := cm.cfClient.(*FakeCfClient) + cfClient, ok := cm.cfClient.(*fakes.FakeCfClient) require.True(t, ok) require.True(t, cfClient.Contains(expectedData)) @@ -273,8 +273,8 @@ func TestCacheManager_RemoveObject(t *testing.T) { mgr, _ := testutils.SetupManager(t, cfg) tracker := readiness.NewTracker(mgr.GetAPIReader(), false, false, false) - makeDataClient := func() *FakeCfClient { - c := &FakeCfClient{} + makeDataClient := func() *fakes.FakeCfClient { + c := &fakes.FakeCfClient{} _, err := c.AddData(context.Background(), &unstructured.Unstructured{Object: unstructuredPod}) require.NoError(t, err) @@ -285,7 +285,7 @@ func TestCacheManager_RemoveObject(t *testing.T) { name string cm *CacheManager expectSyncMetric bool - expectedData map[CfDataKey]interface{} + expectedData map[fakes.CfDataKey]interface{} }{ { name: "RemoveObject happy path", @@ -301,7 +301,7 @@ func TestCacheManager_RemoveObject(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), processExcluder: process.Get(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, expectSyncMetric: false, }, { @@ -313,7 +313,7 @@ func TestCacheManager_RemoveObject(t *testing.T) { syncMetricsCache: syncutil.NewMetricsCache(), processExcluder: process.Get(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, expectSyncMetric: false, }, { @@ -339,7 +339,7 @@ func TestCacheManager_RemoveObject(t *testing.T) { return processExcluder }(), }, - expectedData: map[CfDataKey]interface{}{}, + expectedData: map[fakes.CfDataKey]interface{}{}, expectSyncMetric: false, }, } diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index e451ed10b68..c04aef5efca 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -76,7 +76,7 @@ func TestCacheManager_replay_retries(t *testing.T) { cacheManager := testResources.CacheManager dataStore := testResources.CFDataClient - cfClient, ok := dataStore.(*cachemanager.FakeCfClient) + cfClient, ok := dataStore.(*fakes.FakeCfClient) require.True(t, ok) cm := unstructuredFor(configMapGVK, cm1Name) @@ -84,7 +84,7 @@ func TestCacheManager_replay_retries(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) - cmKey, err := cachemanager.KeyFor(cm) + cmKey, err := fakes.KeyFor(cm) require.NoError(t, err) pod := unstructuredFor(podGVK, pod1Name) @@ -92,13 +92,13 @@ func TestCacheManager_replay_retries(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", pod1Name)) }) - podKey, err := cachemanager.KeyFor(pod) + podKey, err := fakes.KeyFor(pod) require.NoError(t, err) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK, podGVK})) - expected := map[cachemanager.CfDataKey]interface{}{ + expected := map[fakes.CfDataKey]interface{}{ cmKey: nil, podKey: nil, } @@ -110,7 +110,7 @@ func TestCacheManager_replay_retries(t *testing.T) { // this call should schedule a cache wipe and a replay for the configMapGVK require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected2 := map[cachemanager.CfDataKey]interface{}{ + expected2 := map[fakes.CfDataKey]interface{}{ cmKey: nil, } require.Eventually(t, expectedCheck(cfClient, expected2), eventuallyTimeout, eventuallyTicker) @@ -135,7 +135,7 @@ func TestCacheManager_concurrent(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) - cmKey, err := cachemanager.KeyFor(cm) + cmKey, err := fakes.KeyFor(cm) require.NoError(t, err) cm2 := unstructuredFor(configMapGVK, cm2Name) @@ -143,7 +143,7 @@ func TestCacheManager_concurrent(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm2), fmt.Sprintf("deleting resource %s", cm2Name)) }) - cm2Key, err := cachemanager.KeyFor(cm2) + cm2Key, err := fakes.KeyFor(cm2) require.NoError(t, err) pod := unstructuredFor(podGVK, pod1Name) @@ -151,10 +151,10 @@ func TestCacheManager_concurrent(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, pod), fmt.Sprintf("deleting resource %s", pod1Name)) }) - podKey, err := cachemanager.KeyFor(pod) + podKey, err := fakes.KeyFor(pod) require.NoError(t, err) - cfClient, ok := dataStore.(*cachemanager.FakeCfClient) + cfClient, ok := dataStore.(*fakes.FakeCfClient) require.True(t, ok) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} @@ -199,7 +199,7 @@ func TestCacheManager_concurrent(t *testing.T) { require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceTwo, []schema.GroupVersionKind{podGVK})) - expected := map[cachemanager.CfDataKey]interface{}{ + expected := map[fakes.CfDataKey]interface{}{ cmKey: nil, cm2Key: nil, podKey: nil, @@ -221,7 +221,7 @@ func TestCacheManager_concurrent(t *testing.T) { require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceOne)) require.NoError(t, cacheManager.RemoveSource(ctx, syncSourceTwo)) - require.Eventually(t, expectedCheck(cfClient, map[cachemanager.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) + require.Eventually(t, expectedCheck(cfClient, map[fakes.CfDataKey]interface{}{}), eventuallyTimeout, eventuallyTicker) require.True(t, len(agg.GVKs()) == 0) } @@ -236,7 +236,7 @@ func TestCacheManager_instance_updates(t *testing.T) { cacheManager := testResources.CacheManager dataStore := testResources.CFDataClient - cfClient, ok := dataStore.(*cachemanager.FakeCfClient) + cfClient, ok := dataStore.(*fakes.FakeCfClient) require.True(t, ok) cm := unstructuredFor(configMapGVK, cm1Name) @@ -244,13 +244,13 @@ func TestCacheManager_instance_updates(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), fmt.Sprintf("deleting resource %s", cm1Name)) }) - cmKey, err := cachemanager.KeyFor(cm) + cmKey, err := fakes.KeyFor(cm) require.NoError(t, err) syncSourceOne := aggregator.Key{Source: "source_a", ID: "ID_a"} require.NoError(t, cacheManager.UpsertSource(ctx, syncSourceOne, []schema.GroupVersionKind{configMapGVK})) - expected := map[cachemanager.CfDataKey]interface{}{ + expected := map[fakes.CfDataKey]interface{}{ cmKey: nil, } @@ -282,7 +282,7 @@ func deleteResource(ctx context.Context, c client.Client, resounce *unstructured return err } -func expectedCheck(cfClient *cachemanager.FakeCfClient, expected map[cachemanager.CfDataKey]interface{}) func() bool { +func expectedCheck(cfClient *fakes.FakeCfClient, expected map[fakes.CfDataKey]interface{}) func() bool { return func() bool { if cfClient.Len() != len(expected) { return false @@ -325,7 +325,7 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea cancelFunc() }) - cfClient := &cachemanager.FakeCfClient{} + cfClient := &fakes.FakeCfClient{} tracker, err := readiness.SetupTracker(mgr, false, false, false) require.NoError(t, err) processExcluder := process.Get() diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 4932561ae40..cda2859e70b 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -136,7 +136,7 @@ func TestReconcile(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &cachemanager.FakeCfClient{} + opaClient := &fakes.FakeCfClient{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) @@ -290,7 +290,7 @@ func TestReconcile(t *testing.T) { require.NoError(t, cacheManager.AddObject(ctx, fooPod)) // fooPod should be namespace excluded, hence not added to the cache - require.False(t, opaClient.Contains(map[cachemanager.CfDataKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}})) + require.False(t, opaClient.Contains(map[fakes.CfDataKey]interface{}{{Gvk: fooPod.GroupVersionKind(), Key: "default"}: struct{}{}})) cs.Stop() } @@ -418,7 +418,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager // initialize OPA var opaClient cachemanager.CFDataClient if useFakeOpa { - opaClient = &cachemanager.FakeCfClient{} + opaClient = &fakes.FakeCfClient{} } else { driver, err := rego.New(rego.Tracing(true)) if err != nil { @@ -511,7 +511,7 @@ func TestConfig_CacheContents(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm), "deleting configMap config-test-1") }) - cmKey, err := cachemanager.KeyFor(cm) + cmKey, err := fakes.KeyFor(cm) require.NoError(t, err) cm2 := unstructuredFor(configMapGVK, "config-test-2") @@ -520,7 +520,7 @@ func TestConfig_CacheContents(t *testing.T) { t.Cleanup(func() { assert.NoError(t, deleteResource(ctx, c, cm2), "deleting configMap config-test-2") }) - cm2Key, err := cachemanager.KeyFor(cm2) + cm2Key, err := fakes.KeyFor(cm2) require.NoError(t, err) tracker, err := readiness.SetupTracker(mgr, false, false, false) @@ -530,7 +530,7 @@ func TestConfig_CacheContents(t *testing.T) { opa, err := setupController(ctx, mgr, wm, tracker, events, c, true) require.NoError(t, err, "failed to set up controller") - opaClient, ok := opa.(*cachemanager.FakeCfClient) + opaClient, ok := opa.(*fakes.FakeCfClient) require.True(t, ok) testutils.StartManager(ctx, t, mgr) @@ -539,7 +539,7 @@ func TestConfig_CacheContents(t *testing.T) { config := configFor([]schema.GroupVersionKind{nsGVK, configMapGVK}) require.NoError(t, c.Create(ctx, config), "creating Config config") - expected := map[cachemanager.CfDataKey]interface{}{ + expected := map[fakes.CfDataKey]interface{}{ {Gvk: nsGVK, Key: "default"}: nil, cmKey: nil, // kube-system namespace is being excluded, it should not be in opa cache @@ -564,14 +564,14 @@ func TestConfig_CacheContents(t *testing.T) { // Expect our configMap to return at some point // TODO: In the future it will remain instead of having to repopulate. - expected = map[cachemanager.CfDataKey]interface{}{ + expected = map[fakes.CfDataKey]interface{}{ cmKey: nil, } g.Eventually(func() bool { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "waiting for ConfigMap to repopulate in cache") - expected = map[cachemanager.CfDataKey]interface{}{ + expected = map[fakes.CfDataKey]interface{}{ cm2Key: nil, } g.Eventually(func() bool { @@ -617,7 +617,7 @@ func TestConfig_Retries(t *testing.T) { mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) - opaClient := &cachemanager.FakeCfClient{} + opaClient := &fakes.FakeCfClient{} cs := watch.NewSwitch() tracker, err := readiness.SetupTracker(mgr, false, false, false) if err != nil { @@ -705,10 +705,10 @@ func TestConfig_Retries(t *testing.T) { t.Error(err) } }() - cmKey, err := cachemanager.KeyFor(cm) + cmKey, err := fakes.KeyFor(cm) require.NoError(t, err) - expected := map[cachemanager.CfDataKey]interface{}{ + expected := map[fakes.CfDataKey]interface{}{ cmKey: nil, } g.Eventually(func() bool { diff --git a/pkg/cachemanager/fakecfdataclient.go b/pkg/fakes/fakecfdataclient.go similarity index 98% rename from pkg/cachemanager/fakecfdataclient.go rename to pkg/fakes/fakecfdataclient.go index 049a4c119a4..ba81b4fe255 100644 --- a/pkg/cachemanager/fakecfdataclient.go +++ b/pkg/fakes/fakecfdataclient.go @@ -11,7 +11,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package cachemanager +package fakes import ( "context" @@ -36,8 +36,6 @@ type FakeCfClient struct { needsToError bool } -var _ CFDataClient = &FakeCfClient{} - // KeyFor returns a CfDataKey for the provided resource. // Returns error if the resource is not a runtime.Object w/ metadata. func KeyFor(obj interface{}) (CfDataKey, error) { From a6c65bed39d0a0a4fb9f3869bb680f808101f57d Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Thu, 17 Aug 2023 23:34:41 +0000 Subject: [PATCH 52/58] review: make audit use cm Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 35 ++++++++++++++++-- pkg/audit/audit_cache_lister.go | 16 +++++---- pkg/cachemanager/cachemanager.go | 17 ++++++--- pkg/cachemanager/cachemanager_test.go | 1 - .../cachemanager_integration_test.go | 1 - pkg/controller/config/config_controller.go | 5 ++- .../config/config_controller_test.go | 12 ++----- pkg/controller/controller.go | 36 ++++--------------- pkg/readiness/ready_tracker_test.go | 30 ++++++++++++++-- 9 files changed, 92 insertions(+), 61 deletions(-) diff --git a/main.go b/main.go index 7669e5aec66..7b6d17e27fe 100644 --- a/main.go +++ b/main.go @@ -43,6 +43,7 @@ import ( mutationsv1beta1 "github.com/open-policy-agent/gatekeeper/v3/apis/mutations/v1beta1" statusv1beta1 "github.com/open-policy-agent/gatekeeper/v3/apis/status/v1beta1" "github.com/open-policy-agent/gatekeeper/v3/pkg/audit" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" @@ -52,6 +53,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/operations" "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/upgrade" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" @@ -69,6 +71,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" crzap "sigs.k8s.io/controller-runtime/pkg/log/zap" crWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -447,17 +450,43 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle // Setup all Controllers setupLog.Info("setting up controllers") - watchSet := watch.NewSet() + + // Events ch will be used to receive events from dynamic watches registered + // via the registrar below. + events := make(chan event.GenericEvent, 1024) + w, err := wm.NewRegistrar( + cachemanager.RegName, + events) + if err != nil { + setupLog.Error(err, "unable to set up watch registrar for cache manager") + return err + } + + syncMetricsCache := syncutil.NewMetricsCache() + cm, err := cachemanager.NewCacheManager(&cachemanager.Config{ + CfClient: client, + SyncMetricsCache: syncMetricsCache, + Tracker: tracker, + ProcessExcluder: processExcluder, + Registrar: w, + Reader: mgr.GetCache(), + }) + if err != nil { + setupLog.Error(err, "unable to create cache manager") + return err + } + opts := controller.Dependencies{ Opa: client, WatchManger: wm, + EventsCh: events, + CacheMgr: cm, ControllerSwitch: sw, Tracker: tracker, ProcessExcluder: processExcluder, MutationSystem: mutationSystem, ExpansionSystem: expansionSystem, ProviderCache: providerCache, - WatchSet: watchSet, PubsubSystem: pubsubSystem, } @@ -482,7 +511,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle if operations.IsAssigned(operations.Audit) { setupLog.Info("setting up audit") - auditCache := audit.NewAuditCacheLister(mgr.GetCache(), watchSet) + auditCache := audit.NewAuditCacheLister(mgr.GetCache(), cm) auditDeps := audit.Dependencies{ Client: client, ProcessExcluder: processExcluder, diff --git a/pkg/audit/audit_cache_lister.go b/pkg/audit/audit_cache_lister.go index 299cb00cd26..74bddf4cf9e 100644 --- a/pkg/audit/audit_cache_lister.go +++ b/pkg/audit/audit_cache_lister.go @@ -3,7 +3,6 @@ package audit import ( "context" - "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" @@ -11,10 +10,10 @@ import ( // NewAuditCacheLister instantiates a new AuditCache which will read objects in // watched from auditCache. -func NewAuditCacheLister(auditCache client.Reader, watched *watch.Set) *CacheLister { +func NewAuditCacheLister(auditCache client.Reader, lister Delegate) *CacheLister { return &CacheLister{ auditCache: auditCache, - watched: watched, + lister: lister, } } @@ -25,14 +24,19 @@ type CacheLister struct { // Caution: only to be read from while watched is locked, such as through // DoForEach. auditCache client.Reader - // watched is the set of objects watched by the audit cache. - watched *watch.Set + // lister is a delegate like cachemanager that we can use to query a watched set of GKVs. + lister Delegate +} + +// wraps DoForEach from a watch.Set. +type Delegate interface { + DoForEach(listFunc func(gvk schema.GroupVersionKind) error) error } // ListObjects lists all objects from the audit cache. func (l *CacheLister) ListObjects(ctx context.Context) ([]unstructured.Unstructured, error) { var objs []unstructured.Unstructured - err := l.watched.DoForEach(func(gvk schema.GroupVersionKind) error { + err := l.lister.DoForEach(func(gvk schema.GroupVersionKind) error { gvkObjects, err := listObjects(ctx, l.auditCache, gvk) if err != nil { return err diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 260096d6123..6cdd8ad0958 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -21,6 +21,8 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) +const RegName = "cachemanager" + var ( log = logf.Log.WithName("cache-manager") backoff = wait.Backoff{ @@ -37,7 +39,6 @@ type Config struct { Tracker *readiness.Tracker ProcessExcluder *process.Excluder Registrar *watch.Registrar - WatchedSet *watch.Set GVKAggregator *aggregator.GVKAgreggator Reader client.Reader } @@ -67,9 +68,6 @@ type CFDataClient interface { } func NewCacheManager(config *Config) (*CacheManager, error) { - if config.WatchedSet == nil { - return nil, fmt.Errorf("watchedSet must be non-nil") - } if config.Registrar == nil { return nil, fmt.Errorf("registrar must be non-nil") } @@ -93,7 +91,7 @@ func NewCacheManager(config *Config) (*CacheManager, error) { tracker: config.Tracker, processExcluder: config.ProcessExcluder, registrar: config.Registrar, - watchedSet: config.WatchedSet, + watchedSet: watch.NewSet(), reader: config.Reader, gvksToSync: config.GVKAggregator, backgroundManagementTicker: *time.NewTicker(3 * time.Second), @@ -190,6 +188,15 @@ func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { c.excluderChanged = true } +// DoForEach runs fn function for each GVK that is being watched by the cache manager. +func (c *CacheManager) DoForEach(fn func(gvk schema.GroupVersionKind) error) error { + c.mu.Lock() + defer c.mu.Unlock() + + err := c.watchedSet.DoForEach(fn) + return err +} + func (c *CacheManager) watchesGVK(gvk schema.GroupVersionKind) bool { c.mu.RLock() defer c.mu.RUnlock() diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 6bd4672ef84..3eaa4007c1d 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -56,7 +56,6 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, ProcessExcluder: processExcluder, - WatchedSet: watch.NewSet(), Registrar: w, Reader: c, GVKAggregator: aggregator.NewGVKAggregator(), diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index c04aef5efca..9091cddeb26 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -347,7 +347,6 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, ProcessExcluder: processExcluder, - WatchedSet: watch.NewSet(), Registrar: w, Reader: reader, GVKAggregator: aggregator, diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index 3a5243b6609..4d3788d3222 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -43,8 +43,7 @@ import ( ) const ( - CtrlName = "config-controller" - + ctrlName = "config-controller" finalizerName = "finalizers.gatekeeper.sh/config" ) @@ -109,7 +108,7 @@ func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.Controlle // add adds a new Controller to mgr with r as the reconcile.Reconciler. func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(CtrlName, mgr, controller.Options{Reconciler: r}) + c, err := controller.New(ctrlName, mgr, controller.Options{Reconciler: r}) if err != nil { return err } diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index cda2859e70b..81e1f11d6ba 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -146,10 +146,9 @@ func TestReconcile(t *testing.T) { processExcluder := process.Get() processExcluder.Add(instance.Spec.Match) events := make(chan event.GenericEvent, 1024) - watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - CtrlName, + cachemanager.RegName, events) require.NoError(t, err) cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ @@ -157,7 +156,6 @@ func TestReconcile(t *testing.T) { SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - WatchedSet: watchSet, Registrar: w, Reader: c, }) @@ -435,10 +433,9 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager // avoiding conflicts in finalizer cleanup. cs := watch.NewSwitch() processExcluder := process.Get() - watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - CtrlName, + cachemanager.RegName, events) if err != nil { return nil, fmt.Errorf("cannot create registrar: %w", err) @@ -448,7 +445,6 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - WatchedSet: watchSet, Registrar: w, Reader: reader, }) @@ -627,10 +623,9 @@ func TestConfig_Retries(t *testing.T) { processExcluder.Add(instance.Spec.Match) events := make(chan event.GenericEvent, 1024) - watchSet := watch.NewSet() syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - CtrlName, + cachemanager.RegName, events) require.NoError(t, err) cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ @@ -638,7 +633,6 @@ func TestConfig_Retries(t *testing.T) { SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - WatchedSet: watchSet, Registrar: w, Reader: c, }) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 88c2a85c0d0..3c82d138bd5 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -25,7 +25,6 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" cm "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" - "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" syncc "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/sync" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" @@ -33,7 +32,6 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" "github.com/open-policy-agent/gatekeeper/v3/pkg/pubsub" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" - "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" corev1 "k8s.io/api/core/v1" @@ -89,8 +87,9 @@ type Dependencies struct { MutationSystem *mutation.System ExpansionSystem *expansion.System ProviderCache *externaldata.ProviderCache - WatchSet *watch.Set PubsubSystem *pubsub.System + EventsCh chan event.GenericEvent + CacheMgr *cm.CacheManager } type defaultPodGetter struct { @@ -163,38 +162,15 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { deps.GetPod = fakePodGetter } - // Events will be used to receive events from dynamic watches registered - // via the registrar below. - events := make(chan event.GenericEvent, 1024) - syncMetricsCache := syncutil.NewMetricsCache() - w, err := deps.WatchManger.NewRegistrar( - config.CtrlName, - events) - if err != nil { - return err - } - cm, err := cm.NewCacheManager(&cm.Config{ - CfClient: deps.Opa, - SyncMetricsCache: syncMetricsCache, - Tracker: deps.Tracker, - ProcessExcluder: deps.ProcessExcluder, - Registrar: w, - WatchedSet: deps.WatchSet, - Reader: m.GetCache(), - }) - if err != nil { - return err - } - // Adding the CacheManager as a runnable; // manager will start CacheManager. - if err := m.Add(cm); err != nil { + if err := m.Add(deps.CacheMgr); err != nil { return fmt.Errorf("error adding cache manager as a runnable: %w", err) } syncAdder := syncc.Adder{ - Events: events, - CacheManager: cm, + Events: deps.EventsCh, + CacheManager: deps.CacheMgr, } // Create subordinate controller - we will feed it events dynamically via watch if err := syncAdder.Add(m); err != nil { @@ -217,7 +193,7 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { } if a2, ok := a.(CacheManagerInjector); ok { // this is used by the config controller to sync - a2.InjectCacheManager(cm) + a2.InjectCacheManager(deps.CacheMgr) } if err := a.Add(m); err != nil { diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index 7cd18d4abf0..4efe417bd16 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -30,6 +30,7 @@ import ( constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers/rego" frameworksexternaldata "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" + "github.com/open-policy-agent/gatekeeper/v3/pkg/cachemanager" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/expansion" @@ -37,6 +38,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation" mutationtypes "github.com/open-policy-agent/gatekeeper/v3/pkg/mutation/types" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" + "github.com/open-policy-agent/gatekeeper/v3/pkg/syncutil" "github.com/open-policy-agent/gatekeeper/v3/pkg/target" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" @@ -49,6 +51,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -102,7 +105,7 @@ func setupOpa(t *testing.T) *constraintclient.Client { func setupController( mgr manager.Manager, wm *watch.Manager, - opa *constraintclient.Client, + cfClient *constraintclient.Client, mutationSystem *mutation.System, expansionSystem *expansion.System, providerCache *frameworksexternaldata.ProviderCache, @@ -125,9 +128,29 @@ func setupController( processExcluder := process.Get() + events := make(chan event.GenericEvent, 1024) + syncMetricsCache := syncutil.NewMetricsCache() + w, err := wm.NewRegistrar( + cachemanager.RegName, + events) + if err != nil { + return fmt.Errorf("setting up watch manager: %w", err) + } + cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ + CfClient: cfClient, + SyncMetricsCache: syncMetricsCache, + Tracker: tracker, + ProcessExcluder: processExcluder, + Registrar: w, + Reader: mgr.GetCache(), + }) + if err != nil { + return fmt.Errorf("setting up cache manager: %w", err) + } + // Setup all Controllers opts := controller.Dependencies{ - Opa: opa, + Opa: cfClient, WatchManger: wm, ControllerSwitch: sw, Tracker: tracker, @@ -136,7 +159,8 @@ func setupController( MutationSystem: mutationSystem, ExpansionSystem: expansionSystem, ProviderCache: providerCache, - WatchSet: watch.NewSet(), + CacheMgr: cacheManager, + EventsCh: events, } if err := controller.AddToManager(mgr, &opts); err != nil { return fmt.Errorf("registering controllers: %w", err) From 06a4ee6b20fda1cc74d5ba89f46d84e4f07f34de Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 18 Aug 2023 17:55:28 +0000 Subject: [PATCH 53/58] review: naming nits Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 4 ++-- pkg/audit/audit_cache_lister.go | 6 +++--- pkg/cachemanager/cachemanager.go | 2 +- pkg/controller/config/config_controller_test.go | 6 +++--- pkg/controller/controller.go | 4 ++-- pkg/readiness/ready_tracker_test.go | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 7b6d17e27fe..f30bbef4772 100644 --- a/main.go +++ b/main.go @@ -455,7 +455,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle // via the registrar below. events := make(chan event.GenericEvent, 1024) w, err := wm.NewRegistrar( - cachemanager.RegName, + cachemanager.RegistrarName, events) if err != nil { setupLog.Error(err, "unable to set up watch registrar for cache manager") @@ -479,7 +479,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle opts := controller.Dependencies{ Opa: client, WatchManger: wm, - EventsCh: events, + SyncEventsCh: events, CacheMgr: cm, ControllerSwitch: sw, Tracker: tracker, diff --git a/pkg/audit/audit_cache_lister.go b/pkg/audit/audit_cache_lister.go index 74bddf4cf9e..990af0b23bb 100644 --- a/pkg/audit/audit_cache_lister.go +++ b/pkg/audit/audit_cache_lister.go @@ -10,7 +10,7 @@ import ( // NewAuditCacheLister instantiates a new AuditCache which will read objects in // watched from auditCache. -func NewAuditCacheLister(auditCache client.Reader, lister Delegate) *CacheLister { +func NewAuditCacheLister(auditCache client.Reader, lister WatchIterator) *CacheLister { return &CacheLister{ auditCache: auditCache, lister: lister, @@ -25,11 +25,11 @@ type CacheLister struct { // DoForEach. auditCache client.Reader // lister is a delegate like cachemanager that we can use to query a watched set of GKVs. - lister Delegate + lister WatchIterator } // wraps DoForEach from a watch.Set. -type Delegate interface { +type WatchIterator interface { DoForEach(listFunc func(gvk schema.GroupVersionKind) error) error } diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 6cdd8ad0958..30ec09c156e 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -21,7 +21,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -const RegName = "cachemanager" +const RegistrarName = "cachemanager" var ( log = logf.Log.WithName("cache-manager") diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 81e1f11d6ba..fe39bc181fa 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -148,7 +148,7 @@ func TestReconcile(t *testing.T) { events := make(chan event.GenericEvent, 1024) syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - cachemanager.RegName, + cachemanager.RegistrarName, events) require.NoError(t, err) cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ @@ -435,7 +435,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager processExcluder := process.Get() syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - cachemanager.RegName, + cachemanager.RegistrarName, events) if err != nil { return nil, fmt.Errorf("cannot create registrar: %w", err) @@ -625,7 +625,7 @@ func TestConfig_Retries(t *testing.T) { events := make(chan event.GenericEvent, 1024) syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - cachemanager.RegName, + cachemanager.RegistrarName, events) require.NoError(t, err) cacheManager, err := cachemanager.NewCacheManager(&cachemanager.Config{ diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 3c82d138bd5..97f4b1760bc 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -88,7 +88,7 @@ type Dependencies struct { ExpansionSystem *expansion.System ProviderCache *externaldata.ProviderCache PubsubSystem *pubsub.System - EventsCh chan event.GenericEvent + SyncEventsCh chan event.GenericEvent CacheMgr *cm.CacheManager } @@ -169,7 +169,7 @@ func AddToManager(m manager.Manager, deps *Dependencies) error { } syncAdder := syncc.Adder{ - Events: deps.EventsCh, + Events: deps.SyncEventsCh, CacheManager: deps.CacheMgr, } // Create subordinate controller - we will feed it events dynamically via watch diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index 4efe417bd16..f808131c27e 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -131,7 +131,7 @@ func setupController( events := make(chan event.GenericEvent, 1024) syncMetricsCache := syncutil.NewMetricsCache() w, err := wm.NewRegistrar( - cachemanager.RegName, + cachemanager.RegistrarName, events) if err != nil { return fmt.Errorf("setting up watch manager: %w", err) @@ -160,7 +160,7 @@ func setupController( ExpansionSystem: expansionSystem, ProviderCache: providerCache, CacheMgr: cacheManager, - EventsCh: events, + SyncEventsCh: events, } if err := controller.AddToManager(mgr, &opts); err != nil { return fmt.Errorf("registering controllers: %w", err) From 505de58ee535941b69dc072007abb7d3c75eb573 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 22 Aug 2023 19:59:15 +0000 Subject: [PATCH 54/58] review: naming, comments Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- main.go | 4 +- pkg/audit/audit_cache_lister.go | 5 +- pkg/cachemanager/cachemanager.go | 3 +- pkg/cachemanager/cachemanager_test.go | 4 +- .../cachemanager_integration_test.go | 4 +- .../config/config_controller_test.go | 46 +++++-------------- pkg/readiness/ready_tracker_test.go | 4 +- 7 files changed, 25 insertions(+), 45 deletions(-) diff --git a/main.go b/main.go index f30bbef4772..0f68b14d967 100644 --- a/main.go +++ b/main.go @@ -454,7 +454,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle // Events ch will be used to receive events from dynamic watches registered // via the registrar below. events := make(chan event.GenericEvent, 1024) - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( cachemanager.RegistrarName, events) if err != nil { @@ -468,7 +468,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, sw *watch.Controlle SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: mgr.GetCache(), }) if err != nil { diff --git a/pkg/audit/audit_cache_lister.go b/pkg/audit/audit_cache_lister.go index 990af0b23bb..7925d0f3593 100644 --- a/pkg/audit/audit_cache_lister.go +++ b/pkg/audit/audit_cache_lister.go @@ -24,7 +24,10 @@ type CacheLister struct { // Caution: only to be read from while watched is locked, such as through // DoForEach. auditCache client.Reader - // lister is a delegate like cachemanager that we can use to query a watched set of GKVs. + // lister is a delegate like CacheManager that we can use to query a watched set of GKVs. + // Passing our logic as a callback to a watched.Set allows us to take actions while + // holding the lock on the watched.Set. This prevents us from querying the API server + // for kinds that aren't currently being watched by the CacheManager. lister WatchIterator } diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 30ec09c156e..0a92227cb4e 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -188,7 +188,8 @@ func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { c.excluderChanged = true } -// DoForEach runs fn function for each GVK that is being watched by the cache manager. +// DoForEach runs fn for each GVK that is being watched by the cache manager. +// This is handy when we want to take actions while holding the lock on the watched.Set. func (c *CacheManager) DoForEach(fn func(gvk schema.GroupVersionKind) error) error { c.mu.Lock() defer c.mu.Unlock() diff --git a/pkg/cachemanager/cachemanager_test.go b/pkg/cachemanager/cachemanager_test.go index 3eaa4007c1d..3e7fa9bbeec 100644 --- a/pkg/cachemanager/cachemanager_test.go +++ b/pkg/cachemanager/cachemanager_test.go @@ -46,7 +46,7 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { }, }) events := make(chan event.GenericEvent, 1024) - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( "test-cache-manager", events) require.NoError(t, err) @@ -56,7 +56,7 @@ func makeCacheManager(t *testing.T) (*CacheManager, context.Context) { SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: c, GVKAggregator: aggregator.NewGVKAggregator(), }) diff --git a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go index 9091cddeb26..7618c62bf45 100644 --- a/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go +++ b/pkg/cachemanager/cachemanager_test/cachemanager_integration_test.go @@ -336,7 +336,7 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea }, }) events := make(chan event.GenericEvent, 1024) - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( "test-cache-manager", events) require.NoError(t, err) @@ -347,7 +347,7 @@ func makeTestResources(t *testing.T, mgr manager.Manager, wm *watch.Manager, rea SyncMetricsCache: syncutil.NewMetricsCache(), Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: reader, GVKAggregator: aggregator, } diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index fe39bc181fa..89fa018c904 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -17,7 +17,6 @@ package config import ( "fmt" - gosync "sync" "testing" "time" @@ -96,15 +95,9 @@ func setupManager(t *testing.T) (manager.Manager, *watch.Manager) { func TestReconcile(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) - g := gomega.NewGomegaWithT(t) - once := gosync.Once{} - testMgrStopped := func() { - once.Do(func() { - cancelFunc() - }) - } + defer cancelFunc() - defer testMgrStopped() + g := gomega.NewGomegaWithT(t) instance := &configv1alpha1.Config{ ObjectMeta: metav1.ObjectMeta{ @@ -147,7 +140,7 @@ func TestReconcile(t *testing.T) { processExcluder.Add(instance.Spec.Match) events := make(chan event.GenericEvent, 1024) syncMetricsCache := syncutil.NewMetricsCache() - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( cachemanager.RegistrarName, events) require.NoError(t, err) @@ -156,7 +149,7 @@ func TestReconcile(t *testing.T) { SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: c, }) require.NoError(t, err) @@ -321,12 +314,7 @@ func TestConfig_DeleteSyncResources(t *testing.T) { } ctx, cancelFunc := context.WithCancel(context.Background()) - once := gosync.Once{} - defer func() { - once.Do(func() { - cancelFunc() - }) - }() + defer cancelFunc() err := c.Create(ctx, instance) if err != nil { @@ -434,7 +422,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager cs := watch.NewSwitch() processExcluder := process.Get() syncMetricsCache := syncutil.NewMetricsCache() - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( cachemanager.RegistrarName, events) if err != nil { @@ -445,7 +433,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: reader, }) if err != nil { @@ -478,13 +466,7 @@ func setupController(ctx context.Context, mgr manager.Manager, wm *watch.Manager // Verify the Opa cache is populated based on the config resource. func TestConfig_CacheContents(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) - once := gosync.Once{} - testMgrStopped := func() { - once.Do(func() { - cancelFunc() - }) - } - defer testMgrStopped() + defer cancelFunc() // Setup the Manager and Controller. mgr, wm := setupManager(t) @@ -588,13 +570,7 @@ func TestConfig_CacheContents(t *testing.T) { func TestConfig_Retries(t *testing.T) { ctx, cancelFunc := context.WithCancel(context.Background()) - once := gosync.Once{} - testMgrStopped := func() { - once.Do(func() { - cancelFunc() - }) - } - defer testMgrStopped() + defer cancelFunc() g := gomega.NewGomegaWithT(t) nsGVK := schema.GroupVersionKind{ @@ -624,7 +600,7 @@ func TestConfig_Retries(t *testing.T) { events := make(chan event.GenericEvent, 1024) syncMetricsCache := syncutil.NewMetricsCache() - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( cachemanager.RegistrarName, events) require.NoError(t, err) @@ -633,7 +609,7 @@ func TestConfig_Retries(t *testing.T) { SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: c, }) require.NoError(t, err) diff --git a/pkg/readiness/ready_tracker_test.go b/pkg/readiness/ready_tracker_test.go index f808131c27e..3a14351c11c 100644 --- a/pkg/readiness/ready_tracker_test.go +++ b/pkg/readiness/ready_tracker_test.go @@ -130,7 +130,7 @@ func setupController( events := make(chan event.GenericEvent, 1024) syncMetricsCache := syncutil.NewMetricsCache() - w, err := wm.NewRegistrar( + reg, err := wm.NewRegistrar( cachemanager.RegistrarName, events) if err != nil { @@ -141,7 +141,7 @@ func setupController( SyncMetricsCache: syncMetricsCache, Tracker: tracker, ProcessExcluder: processExcluder, - Registrar: w, + Registrar: reg, Reader: mgr.GetCache(), }) if err != nil { From 21f7d578814ab42477fe43cf524fae08701db1dc Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 23 Aug 2023 21:44:20 +0000 Subject: [PATCH 55/58] review: use read locks Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/audit/audit_cache_lister.go | 10 +++++----- pkg/cachemanager/aggregator/aggregator.go | 8 ++++---- pkg/cachemanager/cachemanager.go | 8 ++++---- pkg/syncutil/stats_reporter.go | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/audit/audit_cache_lister.go b/pkg/audit/audit_cache_lister.go index 7925d0f3593..89220128e37 100644 --- a/pkg/audit/audit_cache_lister.go +++ b/pkg/audit/audit_cache_lister.go @@ -12,8 +12,8 @@ import ( // watched from auditCache. func NewAuditCacheLister(auditCache client.Reader, lister WatchIterator) *CacheLister { return &CacheLister{ - auditCache: auditCache, - lister: lister, + auditCache: auditCache, + watchIterator: lister, } } @@ -24,11 +24,11 @@ type CacheLister struct { // Caution: only to be read from while watched is locked, such as through // DoForEach. auditCache client.Reader - // lister is a delegate like CacheManager that we can use to query a watched set of GKVs. + // watchIterator is a delegate like CacheManager that we can use to query a watched set of GKVs. // Passing our logic as a callback to a watched.Set allows us to take actions while // holding the lock on the watched.Set. This prevents us from querying the API server // for kinds that aren't currently being watched by the CacheManager. - lister WatchIterator + watchIterator WatchIterator } // wraps DoForEach from a watch.Set. @@ -39,7 +39,7 @@ type WatchIterator interface { // ListObjects lists all objects from the audit cache. func (l *CacheLister) ListObjects(ctx context.Context) ([]unstructured.Unstructured, error) { var objs []unstructured.Unstructured - err := l.lister.DoForEach(func(gvk schema.GroupVersionKind) error { + err := l.watchIterator.DoForEach(func(gvk schema.GroupVersionKind) error { gvkObjects, err := listObjects(ctx, l.auditCache, gvk) if err != nil { return err diff --git a/pkg/cachemanager/aggregator/aggregator.go b/pkg/cachemanager/aggregator/aggregator.go index e84f2fc2c8d..5b0b78aec63 100644 --- a/pkg/cachemanager/aggregator/aggregator.go +++ b/pkg/cachemanager/aggregator/aggregator.go @@ -100,8 +100,8 @@ func (b *GVKAgreggator) Upsert(k Key, gvks []schema.GroupVersionKind) error { // List returnes the gvk set for a given Key. func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { - b.mu.Lock() - defer b.mu.Unlock() + b.mu.RLock() + defer b.mu.RUnlock() v := b.store[k] cpy := make(map[schema.GroupVersionKind]struct{}, len(v)) @@ -113,8 +113,8 @@ func (b *GVKAgreggator) List(k Key) map[schema.GroupVersionKind]struct{} { // GVKs returns a list of all of the schema.GroupVersionKind that are aggregated. func (b *GVKAgreggator) GVKs() []schema.GroupVersionKind { - b.mu.Lock() - defer b.mu.Unlock() + b.mu.RLock() + defer b.mu.RUnlock() allGVKs := []schema.GroupVersionKind{} for gvk := range b.reverseStore { diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 0a92227cb4e..0b11946e7d0 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -191,8 +191,8 @@ func (c *CacheManager) ExcludeProcesses(newExcluder *process.Excluder) { // DoForEach runs fn for each GVK that is being watched by the cache manager. // This is handy when we want to take actions while holding the lock on the watched.Set. func (c *CacheManager) DoForEach(fn func(gvk schema.GroupVersionKind) error) error { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() err := c.watchedSet.DoForEach(fn) return err @@ -283,8 +283,8 @@ func (c *CacheManager) syncGVK(ctx context.Context, gvk schema.GroupVersionKind) var err error func() { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() // only call List if we are still watching the gvk. if c.watchedSet.Contains(gvk) { diff --git a/pkg/syncutil/stats_reporter.go b/pkg/syncutil/stats_reporter.go index 2e2320c0d0b..42a1b8f2f32 100644 --- a/pkg/syncutil/stats_reporter.go +++ b/pkg/syncutil/stats_reporter.go @@ -109,8 +109,8 @@ func (c *MetricsCache) DeleteObject(key string) { } func (c *MetricsCache) GetTags(key string) *Tags { - c.mux.Lock() - defer c.mux.Unlock() + c.mux.RLock() + defer c.mux.RUnlock() cpy := &Tags{} v, ok := c.Cache[key] @@ -123,8 +123,8 @@ func (c *MetricsCache) GetTags(key string) *Tags { } func (c *MetricsCache) HasObject(key string) bool { - c.mux.Lock() - defer c.mux.Unlock() + c.mux.RLock() + defer c.mux.RUnlock() _, ok := c.Cache[key] return ok From 10e62bf03774c5390bd9da320da8d2439079ca7b Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Wed, 23 Aug 2023 22:42:22 +0000 Subject: [PATCH 56/58] use failure injector in config_c test Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/controller/config/config_controller_test.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 89fa018c904..3ae7d79b980 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -629,19 +629,15 @@ func TestConfig_Retries(t *testing.T) { require.NoError(t, syncAdder.Add(mgr), "registering sync controller") // Use our special reader interceptor to inject controlled failures - failPlease := make(chan string, 1) + fi := fakes.NewFailureInjector() rec.reader = fakes.SpyReader{ Reader: mgr.GetCache(), ListFunc: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - // Return an error the first go-around. - var failKind string - select { - case failKind = <-failPlease: - default: - } - if failKind != "" && list.GetObjectKind().GroupVersionKind().Kind == failKind { + // return as many syntenthic failures as there are registered for this kind + if fi.CheckFailures(list.GetObjectKind().GroupVersionKind().Kind) { return fmt.Errorf("synthetic failure") } + return mgr.GetCache().List(ctx, list, opts...) }, } @@ -685,8 +681,7 @@ func TestConfig_Retries(t *testing.T) { return opaClient.Contains(expected) }, 10*time.Second).Should(gomega.BeTrue(), "checking initial opa cache contents") - // Make List fail once for ConfigMaps as the replay occurs following the reconfig below. - failPlease <- "ConfigMapList" + fi.SetFailures("ConfigMapList", 2) // Reconfigure to force an internal replay. instance = configFor([]schema.GroupVersionKind{configMapGVK}) From 0637f6eb55ded5f0a0e4c519b41034e16913074d Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 29 Aug 2023 17:53:32 +0000 Subject: [PATCH 57/58] remove stale expectations Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- pkg/cachemanager/cachemanager.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/cachemanager/cachemanager.go b/pkg/cachemanager/cachemanager.go index 0b11946e7d0..5af1e2c4b5a 100644 --- a/pkg/cachemanager/cachemanager.go +++ b/pkg/cachemanager/cachemanager.go @@ -141,7 +141,10 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { newWatchSet := watch.NewSet() newWatchSet.Add(c.gvksToSync.GVKs()...) - c.gvksToDeleteFromCache.AddSet(c.watchedSet.Difference(newWatchSet)) + diff := c.watchedSet.Difference(newWatchSet) + c.removeStaleExpectations(diff) + + c.gvksToDeleteFromCache.AddSet(diff) var innerError error c.watchedSet.Replace(newWatchSet, func() { @@ -155,6 +158,13 @@ func (c *CacheManager) replaceWatchSet(ctx context.Context) error { return innerError } +// removeStaleExpectations stops tracking data for any resources that are no longer watched. +func (c *CacheManager) removeStaleExpectations(stale *watch.Set) { + for _, gvk := range stale.Items() { + c.tracker.CancelData(gvk) + } +} + // RemoveSource removes the watches of the GVKs for a given aggregator.Key. Callers are responsible for retrying on error. func (c *CacheManager) RemoveSource(ctx context.Context, sourceKey aggregator.Key) error { c.mu.Lock() From 6ca3fa597c2f51f56ed13c436d465bd8880c4bf0 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Tue, 29 Aug 2023 22:37:58 +0000 Subject: [PATCH 58/58] fix flaky test AFAICT, the root cause is an internal controller race in config_c that triggers an additional reconcile request to be sent down the previous wrapped channel which will block as it's not being read. Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- .../config/config_controller_suite_test.go | 9 +++++---- .../config/config_controller_test.go | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/controller/config/config_controller_suite_test.go b/pkg/controller/config/config_controller_suite_test.go index a10be3452be..3932f427015 100644 --- a/pkg/controller/config/config_controller_suite_test.go +++ b/pkg/controller/config/config_controller_suite_test.go @@ -20,6 +20,7 @@ import ( stdlog "log" "os" "path/filepath" + "sync" "testing" "github.com/open-policy-agent/gatekeeper/v3/apis" @@ -62,12 +63,12 @@ func TestMain(m *testing.M) { // SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and // writes the request to requests after Reconcile is finished. -func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { - requests := make(chan reconcile.Request) +func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, *sync.Map) { + var requests sync.Map fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { result, err := inner.Reconcile(ctx, req) - requests <- req + requests.Store(req, struct{}{}) return result, err }) - return fn, requests + return fn, &requests } diff --git a/pkg/controller/config/config_controller_test.go b/pkg/controller/config/config_controller_test.go index 3ae7d79b980..7e85c13f8ae 100644 --- a/pkg/controller/config/config_controller_test.go +++ b/pkg/controller/config/config_controller_test.go @@ -58,11 +58,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "config", - Namespace: "gatekeeper-system", -}} - const timeout = time.Second * 20 // setupManager sets up a controller-runtime manager with registered watch manager. @@ -123,9 +118,6 @@ func TestReconcile(t *testing.T) { }, }, } - - // Set up the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a - // channel when it is finished. mgr, wm := setupManager(t) c := testclient.NewRetryClient(mgr.GetClient()) @@ -162,6 +154,7 @@ func TestReconcile(t *testing.T) { rec, err := newReconciler(mgr, cacheManager, cs, tracker) require.NoError(t, err) + // Wrap the Controller Reconcile function so it writes each request to a map when it is finished reconciling. recFn, requests := SetupTestReconcile(rec) require.NoError(t, add(mgr, recFn)) @@ -180,7 +173,15 @@ func TestReconcile(t *testing.T) { t.Fatal(err) } }() - g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + g.Eventually(func() bool { + expectedReq := reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "config", + Namespace: "gatekeeper-system", + }} + _, ok := requests.Load(expectedReq) + + return ok + }).WithTimeout(timeout).Should(gomega.BeTrue()) g.Eventually(func() int { return len(wm.GetManagedGVK())