From c5ef42e80e806f4a75a032a12e75caacfe0b7a8c Mon Sep 17 00:00:00 2001 From: Avinash Patnala Date: Mon, 21 Oct 2024 17:19:47 +0000 Subject: [PATCH 1/2] report config metrics Signed-off-by: Avinash Patnala --- pkg/controller/config/config_controller.go | 17 +++++ pkg/controller/config/stats_reporter.go | 54 ++++++++++++++++ pkg/controller/config/stats_reporter_test.go | 66 ++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 pkg/controller/config/stats_reporter.go create mode 100644 pkg/controller/config/stats_reporter_test.go diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index afe0cd7adc7..fb47da09dd4 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -26,6 +26,7 @@ import ( "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/config/process" "github.com/open-policy-agent/gatekeeper/v3/pkg/controller/configstatus" "github.com/open-policy-agent/gatekeeper/v3/pkg/keys" + "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" "github.com/open-policy-agent/gatekeeper/v3/pkg/readiness" "github.com/open-policy-agent/gatekeeper/v3/pkg/util" "github.com/open-policy-agent/gatekeeper/v3/pkg/watch" @@ -93,6 +94,11 @@ func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.Controlle return nil, fmt.Errorf("cacheManager must be non-nil") } + r, err := newStatsReporter() + if err != nil { + return nil, err + } + return &ReconcileConfig{ reader: mgr.GetCache(), writer: mgr.GetClient(), @@ -100,6 +106,7 @@ func newReconciler(mgr manager.Manager, cm *cm.CacheManager, cs *watch.Controlle scheme: mgr.GetScheme(), cs: cs, cacheManager: cm, + metrics: r, tracker: tracker, getPod: getPod, }, nil @@ -139,6 +146,7 @@ type ReconcileConfig struct { scheme *runtime.Scheme cacheManager *cm.CacheManager cs *watch.ControllerSwitch + metrics *reporter tracker *readiness.Tracker @@ -164,6 +172,13 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque } } + reportMetrics := false + defer func() { + if reportMetrics { + r.metrics.reportConfig(ctx, metrics.ActiveStatus, 0) + } + }() + // Fetch the Config instance if request.NamespacedName != keys.Config { log.Info("Ignoring unsupported config name", "namespace", request.NamespacedName.Namespace, "name", request.NamespacedName.Name) @@ -199,6 +214,8 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque newExcluder.Add(instance.Spec.Match) statsEnabled = instance.Spec.Readiness.StatsEnabled + // Report metrics of config only if its not deleted. + reportMetrics = true } // Enable verbose readiness stats if requested. diff --git a/pkg/controller/config/stats_reporter.go b/pkg/controller/config/stats_reporter.go new file mode 100644 index 00000000000..16fab36e151 --- /dev/null +++ b/pkg/controller/config/stats_reporter.go @@ -0,0 +1,54 @@ +package config + +import ( + "context" + "sync" + + "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const ( + cfgMetricName = "config" + statusKey = "status" +) + +func (r *reporter) observeConfig(_ context.Context, observer metric.Int64Observer) error { + r.mux.RLock() + defer r.mux.RUnlock() + for t, v := range r.configReport { + observer.Observe(v, metric.WithAttributes(attribute.String(statusKey, string(t)))) + } + return nil +} + +func (r *reporter) reportConfig(_ context.Context, t metrics.Status, v int64) error { + r.mux.Lock() + defer r.mux.Unlock() + if r.configReport == nil { + r.configReport = make(map[metrics.Status]int64) + } + r.configReport[t] = v + return nil +} + +// newStatsReporter creates a reporter for audit metrics. +func newStatsReporter() (*reporter, error) { + r := &reporter{} + var err error + meter := otel.GetMeterProvider().Meter("gatekeeper") + _, err = meter.Int64ObservableGauge( + cfgMetricName, + metric.WithDescription("Config Status"), metric.WithInt64Callback(r.observeConfig)) + if err != nil { + return nil, err + } + return r, nil +} + +type reporter struct { + mux sync.RWMutex + configReport map[metrics.Status]int64 +} diff --git a/pkg/controller/config/stats_reporter_test.go b/pkg/controller/config/stats_reporter_test.go new file mode 100644 index 00000000000..ef0a8b3d643 --- /dev/null +++ b/pkg/controller/config/stats_reporter_test.go @@ -0,0 +1,66 @@ +package config + +import ( + "context" + "testing" + + "github.com/open-policy-agent/gatekeeper/v3/pkg/metrics" + testmetric "github.com/open-policy-agent/gatekeeper/v3/test/metrics" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" +) + +func initializeTestInstruments(t *testing.T) (rdr *sdkmetric.PeriodicReader, r *reporter) { + var err error + rdr = sdkmetric.NewPeriodicReader(new(testmetric.FnExporter)) + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr)) + r, err = newStatsReporter() + assert.NoError(t, err) + meter := mp.Meter("test") + + // Ensure the pipeline has a callback setup + _, err = meter.Int64ObservableGauge(cfgMetricName, metric.WithInt64Callback(r.observeConfig)) + assert.NoError(t, err) + return rdr, r +} + +func TestReportConfig(t *testing.T) { + tests := []struct { + name string + ctx context.Context + expectedErr error + want metricdata.Metrics + }{ + { + name: "reporting config status", + ctx: context.Background(), + expectedErr: nil, + want: metricdata.Metrics{ + Name: cfgMetricName, + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{ + {Attributes: attribute.NewSet(attribute.String(statusKey, string(metrics.ActiveStatus))), Value: 0}, + {Attributes: attribute.NewSet(attribute.String(statusKey, string(metrics.ErrorStatus))), Value: 0}, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rdr, r := initializeTestInstruments(t) + for _, status := range metrics.AllStatuses { + assert.NoError(t, r.reportConfig(tt.ctx, status, 0)) + } + + rm := &metricdata.ResourceMetrics{} + assert.Equal(t, tt.expectedErr, rdr.Collect(tt.ctx, rm)) + metricdatatest.AssertEqual(t, tt.want, rm.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp()) + }) + } +} From 1ce846166c77bcf6886556ce35ad3198297b4733 Mon Sep 17 00:00:00 2001 From: Avinash Patnala Date: Mon, 21 Oct 2024 17:43:00 +0000 Subject: [PATCH 2/2] handle error Signed-off-by: Avinash Patnala --- pkg/controller/config/config_controller.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/controller/config/config_controller.go b/pkg/controller/config/config_controller.go index fb47da09dd4..e38ebb682b3 100644 --- a/pkg/controller/config/config_controller.go +++ b/pkg/controller/config/config_controller.go @@ -175,7 +175,9 @@ func (r *ReconcileConfig) Reconcile(ctx context.Context, request reconcile.Reque reportMetrics := false defer func() { if reportMetrics { - r.metrics.reportConfig(ctx, metrics.ActiveStatus, 0) + if err := r.metrics.reportConfig(ctx, metrics.ActiveStatus, 0); err != nil { + log.Info("Error when reporting config status metric", err) + } } }()