diff --git a/api/v1beta1/topology.go b/api/v1beta1/topology.go index de7faa347..01cb5384a 100644 --- a/api/v1beta1/topology.go +++ b/api/v1beta1/topology.go @@ -7,8 +7,11 @@ import ( "github.com/kuadrant/policy-machinery/machinery" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1" + + kuad "github.com/kuadrant/kuadrant-operator/pkg/kuadrant" ) var ( @@ -71,8 +74,12 @@ func LinkKuadrantToServiceMonitor(objs controller.Store) machinery.LinkFunc { To: schema.GroupKind{Group: monitoringv1.SchemeGroupVersion.Group, Kind: monitoringv1.ServiceMonitorsKind}, Func: func(child machinery.Object) []machinery.Object { return lo.Filter(kuadrants, func(kuadrant machinery.Object, _ int) bool { - // ServiceMonitors outside the Kuadrant namespace will be outside the topology tree - return kuadrant.GetNamespace() == child.GetNamespace() + if metaObj, ok := child.(metav1.Object); ok { + if val, exists := metaObj.GetLabels()[kuad.ObservabilityLabel]; exists { + return val == "true" + } + } + return false }) }, } @@ -86,8 +93,12 @@ func LinkKuadrantToPodMonitor(objs controller.Store) machinery.LinkFunc { To: schema.GroupKind{Group: monitoringv1.SchemeGroupVersion.Group, Kind: monitoringv1.PodMonitorsKind}, Func: func(child machinery.Object) []machinery.Object { return lo.Filter(kuadrants, func(kuadrant machinery.Object, _ int) bool { - // PodMonitors outside the Kuadrant namespace will be outside the topology tree - return kuadrant.GetNamespace() == child.GetNamespace() + if metaObj, ok := child.(metav1.Object); ok { + if val, exists := metaObj.GetLabels()[kuad.ObservabilityLabel]; exists { + return val == "true" + } + } + return false }) }, } diff --git a/bundle/manifests/kuadrant-operator.clusterserviceversion.yaml b/bundle/manifests/kuadrant-operator.clusterserviceversion.yaml index 9a75e0110..64f996870 100644 --- a/bundle/manifests/kuadrant-operator.clusterserviceversion.yaml +++ b/bundle/manifests/kuadrant-operator.clusterserviceversion.yaml @@ -109,7 +109,7 @@ metadata: capabilities: Basic Install categories: Integration & Delivery containerImage: quay.io/kuadrant/kuadrant-operator:latest - createdAt: "2025-02-24T17:32:12Z" + createdAt: "2025-02-25T10:17:48Z" description: A Kubernetes Operator to manage the lifecycle of the Kuadrant system operators.operatorframework.io/builder: operator-sdk-v1.32.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 @@ -385,6 +385,19 @@ spec: - patch - update - watch + - apiGroups: + - monitoring.coreos.com + resources: + - podmonitors + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - networking.istio.io resources: diff --git a/controllers/observability_reconciler.go b/controllers/observability_reconciler.go index 659b15240..f2c7dddac 100644 --- a/controllers/observability_reconciler.go +++ b/controllers/observability_reconciler.go @@ -18,22 +18,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kuadrantv1beta1 "github.com/kuadrant/kuadrant-operator/api/v1beta1" + "github.com/kuadrant/kuadrant-operator/pkg/kuadrant" ) //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;podmonitors,verbs=get;list;watch;create;update;patch;delete const ( - kuadrantObservabilityLabel = "kuadrant-observability" - kOpMonitorName = "kuadrant-operator-monitor" - dnsOpMonitorName = "dns-operator-monitor" - authOpMonitorName = "authorino-operator-monitor" - limitOpMonitorName = "limitador-operator-monitor" - istiodMonitorName = "istiod-monitor" - istiodMonitorNS = "istio-system" - istioPodMonitorName = "istio-pod-monitor" - envoyGatewayMonitorName = "envoy-gateway-monitor" - envoyGatewayMonitorNS = "envoy-gateway-system" - envoyStatsMonitorName = "envoy-stats-monitor" + kOpMonitorName = "kuadrant-operator-monitor" + dnsOpMonitorName = "dns-operator-monitor" + authOpMonitorName = "authorino-operator-monitor" + limitOpMonitorName = "limitador-operator-monitor" + istiodMonitorName = "istiod-monitor" + istiodMonitorNS = "istio-system" + istioPodMonitorName = "istio-pod-monitor" + envoyGatewayMonitorName = "envoy-gateway-monitor" + envoyGatewayMonitorNS = "envoy-gateway-system" + envoyStatsMonitorName = "envoy-stats-monitor" ) var kOpMonitorSpec = &monitoringv1.ServiceMonitor{ @@ -44,8 +44,8 @@ var kOpMonitorSpec = &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: kOpMonitorName, Labels: map[string]string{ - "control-plane": "controller-manager", - kuadrantObservabilityLabel: "true", + "control-plane": "controller-manager", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.ServiceMonitorSpec{ @@ -71,7 +71,7 @@ var dnsOpMonitorSpec = &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: dnsOpMonitorName, Labels: map[string]string{ - kuadrantObservabilityLabel: "true", + kuadrant.ObservabilityLabel: "true", "control-plane": "controller-manager", "app.kubernetes.io/name": "servicemonitor", "app.kubernetes.io/instance": "controller-manager-metrics-monitor", @@ -103,8 +103,8 @@ var authOpMonitorSpec = &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: authOpMonitorName, Labels: map[string]string{ - "control-plane": "controller-manager", - kuadrantObservabilityLabel: "true", + "control-plane": "controller-manager", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.ServiceMonitorSpec{ @@ -128,8 +128,8 @@ var limitOpMonitorSpec = &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: limitOpMonitorName, Labels: map[string]string{ - "control-plane": "controller-manager", - kuadrantObservabilityLabel: "true", + "control-plane": "controller-manager", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.ServiceMonitorSpec{ @@ -153,7 +153,7 @@ var istiodMonitorSpec = &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: istiodMonitorName, Labels: map[string]string{ - kuadrantObservabilityLabel: "true", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.ServiceMonitorSpec{ @@ -178,7 +178,7 @@ var istioPodMonitorSpec = &monitoringv1.PodMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: istioPodMonitorName, Labels: map[string]string{ - kuadrantObservabilityLabel: "true", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.PodMonitorSpec{ @@ -252,7 +252,7 @@ var envoyGatewayMonitorSpec = &monitoringv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: envoyGatewayMonitorName, Labels: map[string]string{ - kuadrantObservabilityLabel: "true", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.ServiceMonitorSpec{ @@ -275,7 +275,7 @@ var envoyStatsMonitorSpec = &monitoringv1.PodMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: envoyStatsMonitorName, Labels: map[string]string{ - kuadrantObservabilityLabel: "true", + kuadrant.ObservabilityLabel: "true", }, }, Spec: monitoringv1.PodMonitorSpec{ @@ -294,12 +294,14 @@ var envoyStatsMonitorSpec = &monitoringv1.PodMonitor{ type ObservabilityReconciler struct { Client *dynamic.DynamicClient restMapper meta.RESTMapper + namespace string } -func NewObservabilityReconciler(client *dynamic.DynamicClient, rm meta.RESTMapper) *ObservabilityReconciler { +func NewObservabilityReconciler(client *dynamic.DynamicClient, rm meta.RESTMapper, namespace string) *ObservabilityReconciler { return &ObservabilityReconciler{ Client: client, restMapper: rm, + namespace: namespace, } } @@ -341,16 +343,16 @@ func (r *ObservabilityReconciler) Reconcile(ctx context.Context, _ []controller. logger.V(1).Info("observability enabled, creating monitors") // Kuadrant Operator monitor - r.createMonitor(ctx, monitorObjs, kOpMonitorSpec, kObj.Namespace, logger) + r.createMonitor(ctx, monitorObjs, kOpMonitorSpec, r.namespace, logger) // DNS Operator monitor - r.createMonitor(ctx, monitorObjs, dnsOpMonitorSpec, kObj.Namespace, logger) + r.createMonitor(ctx, monitorObjs, dnsOpMonitorSpec, r.namespace, logger) // Authorino operator monitor - r.createMonitor(ctx, monitorObjs, authOpMonitorSpec, kObj.Namespace, logger) + r.createMonitor(ctx, monitorObjs, authOpMonitorSpec, r.namespace, logger) // Limitador operator monitor - r.createMonitor(ctx, monitorObjs, limitOpMonitorSpec, kObj.Namespace, logger) + r.createMonitor(ctx, monitorObjs, limitOpMonitorSpec, r.namespace, logger) // Create monitors for each gateway instance of each gateway class gatewayClasses := topology.Targetables().Items(func(o machinery.Object) bool { diff --git a/controllers/state_of_the_world.go b/controllers/state_of_the_world.go index e5135a827..3de9e7dc0 100644 --- a/controllers/state_of_the_world.go +++ b/controllers/state_of_the_world.go @@ -414,13 +414,13 @@ func (b *BootOptionsBuilder) getObservabilityOptions() []controller.ControllerOp &monitoringv1.ServiceMonitor{}, monitoringv1.SchemeGroupVersion.WithResource("servicemonitors"), metav1.NamespaceAll, - controller.FilterResourcesByLabel[*monitoringv1.ServiceMonitor]("kuadrant-observability=true"), + controller.FilterResourcesByLabel[*monitoringv1.ServiceMonitor](kuadrant.ObservabilityLabel), )), controller.WithRunnable("podmonitor watcher", controller.Watch( &monitoringv1.PodMonitor{}, monitoringv1.SchemeGroupVersion.WithResource("podmonitors"), metav1.NamespaceAll, - controller.FilterResourcesByLabel[*monitoringv1.PodMonitor]("kuadrant-observability=true"), + controller.FilterResourcesByLabel[*monitoringv1.PodMonitor](kuadrant.ObservabilityLabel), )), controller.WithObjectKinds( schema.GroupKind{Group: monitoringv1.SchemeGroupVersion.Group, Kind: monitoringv1.ServiceMonitorsKind}, @@ -447,7 +447,7 @@ func (b *BootOptionsBuilder) Reconciler() controller.ReconcileFunc { NewTLSWorkflow(b.client, b.manager.GetScheme(), b.isGatewayAPIInstalled, b.isCertManagerInstalled).Run, NewDataPlanePoliciesWorkflow(b.client, b.isGatewayAPIInstalled, b.isIstioInstalled, b.isEnvoyGatewayInstalled, b.isLimitadorOperatorInstalled, b.isAuthorinoOperatorInstalled).Run, NewKuadrantStatusUpdater(b.client, b.isGatewayAPIInstalled, b.isGatewayProviderInstalled(), b.isLimitadorOperatorInstalled, b.isAuthorinoOperatorInstalled).Subscription().Reconcile, - NewObservabilityReconciler(b.client, b.manager.GetRESTMapper()).Subscription().Reconcile, + NewObservabilityReconciler(b.client, b.manager.GetRESTMapper(), operatorNamespace).Subscription().Reconcile, }, Postcondition: finalStepsWorkflow(b.client, b.isGatewayAPIInstalled, b.isIstioInstalled, b.isEnvoyGatewayInstalled).Run, } diff --git a/doc/user-guides/observability/monitors.md b/doc/user-guides/observability/monitors.md index 13fc22403..09d367507 100644 --- a/doc/user-guides/observability/monitors.md +++ b/doc/user-guides/observability/monitors.md @@ -26,8 +26,8 @@ spec: enable: true ``` -When enabled, Kuadrant creates ServiceMonitors and PodMonitors for its own components and in each gateway namespace (Envoy Gateway or Istio). -Monitors are also created in the corresponding gateway "system" namespace: +When enabled, Kuadrant creates ServiceMonitors and PodMonitors for its own components in the same namespae as the Kuadrant operator. +Monitors are also created in each gateway namespace (Envoy Gateway or Istio), and in the corresponding gateway "system" namespace: - Istio: `istio-system` namespace for the istiod pod - Envoy Gateway: `envoy-gateway-system` namespace for the envoy gateway pod @@ -35,10 +35,10 @@ Monitors are also created in the corresponding gateway "system" namespace: You can check all created monitors using this command: ```yaml -kubectl get servicemonitor,podmonitor -A -l kuadrant-observability=true +kubectl get servicemonitor,podmonitor -A -l kuadrant.io/observability=true ``` -You can make changes to the monitors after they are created if you have need to. +You can make changes to the monitors after they are created if you need to. Monitors will only ever be created or deleted, not updated or reverted. If you decide the default monitors aren’t suitable, disable the feature by setting `enable: false` and create your own ServiceMonitor/PodMonitor definitions or configure Prometheus directly. For more details on specific metrics, check out the [Metrics reference page](../../observability/metrics.md). diff --git a/make/development-environments.mk b/make/development-environments.mk index b84ea61f2..8b34a9b51 100644 --- a/make/development-environments.mk +++ b/make/development-environments.mk @@ -18,8 +18,8 @@ install-metallb: kustomize yq ## Installs the metallb load balancer allowing use ./utils/docker-network-ipaddresspool.sh kind $(YQ) ${SUBNET_OFFSET} ${CIDR} ${NUM_IPS} | kubectl apply -n metallb-system -f - .PHONY: install-observability-crds -install-observability-crds: $(KUSTOMIZE) - $(KUSTOMIZE) build ./config/observability/| $(CONTAINER_ENGINE) run --rm -i docker.io/ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f - +install-observability-crds: $(KUSTOMIZE) $(YQ) + $(KUSTOMIZE) build ./config/observability/| $(YQ) e 'select(.kind == "CustomResourceDefinition")' | kubectl apply --server-side -f - .PHONY: uninstall-metallb uninstall-metallb: $(KUSTOMIZE) diff --git a/pkg/kuadrant/kuadrant.go b/pkg/kuadrant/kuadrant.go index 79f3daf05..f3a699e5e 100644 --- a/pkg/kuadrant/kuadrant.go +++ b/pkg/kuadrant/kuadrant.go @@ -7,6 +7,7 @@ import ( const ( ControllerName = "kuadrant.io/policy-controller" TopologyLabel = "kuadrant.io/topology" + ObservabilityLabel = "kuadrant.io/observability" KuadrantRateLimitClusterName = "kuadrant-ratelimit-service" KuadrantAuthClusterName = "kuadrant-auth-service" LimitadorName = "limitador" diff --git a/tests/bare_k8s/observability_reconciler_test.go b/tests/bare_k8s/observability_reconciler_test.go index e74518299..bbc6a6142 100644 --- a/tests/bare_k8s/observability_reconciler_test.go +++ b/tests/bare_k8s/observability_reconciler_test.go @@ -23,6 +23,8 @@ var _ = Describe("Observabiltity monitors for kuadrant components", func() { afterEachTimeOut = NodeTimeout(3 * time.Minute) ) + const kuadrantNamespace = "kuadrant-system" + BeforeEach(func(ctx SpecContext) { testNamespace = tests.CreateNamespace(ctx, testClient()) }) @@ -50,7 +52,7 @@ var _ = Describe("Observabiltity monitors for kuadrant components", func() { }, ObjectMeta: metav1.ObjectMeta{ Name: "kuadrant-operator-monitor", - Namespace: testNamespace, + Namespace: kuadrantNamespace, }, } authorinoMonitor := &monitoringv1.ServiceMonitor{ @@ -60,7 +62,7 @@ var _ = Describe("Observabiltity monitors for kuadrant components", func() { }, ObjectMeta: metav1.ObjectMeta{ Name: "authorino-operator-monitor", - Namespace: testNamespace, + Namespace: kuadrantNamespace, }, } limitadorMonitor := &monitoringv1.ServiceMonitor{ @@ -70,7 +72,7 @@ var _ = Describe("Observabiltity monitors for kuadrant components", func() { }, ObjectMeta: metav1.ObjectMeta{ Name: "limitador-operator-monitor", - Namespace: testNamespace, + Namespace: kuadrantNamespace, }, } dnsMonitor := &monitoringv1.ServiceMonitor{ @@ -80,7 +82,7 @@ var _ = Describe("Observabiltity monitors for kuadrant components", func() { }, ObjectMeta: metav1.ObjectMeta{ Name: "dns-operator-monitor", - Namespace: testNamespace, + Namespace: kuadrantNamespace, }, } @@ -120,22 +122,22 @@ var _ = Describe("Observabiltity monitors for kuadrant components", func() { Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(kuadrantMonitor), kuadrantMonitor) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(kuadrantMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(kuadrantMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(authorinoMonitor), authorinoMonitor) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(authorinoMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(authorinoMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(limitadorMonitor), limitadorMonitor) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(limitadorMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(limitadorMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(dnsMonitor), dnsMonitor) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(dnsMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(dnsMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) // Disable observability feature diff --git a/tests/envoygateway/observability_reconciler_test.go b/tests/envoygateway/observability_reconciler_test.go index 77ea07747..1350c254c 100644 --- a/tests/envoygateway/observability_reconciler_test.go +++ b/tests/envoygateway/observability_reconciler_test.go @@ -124,12 +124,12 @@ var _ = Describe("Observabiltity monitors for envoy gateway", func() { Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(envoyGatewayMonitor), envoyGatewayMonitor) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(envoyGatewayMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(envoyGatewayMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(envoyStatsMonitor), envoyStatsMonitor) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(envoyStatsMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(envoyStatsMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) // Unset observability flag to disable the feature diff --git a/tests/istio/observability_reconciler_test.go b/tests/istio/observability_reconciler_test.go index 9a1959d9f..a5ea0f4e7 100644 --- a/tests/istio/observability_reconciler_test.go +++ b/tests/istio/observability_reconciler_test.go @@ -123,12 +123,12 @@ var _ = Describe("Observabiltity monitors for istio gateway", func() { Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(istiodMonitor), istiodMonitor) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(istiodMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(istiodMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) Eventually(func(g Gomega) { err := testClient().Get(ctx, client.ObjectKeyFromObject(istioPodMonitor), istioPodMonitor) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(istioPodMonitor.Labels).To(HaveKeyWithValue("kuadrant-observability", "true")) + g.Expect(istioPodMonitor.Labels).To(HaveKeyWithValue("kuadrant.io/observability", "true")) }).WithContext(ctx).Should(Succeed()) // Unset observability flag to disable the feature