Merge pull request #191 from appuio/alertmanager-config-prune-null

Prune fields with `null` values in user-provided Alertmanager config
appuio · Jan 3, 2024 · c0d3e6b · c0d3e6b
2 parents 50fb5d0 + e13c156
commit c0d3e6b
Show file tree

Hide file tree

Showing 14 changed files with 316 additions and 172 deletions.
diff --git a/component/alert-routing-discovery.libsonnet b/component/alert-routing-discovery.libsonnet
@@ -73,7 +73,7 @@ local teamBasedRouting = std.map(
 
 local alertmanagerConfig =
   local routes = std.get(params.openshift4_monitoring.alertManagerConfig.route, 'routes', []);
-  params.openshift4_monitoring.alertManagerConfig {
+  std.prune(params.openshift4_monitoring.alertManagerConfig) {
     receivers+: [ { name: nullReceiver } ],
     route+: {
       routes: adParams.prepend_routes + teamBasedRouting + adParams.append_routes + routes + if ownerOrFallbackTeam != null then [ {

diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc
@@ -268,6 +268,8 @@ A dictionary holding the configuration for the AlertManager.
 
 See the https://docs.openshift.com/container-platform/latest/monitoring/managing-alerts.html#applying-custom-alertmanager-configuration_managing-alerts[OpenShift docs] for available parameters.
 
+The component will silently drop any fields in the provided config which are empty.
+The component treats `null` as empty for scalar fields.
 
 == `alertManagerAutoDiscovery`
 

diff --git a/...alerts-with-node-labels/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/...alerts-with-node-labels/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -856,8 +856,8 @@ spec:
       rules:
         - alert: SYN_KubePersistentVolumeErrors
           annotations:
-            description: The persistent volume {{ $labels.persistentvolume }} has
-              status {{ $labels.phase }}.
+            description: The persistent volume {{ $labels.persistentvolume }} on Cluster
+              {{ $labels.cluster }} has status {{ $labels.phase }}.
             summary: PersistentVolume is having issues with provisioning.
             syn_component: openshift4-monitoring
           expr: |
@@ -870,8 +870,8 @@ spec:
         - alert: SYN_KubePersistentVolumeFillingUp
           annotations:
             description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
-              }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage
-              }} free.
+              }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.cluster
+              }} is only {{ $value | humanizePercentage }} free.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeFillingUp.md
             summary: PersistentVolume is filling up.
             syn_component: openshift4-monitoring
@@ -883,9 +883,9 @@ spec:
             ) < 0.03
             and
             kubelet_volume_stats_used_bytes{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1m
           labels:
@@ -896,8 +896,8 @@ spec:
           annotations:
             description: Based on recent sampling, the PersistentVolume claimed by
               {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace
-              }} is expected to fill up within four days. Currently {{ $value | humanizePercentage
-              }} is available.
+              }} on Cluster {{ $labels.cluster }} is expected to fill up within four
+              days. Currently {{ $value | humanizePercentage }} is available.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeFillingUp.md
             summary: PersistentVolume is filling up.
             syn_component: openshift4-monitoring
@@ -911,9 +911,9 @@ spec:
             kubelet_volume_stats_used_bytes{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
             and
             predict_linear(kubelet_volume_stats_available_bytes{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1h
           labels:
@@ -923,8 +923,8 @@ spec:
         - alert: SYN_KubePersistentVolumeInodesFillingUp
           annotations:
             description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
-              }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage
-              }} free inodes.
+              }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.cluster
+              }} only has {{ $value | humanizePercentage }} free inodes.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeInodesFillingUp.md
             summary: PersistentVolumeInodes are filling up.
             syn_component: openshift4-monitoring
@@ -936,9 +936,9 @@ spec:
             ) < 0.03
             and
             kubelet_volume_stats_inodes_used{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1m
           labels:
@@ -949,8 +949,9 @@ spec:
           annotations:
             description: Based on recent sampling, the PersistentVolume claimed by
               {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace
-              }} is expected to run out of inodes within four days. Currently {{ $value
-              | humanizePercentage }} of its inodes are free.
+              }} on Cluster {{ $labels.cluster }} is expected to run out of inodes
+              within four days. Currently {{ $value | humanizePercentage }} of its
+              inodes are free.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeInodesFillingUp.md
             summary: PersistentVolumeInodes are filling up.
             syn_component: openshift4-monitoring
@@ -964,9 +965,9 @@ spec:
             kubelet_volume_stats_inodes_used{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
             and
             predict_linear(kubelet_volume_stats_inodes_free{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1h
           labels:
@@ -2172,7 +2173,7 @@ spec:
             description: '{{ $value | humanizePercentage }} of reconciling operations
               failed for {{ $labels.controller }} controller in {{ $labels.namespace
               }} namespace.'
-            summary: Errors while reconciling controller.
+            summary: Errors while reconciling objects.
             syn_component: openshift4-monitoring
           expr: |
             (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) > 0.1
@@ -2195,6 +2196,20 @@ spec:
             severity: warning
             syn: 'true'
             syn_component: openshift4-monitoring
+        - alert: SYN_PrometheusOperatorStatusUpdateErrors
+          annotations:
+            description: '{{ $value | humanizePercentage }} of status update operations
+              failed for {{ $labels.controller }} controller in {{ $labels.namespace
+              }} namespace.'
+            summary: Errors while updating objects status.
+            syn_component: openshift4-monitoring
+          expr: |
+            (sum by (controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) > 0.1
+          for: 10m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift4-monitoring
         - alert: SYN_PrometheusOperatorSyncFailed
           annotations:
             description: Controller {{ $labels.controller }} in {{ $labels.namespace

diff --git a/.../golden/capacity-alerts/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/.../golden/capacity-alerts/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -856,8 +856,8 @@ spec:
       rules:
         - alert: SYN_KubePersistentVolumeErrors
           annotations:
-            description: The persistent volume {{ $labels.persistentvolume }} has
-              status {{ $labels.phase }}.
+            description: The persistent volume {{ $labels.persistentvolume }} on Cluster
+              {{ $labels.cluster }} has status {{ $labels.phase }}.
             summary: PersistentVolume is having issues with provisioning.
             syn_component: openshift4-monitoring
           expr: |
@@ -870,8 +870,8 @@ spec:
         - alert: SYN_KubePersistentVolumeFillingUp
           annotations:
             description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
-              }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage
-              }} free.
+              }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.cluster
+              }} is only {{ $value | humanizePercentage }} free.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeFillingUp.md
             summary: PersistentVolume is filling up.
             syn_component: openshift4-monitoring
@@ -883,9 +883,9 @@ spec:
             ) < 0.03
             and
             kubelet_volume_stats_used_bytes{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1m
           labels:
@@ -896,8 +896,8 @@ spec:
           annotations:
             description: Based on recent sampling, the PersistentVolume claimed by
               {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace
-              }} is expected to fill up within four days. Currently {{ $value | humanizePercentage
-              }} is available.
+              }} on Cluster {{ $labels.cluster }} is expected to fill up within four
+              days. Currently {{ $value | humanizePercentage }} is available.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeFillingUp.md
             summary: PersistentVolume is filling up.
             syn_component: openshift4-monitoring
@@ -911,9 +911,9 @@ spec:
             kubelet_volume_stats_used_bytes{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
             and
             predict_linear(kubelet_volume_stats_available_bytes{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1h
           labels:
@@ -923,8 +923,8 @@ spec:
         - alert: SYN_KubePersistentVolumeInodesFillingUp
           annotations:
             description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
-              }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage
-              }} free inodes.
+              }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.cluster
+              }} only has {{ $value | humanizePercentage }} free inodes.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeInodesFillingUp.md
             summary: PersistentVolumeInodes are filling up.
             syn_component: openshift4-monitoring
@@ -936,9 +936,9 @@ spec:
             ) < 0.03
             and
             kubelet_volume_stats_inodes_used{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1m
           labels:
@@ -949,8 +949,9 @@ spec:
           annotations:
             description: Based on recent sampling, the PersistentVolume claimed by
               {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace
-              }} is expected to run out of inodes within four days. Currently {{ $value
-              | humanizePercentage }} of its inodes are free.
+              }} on Cluster {{ $labels.cluster }} is expected to run out of inodes
+              within four days. Currently {{ $value | humanizePercentage }} of its
+              inodes are free.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubePersistentVolumeInodesFillingUp.md
             summary: PersistentVolumeInodes are filling up.
             syn_component: openshift4-monitoring
@@ -964,9 +965,9 @@ spec:
             kubelet_volume_stats_inodes_used{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"} > 0
             and
             predict_linear(kubelet_volume_stats_inodes_free{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_access_mode{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)", access_mode="ReadOnlyMany"} == 1
-            unless on(namespace, persistentvolumeclaim)
+            unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace=~"(appuio.*|cilium|default|kube-.*|openshift-.*|syn.*)",label_alerts_k8s_io_kube_persistent_volume_filling_up="disabled"} == 1
           for: 1h
           labels:
@@ -2172,7 +2173,7 @@ spec:
             description: '{{ $value | humanizePercentage }} of reconciling operations
               failed for {{ $labels.controller }} controller in {{ $labels.namespace
               }} namespace.'
-            summary: Errors while reconciling controller.
+            summary: Errors while reconciling objects.
             syn_component: openshift4-monitoring
           expr: |
             (sum by (controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) > 0.1
@@ -2195,6 +2196,20 @@ spec:
             severity: warning
             syn: 'true'
             syn_component: openshift4-monitoring
+        - alert: SYN_PrometheusOperatorStatusUpdateErrors
+          annotations:
+            description: '{{ $value | humanizePercentage }} of status update operations
+              failed for {{ $labels.controller }} controller in {{ $labels.namespace
+              }} namespace.'
+            summary: Errors while updating objects status.
+            syn_component: openshift4-monitoring
+          expr: |
+            (sum by (controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) / (sum by (controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="prometheus-operator", namespace=~"openshift-monitoring|openshift-user-workload-monitoring"}[5m]))) > 0.1
+          for: 10m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: openshift4-monitoring
         - alert: SYN_PrometheusOperatorSyncFailed
           annotations:
             description: Controller {{ $labels.controller }} in {{ $labels.namespace