Allow traffic from cluster nodes to all workloads in the monitoring n…

…amespace
appuio · Dec 11, 2024 · 343aa0f · 343aa0f
1 parent 0fc2476
commit 343aa0f
Show file tree

Hide file tree

Showing 15 changed files with 105 additions and 45 deletions.
diff --git a/component/networkpolicy.libsonnet b/component/networkpolicy.libsonnet
@@ -5,6 +5,8 @@ local kube = import 'lib/kube.libjsonnet';
 local inv = kap.inventory();
 local params = inv.parameters.openshift4_monitoring;
 
+local cilium_cluster = std.member(inv.applications, 'cilium');
+
 [
   kube.NetworkPolicy('alertmanager-allow-web') {
     spec: {
@@ -82,4 +84,25 @@ local params = inv.parameters.openshift4_monitoring;
       },
     },
   },
-]
+] + if cilium_cluster then [
+  // allow all traffic from the cluster nodes, so that the HAproxy ingress can
+  // do healthchecks for routes in the openshift-monitoring namespace.
+  {
+    apiVersion: 'cilium.io/v2',
+    kind: 'CiliumNetworkPolicy',
+    metadata: {
+      name: 'allow-from-cluster-nodes',
+    },
+    spec: {
+      endpointSelector: {},
+      ingress: [
+        {
+          fromEntities: [
+            'host',
+            'remote-node',
+          ],
+        },
+      ],
+    },
+  },
+] else []
diff --git a/...alerts-with-node-labels/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/...alerts-with-node-labels/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1077,13 +1077,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/.../golden/capacity-alerts/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/.../golden/capacity-alerts/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1077,13 +1077,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/custom-rules/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/custom-rules/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1086,14 +1086,15 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubeAggregatedAPIErrors.md
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/...s/golden/ovn-kubernetes/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/...s/golden/ovn-kubernetes/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1228,13 +1228,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/release-4.14/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/release-4.14/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1058,13 +1058,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/release-4.15/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/release-4.15/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1077,13 +1077,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/release-4.16/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/release-4.16/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1083,14 +1083,15 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/KubeAggregatedAPIErrors.md
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/remote-write/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/remote-write/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1077,13 +1077,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/team-routing/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/team-routing/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1132,13 +1132,14 @@ spec:
             syn_team: clumsy-donkeys
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/...ser-workload-monitoring/openshift4-monitoring/openshift4-monitoring/20_networkpolicy.yaml b/...ser-workload-monitoring/openshift4-monitoring/openshift4-monitoring/20_networkpolicy.yaml
@@ -60,3 +60,15 @@ spec:
           - alertmanager
   policyTypes:
     - Ingress
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-from-cluster-nodes
+  namespace: openshift-monitoring
+spec:
+  endpointSelector: {}
+  ingress:
+    - fromEntities:
+        - host
+        - remote-node
diff --git a/...onitoring/openshift4-monitoring/openshift4-monitoring/20_user_workload_networkpolicy.yaml b/...onitoring/openshift4-monitoring/openshift4-monitoring/20_user_workload_networkpolicy.yaml
@@ -60,3 +60,15 @@ spec:
           - alertmanager
   policyTypes:
     - Ingress
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-from-cluster-nodes
+  namespace: openshift-user-workload-monitoring
+spec:
+  endpointSelector: {}
+  ingress:
+    - fromEntities:
+        - host
+        - remote-node
diff --git a/...ser-workload-monitoring/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/...ser-workload-monitoring/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1077,13 +1077,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/golden/vsphere/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml b/tests/golden/vsphere/openshift4-monitoring/openshift4-monitoring/prometheus_rules.yaml
@@ -1077,13 +1077,14 @@ spec:
             syn_component: openshift4-monitoring
         - alert: SYN_KubeAggregatedAPIErrors
           annotations:
-            description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace
-              }} has reported errors. It has appeared unavailable {{ $value | humanize
-              }} times averaged over the past 10m.
+            description: Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name
+              }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster
+              }}.
             summary: Kubernetes aggregated API has reported errors.
             syn_component: openshift4-monitoring
           expr: |
-            sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
+            sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[1m])) > 0
+          for: 10m
           labels:
             severity: warning
             syn: 'true'

diff --git a/tests/user-workload-monitoring.yml b/tests/user-workload-monitoring.yml
@@ -1,3 +1,5 @@
+applications:
+  - cilium
 parameters:
   kapitan:
     dependencies: