Support multiple kafkaClusters in kafka alerts

monitoring/kafka/alerts.test.yaml

@@ -35,6 +35,7 @@ tests:
               description: 'Kafka: Broker count is down'
             exp_labels:
               severity: warning
+              kafkaCluster: ${cluster}
       - alertname: BrokersCountCritical
         eval_time: 3m
         exp_alerts: []
@@ -50,6 +51,7 @@ tests:
             exp_labels:
               namespace: zenko
               service: artesca-data-base-queue
+              kafkaCluster: ${cluster}
               severity: critical
 
   # ActiveControllerCritical
@@ -78,6 +80,7 @@ tests:
               summary: 'Kafka: No active controller'
             exp_labels:
               severity: critical
+              kafkaCluster: ${cluster}
       - alertname: ActiveControllerCritical
         eval_time: 3m
         exp_alerts: []
@@ -108,6 +111,7 @@ tests:
               summary: 'Kafka: 1 under-replicated partitons'
             exp_labels:
               severity: critical
+              kafkaCluster: ${cluster}
       - alertname: UnderReplicatedPartitions
         eval_time: 3m
         exp_alerts:
@@ -119,6 +123,7 @@ tests:
               summary: 'Kafka: 2 under-replicated partitons'
             exp_labels:
               severity: critical
+              kafkaCluster: ${cluster}
 
   # OfflinePartitons
   ##################################################################################################
@@ -147,6 +152,7 @@ tests:
               summary: 'Kafka: 1 offline partitons'
             exp_labels:
               severity: critical
+              kafkaCluster: ${cluster}
       - alertname: OfflinePartitons
         eval_time: 3m
         exp_alerts:
@@ -159,6 +165,7 @@ tests:
               summary: 'Kafka: 2 offline partitons'
             exp_labels:
               severity: critical
+              kafkaCluster: ${cluster}
 
   # RemainingDiskSpaceWarning
   ##################################################################################################
@@ -198,6 +205,7 @@ tests:
               namespace: zenko
               persistentvolumeclaim: artesca-data-base-queue-1
               severity: warning
+              kafkaCluster: ${cluster}
       - alertname: RemainingDiskSpaceWarning
         eval_time: 5d8h
         exp_alerts: []
@@ -225,6 +233,7 @@ tests:
               summary: Zookeeper Sync Disconected
             exp_labels:
               severity: warning
+              kafkaCluster: ${cluster}
 
    # ConsumerLagWarning
   ##################################################################################################
@@ -274,6 +283,7 @@ tests:
               cluster_name: artesca-data-base-queue
               group: notification
               severity: warning
+              kafkaCluster: ${cluster}
       - alertname: ConsumerLagWarning
         eval_time: 20m
         exp_alerts:
@@ -290,6 +300,7 @@ tests:
               cluster_name: artesca-data-base-queue
               group: replication
               severity: warning
+              kafkaCluster: ${cluster}
           - exp_annotations:
               description:  |
                 Kafka consumer lag has been more more than 300 seconds
@@ -303,3 +314,4 @@ tests:
               cluster_name: artesca-data-base-queue
               group: notification
               severity: warning
+              kafkaCluster: ${cluster}

monitoring/kafka/alerts.yaml

@@ -34,6 +34,7 @@ groups:
     for: 1m
     labels:
       severity: warning
+      kafkaCluster: ${cluster}
     annotations:
       summary: 'Not all expected brokers are online.'
       description: 'Kafka: Broker count is down'
@@ -44,6 +45,7 @@ groups:
     for: 1m
     labels:
       severity: critical
+      kafkaCluster: ${cluster}
     annotations:
       summary: 'No Brokers online'
       description: 'Kafka: Broker count is 0'
@@ -53,6 +55,7 @@ groups:
     for: 1m
     labels:
       severity: critical
+      kafkaCluster: ${cluster}
     annotations:
       description: >-
         No broker in the cluster is reporting as the active controller in the last 1 minute interval. During steady state there should
@@ -64,6 +67,7 @@ groups:
     for: 1m
     labels:
       severity: critical
+      kafkaCluster: ${cluster}
     annotations:
       description: >-
         Under-replicated partitions means that one or more replicas are not available. This is usually because a broker is down.  Restart
@@ -75,6 +79,7 @@ groups:
     for: 1m
     labels:
       severity: critical
+      kafkaCluster: ${cluster}
     annotations:
       description: >-
         After successful leader election, if the leader for partition dies, then the partition moves to the OfflinePartition state.
@@ -91,6 +96,7 @@ groups:
     for: 2m
     labels:
       severity: warning
+      kafkaCluster: ${cluster}
     annotations:
       description: 'Kafka Broker has low disk space'
       summary: 'Kafka Broker has low disk space'
@@ -101,6 +107,7 @@ groups:
     for: 1m
     labels:
       severity: warning
+      kafkaCluster: ${cluster}
     annotations:
       summary: 'Zookeeper Sync Disconected'
       description: 'Kafka Zookeeper Sync Disconected'
@@ -116,6 +123,7 @@ groups:
     for: 5m
     labels:
       severity: warning
+      kafkaCluster: ${cluster}
     annotations:
       summary: 'Kafka: consumer lag is too high for {{ $labels.group }}'
       description: |