fix: change headless service to gRPC and expose 9094 TCP

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## master / unreleased
 
+* [CHANGE] Replace `http-metrics` port with `grpc` port #494 
+* [CHANGE] Expose 9094 TCP and UDP for gossip cluster - #494
+  * If the AlertManager headless service existed prior to applying the change, it will have only one port set, which is a known issue. See [kubernetes/kubernetes#39188](https://github.com/kubernetes/kubernetes/issues/39188). Re-creating the headless service can resolve this issue 
+
 ## 2.2.0 / 2024-01-16
 
 * [CHANGE] Removed `config.storage.engine` and any reference of it #488

README.md

@@ -216,6 +216,7 @@ Kubernetes: `^1.19.0-0`
 | compactor.​terminationGracePeriodSeconds | int | `240` |  |
 | compactor.​tolerations | list | `[]` |  |
 | compactor.​topologySpreadConstraints | list | `[]` |  |
+| config.​alertmanager.​cluster | object | `{"listen_address":"0.0.0.0:9094"}` | Disable alertmanager gossip cluster by setting empty listen_address to empty string |
 | config.​alertmanager.​enable_api | bool | `false` | Enable the experimental alertmanager config api. |
 | config.​alertmanager.​external_url | string | `"/api/prom/alertmanager"` |  |
 | config.​api.​prometheus_http_prefix | string | `"/prometheus"` |  |

ci/test-deployment-values.yaml

@@ -125,7 +125,7 @@ runtimeconfigmap:
   annotations:
     foo: bar
 alertmanager:
-  replicas: 1
+  replicas: 3
   statefulSet:
     enabled: false
   extraVolumes:

ci/test-sts-values.yaml

@@ -116,7 +116,7 @@ runtimeconfigmap:
   annotations:
     foo: bar
 alertmanager:
-  replicas: 1
+  replicas: 3
   statefulSet:
     enabled: true
   extraVolumes:

templates/alertmanager/alertmanager-statefulset.yaml

@@ -1,3 +1,5 @@
+{{- $svcClusterAddress := ((.Values.config.alertmanager.cluster).listen_address) | default "0.0.0.0:9094" }}
+{{- $svcClusterPort := (split ":" $svcClusterAddress)._1 }}
 {{- if .Values.alertmanager.enabled -}}
 {{- if .Values.alertmanager.statefulSet.enabled -}}
 apiVersion: apps/v1
@@ -152,6 +154,15 @@ spec:
           args:
             - "-target=alertmanager"
             - "-config.file=/etc/cortex/cortex.yaml"
+            {{- if and (gt (int .Values.alertmanager.replicas) 1) (ne .Values.config.alertmanager.cluster.listen_address "") }}
+            {{- $fullName := include "cortex.alertmanagerFullname" . }}
+            {{- $peers := list }}
+            {{- range $i := until (int .Values.alertmanager.replicas) }}
+	      {{- $peer := printf "%s-%d.%s-headless.%s.svc.cluster.local:%s" $fullName $i $fullName $.Release.Namespace $svcClusterPort }}
+              {{- $peers = append $peers $peer }}
+            {{- end }}
+            - "-alertmanager.cluster.peers={{ join "," $peers }}"
+            {{- end }}
             {{- range $key, $value := .Values.alertmanager.extraArgs }}
             - "-{{ $key }}={{ $value }}"
             {{- end }}
@@ -175,6 +186,15 @@ spec:
             - name: gossip
               containerPort: {{ .Values.config.memberlist.bind_port }}
               protocol: TCP
+            - name: grpc
+              containerPort: {{ .Values.config.server.grpc_listen_port }}
+              protocol: TCP
+            - containerPort: {{ $svcClusterPort }}
+              name: alert-clu-tcp
+              protocol: TCP
+            - containerPort: {{ $svcClusterPort }}
+              name: alert-clu-udp
+              protocol: UDP
           startupProbe:
             {{- toYaml .Values.alertmanager.startupProbe | nindent 12 }}
           livenessProbe:

templates/alertmanager/alertmanager-svc-headless.yaml

@@ -1,5 +1,4 @@
 {{- if .Values.alertmanager.enabled -}}
-{{- if .Values.alertmanager.statefulSet.enabled -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -17,11 +16,18 @@ spec:
   clusterIP: None
   publishNotReadyAddresses: true
   ports:
-    - port: {{ .Values.config.server.http_listen_port }}
+    - port: {{ .Values.config.server.grpc_listen_port }}
       protocol: TCP
-      name: http-metrics
-      targetPort: http-metrics
+      name: grpc
+      targetPort: grpc
+    - port: 9094
+      protocol: UDP
+      name: alert-clu-udp
+      targetPort: alert-clu-udp
+    - port: 9094
+      protocol: TCP
+      name: alert-clu-tcp
+      targetPort: alert-clu-tcp
   selector:
     {{- include "cortex.alertmanagerSelectorLabels" . | nindent 4 }}
 {{- end -}}
-{{- end -}}

values.yaml

@@ -126,6 +126,10 @@ config:
   runtime_config:
     file: /etc/cortex-runtime-config/runtime_config.yaml
   alertmanager:
+    # -- Enable alertmanager gossip cluster
+    # -- Disable alertmanager gossip cluster by setting empty listen_address to empty string
+    cluster:
+      listen_address: '0.0.0.0:9094'
     # -- Enable the experimental alertmanager config api.
     enable_api: false
     external_url: '/api/prom/alertmanager'