From 2bbf0a3fa799b52fa64ae51aa4465df90c8dc325 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 12 Jan 2023 15:31:01 -0800
Subject: [PATCH 01/35] tezos signer forwarder chart

The last remaining piece of https://github.com/midl-dev/tezos-on-gke/ to
move into tezos-k8s, tezos-signer-forwarder is a terminating pod for ssh
tunnels exposing a tezos signing endpoint from an on-prem location.
---
 charts/tezos-signer-forwarder/Chart.yaml      | 24 ++++++
 .../scripts/entrypoint.sh                     |  3 +
 .../scripts/signer_exporter.py                | 28 +++++++
 .../templates/_helpers.tpl                    | 62 ++++++++++++++
 .../templates/alertmanagerconfig.yaml         | 54 ++++++++++++
 .../templates/config.yaml                     | 10 +++
 .../templates/prometheusrule.yaml             | 51 ++++++++++++
 .../templates/secret.yaml                     |  7 ++
 .../templates/service.yaml                    | 37 +++++++++
 .../templates/servicemonitor.yaml             | 38 +++++++++
 .../templates/statefulset.yaml                | 81 ++++++++++++++++++
 charts/tezos-signer-forwarder/values.yaml     | 82 +++++++++++++++++++
 signerForwarder/Dockerfile                    | 23 ++++++
 13 files changed, 500 insertions(+)
 create mode 100644 charts/tezos-signer-forwarder/Chart.yaml
 create mode 100644 charts/tezos-signer-forwarder/scripts/entrypoint.sh
 create mode 100644 charts/tezos-signer-forwarder/scripts/signer_exporter.py
 create mode 100644 charts/tezos-signer-forwarder/templates/_helpers.tpl
 create mode 100644 charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
 create mode 100644 charts/tezos-signer-forwarder/templates/config.yaml
 create mode 100644 charts/tezos-signer-forwarder/templates/prometheusrule.yaml
 create mode 100644 charts/tezos-signer-forwarder/templates/secret.yaml
 create mode 100644 charts/tezos-signer-forwarder/templates/service.yaml
 create mode 100644 charts/tezos-signer-forwarder/templates/servicemonitor.yaml
 create mode 100644 charts/tezos-signer-forwarder/templates/statefulset.yaml
 create mode 100644 charts/tezos-signer-forwarder/values.yaml
 create mode 100644 signerForwarder/Dockerfile

diff --git a/charts/tezos-signer-forwarder/Chart.yaml b/charts/tezos-signer-forwarder/Chart.yaml
new file mode 100644
index 000000000..00db4f455
--- /dev/null
+++ b/charts/tezos-signer-forwarder/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: tezos-signer-forwarder
+description: A chart for tezos-signer-forwarder
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "10.0"
diff --git a/charts/tezos-signer-forwarder/scripts/entrypoint.sh b/charts/tezos-signer-forwarder/scripts/entrypoint.sh
new file mode 100644
index 000000000..41233320e
--- /dev/null
+++ b/charts/tezos-signer-forwarder/scripts/entrypoint.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+/usr/sbin/sshd -D -e -p ${TUNNEL_ENDPOINT_PORT}
diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
new file mode 100644
index 000000000..4ded9a154
--- /dev/null
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import os
+from flask import Flask, request, jsonify
+import requests
+
+import logging
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR)
+
+application = Flask(__name__)
+
+readiness_probe_path = os.getenv("READINESS_PROBE_PATH")
+
+@application.route('/metrics', methods=['GET'])
+def prometheus_metrics():
+    '''
+    Prometheus endpoint
+    '''
+    try:
+        probe = requests.get(f"http://localhost:8443{readiness_probe_path}")
+    except requests.exceptions.RequestException:
+        probe = None
+    return f'''# number of unhealthy signers - should be 0 or 1
+unhealthy_signers_total {0 if probe else 1}
+'''
+
+if __name__ == "__main__":
+   application.run(host = "0.0.0.0", port = 31732, debug = False)
diff --git a/charts/tezos-signer-forwarder/templates/_helpers.tpl b/charts/tezos-signer-forwarder/templates/_helpers.tpl
new file mode 100644
index 000000000..c3113e365
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/_helpers.tpl
@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "tezos-signer-forwarder.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "tezos-signer-forwarder.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name $.Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" $.Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "tezos-signer-forwarder.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "tezos-signer-forwarder.labels" -}}
+helm.sh/chart: {{ include "tezos-signer-forwarder.chart" . }}
+{{ include "tezos-signer-forwarder.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "tezos-signer-forwarder.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "tezos-signer-forwarder.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "tezos-signer-forwarder.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "tezos-signer-forwarder.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
new file mode 100644
index 000000000..d5170fd4b
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
@@ -0,0 +1,54 @@
+{{- if .Values.alertmanagerConfig.enabled }}
+{{- range .Values.endpoints }}
+{{- if .monitoring_email }}
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: tezos-remote-signer-alerts-{{ .name }}
+  labels:
+{{- toYaml $.Values.alertmanagerConfig.labels | nindent 4 }}
+spec:
+  route:
+    groupBy: ['job']
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 12h
+    receiver: 'email_{{ .name }}'
+    matchers:
+    - name: service
+      value: tezos-remote-signer-{{ .name }}
+      regex: false
+    - name: alertType
+      value: tezos-remote-signer-alert
+      regex: false
+    continue: false
+  receivers:
+  - name: 'email_{{ .name }}'
+    emailConfigs:
+    - to: "{{ .monitoring_email }}"
+      sendResolved: true
+      headers:
+      - key: subject
+        value: '{{`[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}`}}'
+      html: >-
+        {{`{{ if eq .Status "firing" }}
+        Your attention is required regarding the following Tezos Remote Signer alert:
+        {{ else }}
+        The following Tezos Remote Signer Alert is resolved:
+        {{ end }}
+        {{ range .Alerts -}}
+        {{ .Annotations.summary }}
+        {{ end }}`}}
+      text: >-
+        {{`{{ if eq .Status "firing" }}
+        Your attention is required regarding the following Tezos Remote Signer alert:
+        {{ else }}
+        The following Tezos Remote Signer Alert is resolved:
+        {{ end }}
+        {{ range .Alerts -}}
+        {{ .Annotations.summary }}
+        {{ end }}`}}
+---
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/config.yaml b/charts/tezos-signer-forwarder/templates/config.yaml
new file mode 100644
index 000000000..0fb534f96
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/config.yaml
@@ -0,0 +1,10 @@
+{{- range .Values.endpoints }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tezos-signer-forwarder-config-{{ .name }}
+data:
+  authorized_keys: "{{ .ssh_pubkey }} signer"
+  tunnel_endpoint_port: "{{ .tunnel_endpoint_port }}"
+---
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
new file mode 100644
index 000000000..e6bafc3ab
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
@@ -0,0 +1,51 @@
+{{- if .Values.prometheusRule.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+{{- toYaml .Values.prometheusRule.labels | nindent 4 }}
+  name: tezos-remote-signer-rules
+spec:
+  groups:
+  - name: tezos-remote-signer.rules
+    rules:
+    - alert: SignerPowerLoss
+      annotations:
+        description: 'Remote signer has lost power'
+        summary: Tezos remote signer has lost power
+      expr: power{namespace="{{ .Release.Namespace }}"} != 0
+      for: 1m
+      labels:
+        severity: critical
+        alertType: tezos-remote-signer-alert
+    - alert: SignerWiredNetworkLoss
+      annotations:
+        description: 'Remote signer has lost wired internet connection'
+        summary: Tezos remote signer has lost wired internet connection
+      expr: wired_network{namespace="{{ .Release.Namespace }}"} != 0
+      for: 1m
+      labels:
+        severity: critical
+        alertType: tezos-remote-signer-alert
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+{{- toYaml .Values.prometheusRule.labels | nindent 4 }}
+  name: tezos-remote-signer-reachability-rules
+spec:
+  groups:
+  - name: tezos-remote-signer.rules
+    rules:
+    - alert: NoRemoteSigner
+      annotations:
+        description: 'Remote signer is down'
+        summary: Remote signer is down or unable to sign.
+      expr: unhealthy_signers_total{namespace="{{ .Release.Namespace }}"} != 0
+      for: 1m
+      labels:
+        severity: critical
+        alertType: tezos-remote-signer-alert
+---
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/secret.yaml b/charts/tezos-signer-forwarder/templates/secret.yaml
new file mode 100644
index 000000000..b330b7cdb
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/secret.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: tezos-signer-forwarder-secret-{{ .Values.name }}
+data:
+  ssh_host_ecdsa_key: |
+{{ println .Values.secrets.signer_target_host_key | b64enc | indent 4 -}}
diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
new file mode 100644
index 000000000..efdf727b7
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: tezos-remote-signer-ssh-ingress-{{ .Values.name }}
+  annotations:
+{{ toYaml .Values.service_annotations | indent 4 }}
+spec:
+  type: LoadBalancer
+  selector:
+    app.kubernetes.io/name: tezos-signer-forwarder
+  ports:
+{{- range .Values.endpoints }}
+  - port: {{ .tunnel_endpoint_port }}
+    name: tunnel-{{ .name }}
+{{- end }}
+  # ensures that remote signers can always ssh
+  publishNotReadyAddresses: true
+---
+{{- range .Values.endpoints }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: tezos-remote-signer-{{ .name }}
+  labels:
+    app.kubernetes.io/name: tezos-signer-forwarder
+    app.kubernetes.io/baker-name: {{ .name }}
+spec:
+  selector:
+    app.kubernetes.io/name: tezos-signer-forwarder
+    app.kubernetes.io/baker-name: {{ .name }}
+  ports:
+  - port: 8443
+    name: signer
+  - port: 31732
+    name: metrics
+---
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
new file mode 100644
index 000000000..a86932569
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -0,0 +1,38 @@
+{{- if .Values.serviceMonitor.enabled }}
+{{- range .Values.endpoints }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: tezos-signer-forwarder
+  name: tezos-remote-signer-monitoring-{{ .name }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  endpoints:
+  - interval: 20s
+    port: signer
+    path: /healthz
+    scrapeTimeout: 20s
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tezos-signer-forwarder
+      app.kubernetes.io/baker-name: {{ .name }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/name: tezos-signer-forwarder
+  name: tezos-remote-signer-reachability-{{ .name }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  endpoints:
+  - port: metrics
+    path: /metrics
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tezos-signer-forwarder
+      app.kubernetes.io/baker-name: {{ .name }}
+---
+{{- end }}
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
new file mode 100644
index 000000000..fd7a6cab3
--- /dev/null
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -0,0 +1,81 @@
+{{- range .Values.endpoints }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: tezos-signer-forwarder-{{ .name}}
+  annotations:
+    "pulumi.com/skipAwait": "true"
+spec:
+  replicas: 1
+  serviceName: tezos-remote-signer-{{ .name }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tezos-signer-forwarder
+  template:
+    metadata:
+      {{- with $.Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        app.kubernetes.io/name: tezos-signer-forwarder
+        app.kubernetes.io/baker-name: {{ .name }}
+    spec:
+      volumes:
+      - name: config-volume
+        configMap:
+           name: tezos-signer-forwarder-config-{{ .name }}
+      - name: secret-volume
+        secret:
+           secretName: tezos-signer-forwarder-secret-{{ $.Values.name }}
+           defaultMode: 0400
+      containers:
+      - name: tezos-signer-forwarder
+        image: {{ $.Values.images.tezos_signer_forwarder }}
+        imagePullPolicy: IfNotPresent
+        command:
+          - /bin/sh
+        args:
+          - "-c"
+          - |
+{{ tpl ($.Files.Get (print "scripts/entrypoint.sh")) $ | indent 12 }}
+        volumeMounts:
+        - name: config-volume
+          mountPath: /home/signer/.ssh/authorized_keys
+          subPath: authorized_keys
+        - name: secret-volume
+          mountPath: /etc/ssh/ssh_host_ecdsa_key
+          subPath: ssh_host_ecdsa_key
+        env:
+        - name: TUNNEL_ENDPOINT_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: tezos-signer-forwarder-config-{{ .name }}
+              key: tunnel_endpoint_port
+        ports:
+        - name: signer
+          containerPort: 8443
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: {{ .readiness_probe_path }}
+            port: 8443
+      - name: prom-exporter
+        image: {{ $.Values.tezos_k8s_images.utils }}
+        ports:
+          - name: metrics
+            containerPort: 31732
+            protocol: TCP
+        env:
+        - name: READINESS_PROBE_PATH
+          value: {{ .readiness_probe_path | quote }}
+        command:
+          - /usr/local/bin/python
+        args:
+          - "-c"
+          - |
+{{ tpl ($.Files.Get (print "scripts/signer_exporter.py")) $ | indent 12 }}
+      nodeSelector:
+ {{ toYaml $.Values.node_selector | indent 8 }}
+---
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
new file mode 100644
index 000000000..a2866c15a
--- /dev/null
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -0,0 +1,82 @@
+images:
+  tezos_signer_forwarder: localhost/tezos-k8s-signerforwarder:dev
+tezos_k8s_images:
+  utils: ghcr.io/oxheadalpha/tezos-k8s-utils:master
+
+# List the endpoints below.
+# Each endpoint represents a ssh server.
+# To handle several endpoints, you can either:
+# * instantiate several replicas of this chart, or
+# * list several endpoints below.
+
+# Since this chart instantiates a service of type Loadbalancer,
+# it may be the case that each such service comes with its own
+# auto-assigned IP, increasing costs.
+# Listing several endpoints below will put all
+# associated pods behind the same LoadBalancer service.
+# Consequenty, the same IP will be re-used between signers.
+# If you prefer to have one IP per signer, instantiate this chart
+# several times.
+endpoints:
+  # the public key that the server is expecting.
+  # The signer should authenticate with the corresponding secret key.
+- ssh_pubkey: "ssh-rsa AAAA...."
+  
+  # ssh tunnel connection establishes to this port
+  tunnel_endpoint_port: 50000
+
+  # endpoint name - to disambiguate them
+  name: myendpoint
+
+  # Set a readiness probe path for your signer.
+  # By default, it is the known path implemented by every signer "/authorized_keys"
+  # When using tezos-remote-signer-os, you can set it to a path that performs more
+  # checks, such as:
+  #  "/statusz/${PUBLIC_BAKING_KEY_HASH}?ledger_url=${LEDGER_AUTHORIZED_PATH_ENCODED}"
+  readiness_probe_path: /authorized_keys
+
+  # Enter email address to send alerts to.
+  # monitoring_email:
+
+# Name that goes into the service
+# e.g tezos-signer-mybaker
+# useful when one baker bakes for several addresses
+# on different remote signers.
+name: mybaker
+
+# to deploy in a specific node pool, put label here
+node_selector: {}
+
+# LoadBalancer service annotations. On some cloud providers, it can
+# be used to assign a static ip address.
+service_annotations: {}
+
+secrets:
+  # The ssh host key must be passed as input.
+  # Otherwise, when destroying and respinning the infra,
+  # the signer would not recognize the host and refuse to
+  # connect.
+  signer_target_host_key: |
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    xxx
+    -----END OPENSSH PRIVATE KEY-----
+
+# Prometheus Operator is required in your cluster in order to enable
+# serviceMonitor and prometheusRule below.
+# Enable service monitor to scrape the /healthz endpoint of your
+# remote signer.
+# The /healthz endpoint is exposed by tezos-remote-signer-os:
+#    https://github.com/midl-dev/tezos-remote-signer-os
+serviceMonitor:
+  enabled: false
+# Enable Prometheus Rule to be alerted when your hardware remote signer
+# provisioned with tezos-remote-signer-os loses power or wired network
+# connectivity.
+# For rules to be picked up by the Prometheus Operator, proper labels need
+# to be set below. Refer to Prometheus operator documentation for details.
+prometheusRule:
+  enabled: false
+  labels: {}
+alertmanagerConfig:
+  enabled: false
+  labels: {}
diff --git a/signerForwarder/Dockerfile b/signerForwarder/Dockerfile
new file mode 100644
index 000000000..d1a692b4a
--- /dev/null
+++ b/signerForwarder/Dockerfile
@@ -0,0 +1,23 @@
+FROM alpine:edge
+
+# add openssh and clean
+RUN apk add --no-cache openssh shadow
+
+RUN adduser --system signer
+# * is the hash of the password. Effectively, password login is disabled.
+# but I need to do that otherwise sshd says account is locked. see:
+#  https://unix.stackexchange.com/a/193131/81131
+RUN usermod -p '*' signer
+
+#allow forwarding
+RUN sed -ri 's/^.*GatewayPorts.*$/GatewayPorts yes/g' /etc/ssh/sshd_config
+RUN sed -ri 's/^.*AllowTcpForwarding.*$/AllowTcpForwarding yes/g' /etc/ssh/sshd_config
+RUN sed -ri 's/^.*PasswordAuthentication.*$/PasswordAuthentication no/g' /etc/ssh/sshd_config
+RUN sed -ri 's/^.*ClientAliveInterval.*$/ClientAliveInterval 10/g' /etc/ssh/sshd_config
+RUN sed -ri 's/^.*ClientAliveCountMax.*$/ClientAliveCountMax 2/g' /etc/ssh/sshd_config
+RUN printf "AllowUsers signer\n" >> /etc/ssh/sshd_config
+RUN cat /etc/ssh/sshd_config
+
+RUN mkdir /home/signer/.ssh && chown -R signer /home/signer
+
+CMD ["/usr/sbin/sshd"]

From 035ee60be91fb913e9d49e6bb9285e8f5e0a6222 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 9 Feb 2023 12:23:30 -0800
Subject: [PATCH 02/35] support for HA signers

---
 .../templates/alertmanagerconfig.yaml         |  2 +-
 .../templates/config.yaml                     | 13 +++++++----
 .../templates/service.yaml                    | 15 ++++++++----
 .../templates/servicemonitor.yaml             |  2 +-
 .../templates/statefulset.yaml                | 22 +++++++++++-------
 charts/tezos-signer-forwarder/values.yaml     | 23 +++++++++++--------
 6 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
index d5170fd4b..3cdfbf56d 100644
--- a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
+++ b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.alertmanagerConfig.enabled }}
-{{- range .Values.endpoints }}
+{{- range .Values.signers }}
 {{- if .monitoring_email }}
 apiVersion: monitoring.coreos.com/v1alpha1
 kind: AlertmanagerConfig
diff --git a/charts/tezos-signer-forwarder/templates/config.yaml b/charts/tezos-signer-forwarder/templates/config.yaml
index 0fb534f96..c64c86773 100644
--- a/charts/tezos-signer-forwarder/templates/config.yaml
+++ b/charts/tezos-signer-forwarder/templates/config.yaml
@@ -1,10 +1,15 @@
-{{- range .Values.endpoints }}
+{{- range .Values.signers }}
+{{- $name := .name }}
+{{- $signers := . }}
+{{- range $i := until (len .endpoints) }}
+{{- $endpoint := index $signers.endpoints $i }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: tezos-signer-forwarder-config-{{ .name }}
+  name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
 data:
-  authorized_keys: "{{ .ssh_pubkey }} signer"
-  tunnel_endpoint_port: "{{ .tunnel_endpoint_port }}"
+  authorized_keys: "{{ $endpoint.ssh_pubkey }} signer"
+  tunnel_endpoint_port: "{{ $endpoint.tunnel_endpoint_port }}"
 ---
 {{- end }}
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index efdf727b7..a419af86a 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -9,14 +9,19 @@ spec:
   selector:
     app.kubernetes.io/name: tezos-signer-forwarder
   ports:
-{{- range .Values.endpoints }}
-  - port: {{ .tunnel_endpoint_port }}
-    name: tunnel-{{ .name }}
+{{- range .Values.signers }}
+{{- $name := .name }}
+{{- $signers := . }}
+{{- range $i := until (len .endpoints) }}
+{{- $endpoint := index $signers.endpoints $i }}
+  - port: {{ $endpoint.tunnel_endpoint_port }}
+    name: tunnel-{{ $name }}-{{ $i }}
+{{- end }}
 {{- end }}
   # ensures that remote signers can always ssh
   publishNotReadyAddresses: true
 ---
-{{- range .Values.endpoints }}
+{{- range .Values.signers }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -33,5 +38,7 @@ spec:
     name: signer
   - port: 31732
     name: metrics
+  # make sure that the service always targets the same signer, when HA is in use.
+  sessionAffinity: ClientIP
 ---
 {{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index a86932569..de887f432 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.serviceMonitor.enabled }}
-{{- range .Values.endpoints }}
+{{- range .Values.signers }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index fd7a6cab3..1d032e258 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -1,13 +1,18 @@
-{{- range .Values.endpoints }}
+{{- range .Values.signers }}
+{{- $name := .name }}
+{{- $readiness_probe_path := .readiness_probe_path }}
+{{- $signers := . }}
+{{- range $i := until (len .endpoints) }}
+{{- $endpoint := index $signers.endpoints $i }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: tezos-signer-forwarder-{{ .name}}
+  name: tezos-signer-forwarder-{{ $name}}-{{ $i }}
   annotations:
     "pulumi.com/skipAwait": "true"
 spec:
   replicas: 1
-  serviceName: tezos-remote-signer-{{ .name }}
+  serviceName: tezos-remote-signer-{{ $name }}
   selector:
     matchLabels:
       app.kubernetes.io/name: tezos-signer-forwarder
@@ -19,12 +24,12 @@ spec:
       {{- end }}
       labels:
         app.kubernetes.io/name: tezos-signer-forwarder
-        app.kubernetes.io/baker-name: {{ .name }}
+        app.kubernetes.io/baker-name: {{ $name }}
     spec:
       volumes:
       - name: config-volume
         configMap:
-           name: tezos-signer-forwarder-config-{{ .name }}
+           name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
       - name: secret-volume
         secret:
            secretName: tezos-signer-forwarder-secret-{{ $.Values.name }}
@@ -50,7 +55,7 @@ spec:
         - name: TUNNEL_ENDPOINT_PORT
           valueFrom:
             configMapKeyRef:
-              name: tezos-signer-forwarder-config-{{ .name }}
+              name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
               key: tunnel_endpoint_port
         ports:
         - name: signer
@@ -58,7 +63,7 @@ spec:
           protocol: TCP
         readinessProbe:
           httpGet:
-            path: {{ .readiness_probe_path }}
+            path: {{ $readiness_probe_path }}
             port: 8443
       - name: prom-exporter
         image: {{ $.Values.tezos_k8s_images.utils }}
@@ -68,7 +73,7 @@ spec:
             protocol: TCP
         env:
         - name: READINESS_PROBE_PATH
-          value: {{ .readiness_probe_path | quote }}
+          value: {{ $readiness_probe_path | quote }}
         command:
           - /usr/local/bin/python
         args:
@@ -79,3 +84,4 @@ spec:
  {{ toYaml $.Values.node_selector | indent 8 }}
 ---
 {{- end }}
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index a2866c15a..2b4c7072d 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -12,21 +12,24 @@ tezos_k8s_images:
 # Since this chart instantiates a service of type Loadbalancer,
 # it may be the case that each such service comes with its own
 # auto-assigned IP, increasing costs.
-# Listing several endpoints below will put all
+# Listing several signers below will put all
 # associated pods behind the same LoadBalancer service.
 # Consequenty, the same IP will be re-used between signers.
 # If you prefer to have one IP per signer, instantiate this chart
 # several times.
-endpoints:
-  # the public key that the server is expecting.
-  # The signer should authenticate with the corresponding secret key.
-- ssh_pubkey: "ssh-rsa AAAA...."
-  
-  # ssh tunnel connection establishes to this port
-  tunnel_endpoint_port: 50000
+signers:
+  # signer name - to disambiguate them
+- name: mysigner
 
-  # endpoint name - to disambiguate them
-  name: myendpoint
+  # Signer may have several endpoints: signer replicas signing for the same key.
+  # This is useful for a highly-available setup.
+  endpoints:
+    # the public key that the server is expecting.
+    # The signer should authenticate with the corresponding secret key.
+  - ssh_pubkey: "ssh-rsa AAAA...."
+
+    # ssh tunnel connection establishes to this port
+    tunnel_endpoint_port: 50000
 
   # Set a readiness probe path for your signer.
   # By default, it is the known path implemented by every signer "/authorized_keys"

From 5f9918bcc83f4625cd11e1b27980f894d08a8582 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Sat, 18 Feb 2023 11:02:42 -0800
Subject: [PATCH 03/35] support for loadbalancerip instead of annotation

---
 charts/tezos-signer-forwarder/templates/service.yaml | 3 +++
 charts/tezos-signer-forwarder/values.yaml            | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index a419af86a..a385f8d66 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -20,6 +20,9 @@ spec:
 {{- end }}
   # ensures that remote signers can always ssh
   publishNotReadyAddresses: true
+{{ if .Values.load_balancer_ip }}
+  loadBalancerIP: {{ .Values.load_balancer_ip }}
+{{ end }}
 ---
 {{- range .Values.signers }}
 apiVersion: v1
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 2b4c7072d..0471d78c1 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -54,6 +54,10 @@ node_selector: {}
 # be used to assign a static ip address.
 service_annotations: {}
 
+# Load Balancer IP to set the ssh service.
+# In some cloud providers, it is used to assign static ip.
+#load_balancer_ip: 
+
 secrets:
   # The ssh host key must be passed as input.
   # Otherwise, when destroying and respinning the infra,

From 51fff35464bd71308786c212c772b0e019c48147 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 23 Feb 2023 21:59:48 -0800
Subject: [PATCH 04/35] instead of 2 service monitors, relabel the alerts from
 signer enable cold standby

---
 .../scripts/signer_exporter.py                | 37 +++++++++++++++++--
 .../templates/alertmanagerconfig.yaml         | 27 +++++++++-----
 .../templates/prometheusrule.yaml             | 12 +++---
 .../templates/servicemonitor.yaml             | 18 ---------
 .../templates/statefulset.yaml                |  5 +++
 charts/tezos-signer-forwarder/values.yaml     |  6 +++
 6 files changed, 69 insertions(+), 36 deletions(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index 4ded9a154..028c460be 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -2,6 +2,7 @@
 import os
 from flask import Flask, request, jsonify
 import requests
+import re
 
 import logging
 log = logging.getLogger('werkzeug')
@@ -10,19 +11,49 @@
 application = Flask(__name__)
 
 readiness_probe_path = os.getenv("READINESS_PROBE_PATH")
+endpoint_alias = os.getenv("ENDPOINT_ALIAS")
+baker_alias = os.getenv("BAKER_ALIAS")
+
+def relabel(prometheus_metrics,extra_labels):
+    '''
+    Add labels to existing prometheus_metrics
+    '''
+    relabeled_metrics = ""
+    for line in prometheus_metrics.splitlines():
+        if line.startswith("#"):
+            relabeled_metrics += line + "\n"
+        else:
+            if "{" in line:
+                # line has labels 
+                labeled_line = re.sub(r'{(.*)}', r'{\1,%s}' % extra_labels, line)
+            else:
+                labeled_line = re.sub(r'(.*) ', r'\1{%s} ' % extra_labels, line)
+            relabeled_metrics += labeled_line + "\n"
+    return relabeled_metrics
 
 @application.route('/metrics', methods=['GET'])
 def prometheus_metrics():
     '''
     Prometheus endpoint
+    Combines the readiness probe URL (i.e. checks that ledger is configured)
+    and the health probe (check power status for example)
     '''
+    extra_labels = 'midl_endpoint_alias="%s",midl_baker_alias="%s"' % (endpoint_alias, baker_alias)
+
     try:
         probe = requests.get(f"http://localhost:8443{readiness_probe_path}")
     except requests.exceptions.RequestException:
         probe = None
-    return f'''# number of unhealthy signers - should be 0 or 1
-unhealthy_signers_total {0 if probe else 1}
-'''
+    if probe:
+        try:
+            healthz = relabel(requests.get(f"http://localhost:8443/healthz").text, extra_labels)
+        except requests.exceptions.RequestException:
+            healthz = None
+    else:
+        healthz = None
+    return '''# number of unhealthy signers - should be 0 or 1
+unhealthy_signers_total{%s} %s
+%s''' % (extra_labels, 0 if probe else 1, healthz or "")
 
 if __name__ == "__main__":
    application.run(host = "0.0.0.0", port = 31732, debug = False)
diff --git a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
index 3cdfbf56d..73146cb61 100644
--- a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
+++ b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
@@ -1,10 +1,13 @@
 {{- if .Values.alertmanagerConfig.enabled }}
 {{- range .Values.signers }}
 {{- if .monitoring_email }}
+{{ $signer := . }}
+{{- range .endpoints }}
+{{- if .alert_when_down }}
 apiVersion: monitoring.coreos.com/v1alpha1
 kind: AlertmanagerConfig
 metadata:
-  name: tezos-remote-signer-alerts-{{ .name }}
+  name: tezos-signer-{{ $signer.name }}-{{ .alias }}-email
   labels:
 {{- toYaml $.Values.alertmanagerConfig.labels | nindent 4 }}
 spec:
@@ -13,37 +16,41 @@ spec:
     groupWait: 30s
     groupInterval: 5m
     repeatInterval: 12h
-    receiver: 'email_{{ .name }}'
+    receiver: 'email_{{ $signer.name }}'
     matchers:
     - name: service
-      value: tezos-remote-signer-{{ .name }}
+      value: tezos-remote-signer-{{ $signer.name }}
       regex: false
     - name: alertType
       value: tezos-remote-signer-alert
       regex: false
+    - name: midl_endpoint_alias
+      value: {{ .alias }}
+      regex: false
     continue: false
+
   receivers:
-  - name: 'email_{{ .name }}'
+  - name: 'email_{{ $signer.name }}'
     emailConfigs:
-    - to: "{{ .monitoring_email }}"
+    - to: "{{ $signer.monitoring_email }}"
       sendResolved: true
       headers:
       - key: subject
         value: '{{`[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}`}}'
       html: >-
         {{`{{ if eq .Status "firing" }}
-        Your attention is required regarding the following Tezos Remote Signer alert:
+        Attention Required for Tezos Remote Signer:
         {{ else }}
-        The following Tezos Remote Signer Alert is resolved:
+        Resolved Alert for Tezos Remote Signer:
         {{ end }}
         {{ range .Alerts -}}
         {{ .Annotations.summary }}
         {{ end }}`}}
       text: >-
         {{`{{ if eq .Status "firing" }}
-        Your attention is required regarding the following Tezos Remote Signer alert:
+        Attention Required for Tezos Remote Signer:
         {{ else }}
-        The following Tezos Remote Signer Alert is resolved:
+        Resolved Alert for Tezos Remote Signer:
         {{ end }}
         {{ range .Alerts -}}
         {{ .Annotations.summary }}
@@ -52,3 +59,5 @@ spec:
 {{- end }}
 {{- end }}
 {{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
index e6bafc3ab..a59c073ca 100644
--- a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
+++ b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
@@ -11,8 +11,8 @@ spec:
     rules:
     - alert: SignerPowerLoss
       annotations:
-        description: 'Remote signer has lost power'
-        summary: Tezos remote signer has lost power
+        description: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost power'
+        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost power'
       expr: power{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
@@ -20,8 +20,8 @@ spec:
         alertType: tezos-remote-signer-alert
     - alert: SignerWiredNetworkLoss
       annotations:
-        description: 'Remote signer has lost wired internet connection'
-        summary: Tezos remote signer has lost wired internet connection
+        description: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost wired internet connection'
+        summary: 'Tezos remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost wired internet connection'
       expr: wired_network{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
@@ -40,8 +40,8 @@ spec:
     rules:
     - alert: NoRemoteSigner
       annotations:
-        description: 'Remote signer is down'
-        summary: Remote signer is down or unable to sign.
+        description: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" is down'
+        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" is down or unable to sign.'
       expr: unhealthy_signers_total{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index de887f432..68c9b97c1 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -7,24 +7,6 @@ metadata:
     app.kubernetes.io/name: tezos-signer-forwarder
   name: tezos-remote-signer-monitoring-{{ .name }}
   namespace: {{ $.Release.Namespace }}
-spec:
-  endpoints:
-  - interval: 20s
-    port: signer
-    path: /healthz
-    scrapeTimeout: 20s
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: tezos-signer-forwarder
-      app.kubernetes.io/baker-name: {{ .name }}
----
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app.kubernetes.io/name: tezos-signer-forwarder
-  name: tezos-remote-signer-reachability-{{ .name }}
-  namespace: {{ $.Release.Namespace }}
 spec:
   endpoints:
   - port: metrics
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 1d032e258..8260aa66d 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -65,6 +65,7 @@ spec:
           httpGet:
             path: {{ $readiness_probe_path }}
             port: 8443
+          timeoutSeconds: 5
       - name: prom-exporter
         image: {{ $.Values.tezos_k8s_images.utils }}
         ports:
@@ -74,6 +75,10 @@ spec:
         env:
         - name: READINESS_PROBE_PATH
           value: {{ $readiness_probe_path | quote }}
+        - name: ENDPOINT_ALIAS
+          value: {{ $endpoint.alias | quote }}
+        - name: BAKER_ALIAS
+          value: {{ $name | quote }}
         command:
           - /usr/local/bin/python
         args:
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 0471d78c1..604ecb652 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -31,6 +31,12 @@ signers:
     # ssh tunnel connection establishes to this port
     tunnel_endpoint_port: 50000
 
+    # Alias to distinguish the endpoints from one another
+    alias: "my home signer"
+
+    # Whether to send alerts when down. set to false for cold standbys.
+    alert_when_down: true
+
   # Set a readiness probe path for your signer.
   # By default, it is the known path implemented by every signer "/authorized_keys"
   # When using tezos-remote-signer-os, you can set it to a path that performs more

From 55c469700eaf64919630279ba09e4bd666426c65 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 16 Mar 2023 22:19:52 -0700
Subject: [PATCH 05/35] support for selection of the signer port

---
 charts/tezos-signer-forwarder/scripts/signer_exporter.py | 5 +++--
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 7 +++++--
 charts/tezos-signer-forwarder/values.yaml                | 4 ++++
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index 028c460be..fe9df88b3 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -11,6 +11,7 @@
 application = Flask(__name__)
 
 readiness_probe_path = os.getenv("READINESS_PROBE_PATH")
+signer_port = os.getenv("SIGNER_PORT")
 endpoint_alias = os.getenv("ENDPOINT_ALIAS")
 baker_alias = os.getenv("BAKER_ALIAS")
 
@@ -41,12 +42,12 @@ def prometheus_metrics():
     extra_labels = 'midl_endpoint_alias="%s",midl_baker_alias="%s"' % (endpoint_alias, baker_alias)
 
     try:
-        probe = requests.get(f"http://localhost:8443{readiness_probe_path}")
+        probe = requests.get(f"http://localhost:{signer_port}{readiness_probe_path}")
     except requests.exceptions.RequestException:
         probe = None
     if probe:
         try:
-            healthz = relabel(requests.get(f"http://localhost:8443/healthz").text, extra_labels)
+            healthz = relabel(requests.get(f"http://localhost:{signer_port}/healthz").text, extra_labels)
         except requests.exceptions.RequestException:
             healthz = None
     else:
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 8260aa66d..ebb3ebd0d 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -1,6 +1,7 @@
 {{- range .Values.signers }}
 {{- $name := .name }}
 {{- $readiness_probe_path := .readiness_probe_path }}
+{{- $signer_port := .signer_port }}
 {{- $signers := . }}
 {{- range $i := until (len .endpoints) }}
 {{- $endpoint := index $signers.endpoints $i }}
@@ -59,12 +60,12 @@ spec:
               key: tunnel_endpoint_port
         ports:
         - name: signer
-          containerPort: 8443
+          containerPort: {{ $signer_port }}
           protocol: TCP
         readinessProbe:
           httpGet:
             path: {{ $readiness_probe_path }}
-            port: 8443
+            port: {{ $signer_port }}
           timeoutSeconds: 5
       - name: prom-exporter
         image: {{ $.Values.tezos_k8s_images.utils }}
@@ -75,6 +76,8 @@ spec:
         env:
         - name: READINESS_PROBE_PATH
           value: {{ $readiness_probe_path | quote }}
+        - name: SIGNER_PORT
+          value: {{ $signer_port | quote }}
         - name: ENDPOINT_ALIAS
           value: {{ $endpoint.alias | quote }}
         - name: BAKER_ALIAS
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 604ecb652..500429154 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -44,6 +44,10 @@ signers:
   #  "/statusz/${PUBLIC_BAKING_KEY_HASH}?ledger_url=${LEDGER_AUTHORIZED_PATH_ENCODED}"
   readiness_probe_path: /authorized_keys
 
+  # The signer port that is being tunneled by the remote signer.
+  # When using HA signer, must be identical for all.
+  signer_port: 6732
+
   # Enter email address to send alerts to.
   # monitoring_email:
 

From 50944be6634960a77036f36340e72f39f6bcab43 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 16 Mar 2023 23:07:20 -0700
Subject: [PATCH 06/35] set port for service as well

---
 charts/tezos-signer-forwarder/templates/service.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index a385f8d66..60d2ea5d6 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -37,7 +37,7 @@ spec:
     app.kubernetes.io/name: tezos-signer-forwarder
     app.kubernetes.io/baker-name: {{ .name }}
   ports:
-  - port: 8443
+  - port: {{ .signer_port }}
     name: signer
   - port: 31732
     name: metrics

From 44681f074eff752b7b90ee10e5b18e3e457b2b41 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 16 Mar 2023 23:24:22 -0700
Subject: [PATCH 07/35] set scrape timeout for remote signers to 20s

---
 charts/tezos-signer-forwarder/templates/servicemonitor.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index 68c9b97c1..7f8fd9e8a 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -11,6 +11,8 @@ spec:
   endpoints:
   - port: metrics
     path: /metrics
+    # default scrape timeout of 10 can be too small for remote raspberry pis
+    scrapeTimeout: "20s"
   selector:
     matchLabels:
       app.kubernetes.io/name: tezos-signer-forwarder

From e11894703ea90881d1d4788ec9a20d91773f6f6f Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Thu, 16 Mar 2023 23:45:42 -0700
Subject: [PATCH 08/35] add toggle for signer metrics

---
 charts/tezos-signer-forwarder/scripts/signer_exporter.py | 3 ++-
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 3 +++
 charts/tezos-signer-forwarder/values.yaml                | 6 ++++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index fe9df88b3..f4785b8fc 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -14,6 +14,7 @@
 signer_port = os.getenv("SIGNER_PORT")
 endpoint_alias = os.getenv("ENDPOINT_ALIAS")
 baker_alias = os.getenv("BAKER_ALIAS")
+signer_metrics = os.getenv("SIGNER_METRICS") == "true"
 
 def relabel(prometheus_metrics,extra_labels):
     '''
@@ -45,7 +46,7 @@ def prometheus_metrics():
         probe = requests.get(f"http://localhost:{signer_port}{readiness_probe_path}")
     except requests.exceptions.RequestException:
         probe = None
-    if probe:
+    if probe and signer_metrics:
         try:
             healthz = relabel(requests.get(f"http://localhost:{signer_port}/healthz").text, extra_labels)
         except requests.exceptions.RequestException:
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index ebb3ebd0d..ff235c560 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -2,6 +2,7 @@
 {{- $name := .name }}
 {{- $readiness_probe_path := .readiness_probe_path }}
 {{- $signer_port := .signer_port }}
+{{- $signer_metrics := .signer_metrics }}
 {{- $signers := . }}
 {{- range $i := until (len .endpoints) }}
 {{- $endpoint := index $signers.endpoints $i }}
@@ -78,6 +79,8 @@ spec:
           value: {{ $readiness_probe_path | quote }}
         - name: SIGNER_PORT
           value: {{ $signer_port | quote }}
+        - name: SIGNER_METRICS
+          value: {{ $signer_metrics | quote }}
         - name: ENDPOINT_ALIAS
           value: {{ $endpoint.alias | quote }}
         - name: BAKER_ALIAS
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 500429154..0d5a785e9 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -48,6 +48,12 @@ signers:
   # When using HA signer, must be identical for all.
   signer_port: 6732
 
+  # Whether the remote signer exposes prometheus metrics.
+  # Typically these will be hardware metrics of the signer.
+  # If true, these metrics will be labeled and scraped into
+  # the cluster's prometheus.
+  signer_metrics: false
+
   # Enter email address to send alerts to.
   # monitoring_email:
 

From c41de6ca5ec989636d278ab24943a616cb506c90 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Mon, 20 Mar 2023 17:16:20 -0700
Subject: [PATCH 09/35] name ports in statefulset as well

---
 charts/tezos-signer-forwarder/templates/service.yaml     | 7 ++++++-
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 3 +++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index 60d2ea5d6..e2a2178be 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -12,10 +12,15 @@ spec:
 {{- range .Values.signers }}
 {{- $name := .name }}
 {{- $signers := . }}
+  # undocumented k8s feature to make a service route to different pods
+  # based on the port - allows to reuse the same public ip in all cloud
+  # providers. For it to work, ports need to have names.
+  # https://github.com/kubernetes/kubernetes/issues/24875#issuecomment-794596576
 {{- range $i := until (len .endpoints) }}
 {{- $endpoint := index $signers.endpoints $i }}
   - port: {{ $endpoint.tunnel_endpoint_port }}
-    name: tunnel-{{ $name }}-{{ $i }}
+    name: ssh-{{ trunc 9 $name }}-{{ $i }}
+    targetPort: ssh-{{ trunc 9 $name }}-{{ $i }}
 {{- end }}
 {{- end }}
   # ensures that remote signers can always ssh
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index ff235c560..5e2777dd7 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -63,6 +63,9 @@ spec:
         - name: signer
           containerPort: {{ $signer_port }}
           protocol: TCP
+        - name: ssh-{{ trunc 9 $name }}-{{ $i }}
+          containerPort: {{ $endpoint.tunnel_endpoint_port }}
+          protocol: TCP
         readinessProbe:
           httpGet:
             path: {{ $readiness_probe_path }}

From 4a43c1c3252a39ed56d3d18b5a52422fda30ca95 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 21 Mar 2023 21:05:56 -0700
Subject: [PATCH 10/35] make sure signer-forwarder pod restarts when endpoint
 config changes

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 5e2777dd7..d743c617c 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -20,8 +20,11 @@ spec:
       app.kubernetes.io/name: tezos-signer-forwarder
   template:
     metadata:
-      {{- with $.Values.podAnnotations }}
       annotations:
+        # ensure that the pod bounces each time endpoint config changes
+        # https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
+        checksum/config: {{ print "$endpoint" | sha256sum }} 
+      {{- with $.Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:

From 4edfd74e447a1c06e1cfa12f3a95b2ebecf3cd84 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 11 Apr 2023 12:50:17 -0700
Subject: [PATCH 11/35] replace janky python injection script with
 `targetLabels`

I didn't know about `targetLabels` but it seems more natural to do it
this way.
---
 charts/tezos-signer-forwarder/scripts/signer_exporter.py    | 3 +--
 charts/tezos-signer-forwarder/templates/service.yaml        | 4 ++--
 charts/tezos-signer-forwarder/templates/servicemonitor.yaml | 4 +++-
 charts/tezos-signer-forwarder/templates/statefulset.yaml    | 4 +---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index f4785b8fc..fd8306f94 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -13,7 +13,6 @@
 readiness_probe_path = os.getenv("READINESS_PROBE_PATH")
 signer_port = os.getenv("SIGNER_PORT")
 endpoint_alias = os.getenv("ENDPOINT_ALIAS")
-baker_alias = os.getenv("BAKER_ALIAS")
 signer_metrics = os.getenv("SIGNER_METRICS") == "true"
 
 def relabel(prometheus_metrics,extra_labels):
@@ -40,7 +39,7 @@ def prometheus_metrics():
     Combines the readiness probe URL (i.e. checks that ledger is configured)
     and the health probe (check power status for example)
     '''
-    extra_labels = 'midl_endpoint_alias="%s",midl_baker_alias="%s"' % (endpoint_alias, baker_alias)
+    extra_labels = 'midl_endpoint_alias="%s"' % endpoint_alias
 
     try:
         probe = requests.get(f"http://localhost:{signer_port}{readiness_probe_path}")
diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index e2a2178be..1c2ff017a 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -36,11 +36,11 @@ metadata:
   name: tezos-remote-signer-{{ .name }}
   labels:
     app.kubernetes.io/name: tezos-signer-forwarder
-    app.kubernetes.io/baker-name: {{ .name }}
+    midl_baker_name: {{ .name }}
 spec:
   selector:
     app.kubernetes.io/name: tezos-signer-forwarder
-    app.kubernetes.io/baker-name: {{ .name }}
+    midl_baker_name: {{ .name }}
   ports:
   - port: {{ .signer_port }}
     name: signer
diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index 7f8fd9e8a..1acf2887d 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -16,7 +16,9 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: tezos-signer-forwarder
-      app.kubernetes.io/baker-name: {{ .name }}
+      midl_baker_name: {{ .name }}
+  targetLabels:
+  - midl_baker_name
 ---
 {{- end }}
 {{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index d743c617c..7999f5ee6 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -29,7 +29,7 @@ spec:
       {{- end }}
       labels:
         app.kubernetes.io/name: tezos-signer-forwarder
-        app.kubernetes.io/baker-name: {{ $name }}
+        midl_baker_name: {{ $name }}
     spec:
       volumes:
       - name: config-volume
@@ -89,8 +89,6 @@ spec:
           value: {{ $signer_metrics | quote }}
         - name: ENDPOINT_ALIAS
           value: {{ $endpoint.alias | quote }}
-        - name: BAKER_ALIAS
-          value: {{ $name | quote }}
         command:
           - /usr/local/bin/python
         args:

From 82906a6e6245fc327ef6f8e6e0832f653e0144a7 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 11 Apr 2023 15:14:29 -0700
Subject: [PATCH 12/35] last part - replace ad-hoc relabeling with proper
 ServiceMonitor config

---
 .../scripts/signer_exporter.py                | 25 +++----------------
 .../templates/alertmanagerconfig.yaml         |  2 +-
 .../templates/prometheusrule.yaml             | 12 ++++-----
 .../templates/servicemonitor.yaml             |  2 ++
 .../templates/statefulset.yaml                |  3 +--
 5 files changed, 13 insertions(+), 31 deletions(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index fd8306f94..deafc6e64 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -12,26 +12,8 @@
 
 readiness_probe_path = os.getenv("READINESS_PROBE_PATH")
 signer_port = os.getenv("SIGNER_PORT")
-endpoint_alias = os.getenv("ENDPOINT_ALIAS")
 signer_metrics = os.getenv("SIGNER_METRICS") == "true"
 
-def relabel(prometheus_metrics,extra_labels):
-    '''
-    Add labels to existing prometheus_metrics
-    '''
-    relabeled_metrics = ""
-    for line in prometheus_metrics.splitlines():
-        if line.startswith("#"):
-            relabeled_metrics += line + "\n"
-        else:
-            if "{" in line:
-                # line has labels 
-                labeled_line = re.sub(r'{(.*)}', r'{\1,%s}' % extra_labels, line)
-            else:
-                labeled_line = re.sub(r'(.*) ', r'\1{%s} ' % extra_labels, line)
-            relabeled_metrics += labeled_line + "\n"
-    return relabeled_metrics
-
 @application.route('/metrics', methods=['GET'])
 def prometheus_metrics():
     '''
@@ -39,7 +21,6 @@ def prometheus_metrics():
     Combines the readiness probe URL (i.e. checks that ledger is configured)
     and the health probe (check power status for example)
     '''
-    extra_labels = 'midl_endpoint_alias="%s"' % endpoint_alias
 
     try:
         probe = requests.get(f"http://localhost:{signer_port}{readiness_probe_path}")
@@ -47,14 +28,14 @@ def prometheus_metrics():
         probe = None
     if probe and signer_metrics:
         try:
-            healthz = relabel(requests.get(f"http://localhost:{signer_port}/healthz").text, extra_labels)
+            healthz = requests.get(f"http://localhost:{signer_port}/healthz").text
         except requests.exceptions.RequestException:
             healthz = None
     else:
         healthz = None
     return '''# number of unhealthy signers - should be 0 or 1
-unhealthy_signers_total{%s} %s
-%s''' % (extra_labels, 0 if probe else 1, healthz or "")
+unhealthy_signers_total %s
+%s''' % (0 if probe else 1, healthz or "")
 
 if __name__ == "__main__":
    application.run(host = "0.0.0.0", port = 31732, debug = False)
diff --git a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
index 73146cb61..4cd6bff15 100644
--- a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
+++ b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
@@ -24,7 +24,7 @@ spec:
     - name: alertType
       value: tezos-remote-signer-alert
       regex: false
-    - name: midl_endpoint_alias
+    - name: midl_endpoint_name
       value: {{ .alias }}
       regex: false
     continue: false
diff --git a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
index a59c073ca..fe4773822 100644
--- a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
+++ b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
@@ -11,8 +11,8 @@ spec:
     rules:
     - alert: SignerPowerLoss
       annotations:
-        description: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost power'
-        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost power'
+        description: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost power'
+        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost power'
       expr: power{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
@@ -20,8 +20,8 @@ spec:
         alertType: tezos-remote-signer-alert
     - alert: SignerWiredNetworkLoss
       annotations:
-        description: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost wired internet connection'
-        summary: 'Tezos remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" has lost wired internet connection'
+        description: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost wired internet connection'
+        summary: 'Tezos remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost wired internet connection'
       expr: wired_network{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
@@ -40,8 +40,8 @@ spec:
     rules:
     - alert: NoRemoteSigner
       annotations:
-        description: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" is down'
-        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_alias }}`}}" for baker "{{`{{ $labels.midl_baker_alias }}`}}" is down or unable to sign.'
+        description: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" is down'
+        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" is down or unable to sign.'
       expr: unhealthy_signers_total{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index 1acf2887d..be54a6a04 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -19,6 +19,8 @@ spec:
       midl_baker_name: {{ .name }}
   targetLabels:
   - midl_baker_name
+  podTargetLabels:
+  - midl_endpoint_name
 ---
 {{- end }}
 {{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 7999f5ee6..7b42d820d 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -30,6 +30,7 @@ spec:
       labels:
         app.kubernetes.io/name: tezos-signer-forwarder
         midl_baker_name: {{ $name }}
+        midl_endpoint_name: {{ $endpoint.alias }}
     spec:
       volumes:
       - name: config-volume
@@ -87,8 +88,6 @@ spec:
           value: {{ $signer_port | quote }}
         - name: SIGNER_METRICS
           value: {{ $signer_metrics | quote }}
-        - name: ENDPOINT_ALIAS
-          value: {{ $endpoint.alias | quote }}
         command:
           - /usr/local/bin/python
         args:

From c73fb8cd4971b1d9d5e5bded544ce0ee2cb04761 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 11 Apr 2023 19:36:02 -0700
Subject: [PATCH 13/35] lint

---
 charts/tezos-signer-forwarder/Chart.yaml       | 18 ------------------
 .../templates/statefulset.yaml                 |  2 +-
 charts/tezos-signer-forwarder/values.yaml      |  2 +-
 3 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/charts/tezos-signer-forwarder/Chart.yaml b/charts/tezos-signer-forwarder/Chart.yaml
index 00db4f455..9089b9ac4 100644
--- a/charts/tezos-signer-forwarder/Chart.yaml
+++ b/charts/tezos-signer-forwarder/Chart.yaml
@@ -1,24 +1,6 @@
 apiVersion: v2
 name: tezos-signer-forwarder
 description: A chart for tezos-signer-forwarder
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
 type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
 version: 0.0.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
 appVersion: "10.0"
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 7b42d820d..1d5d0d21c 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -23,7 +23,7 @@ spec:
       annotations:
         # ensure that the pod bounces each time endpoint config changes
         # https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: {{ print "$endpoint" | sha256sum }} 
+        checksum/config: {{ print "$endpoint" | sha256sum }}
       {{- with $.Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 0d5a785e9..4b6912267 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -72,7 +72,7 @@ service_annotations: {}
 
 # Load Balancer IP to set the ssh service.
 # In some cloud providers, it is used to assign static ip.
-#load_balancer_ip: 
+#load_balancer_ip:
 
 secrets:
   # The ssh host key must be passed as input.

From 7cf5efe76b90c920dbae8eec6b6220f98c6bafdb Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 11 Apr 2023 20:31:59 -0700
Subject: [PATCH 14/35] better explanation for sidecar

---
 charts/tezos-signer-forwarder/scripts/signer_exporter.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index deafc6e64..1e511060b 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -18,8 +18,11 @@
 def prometheus_metrics():
     '''
     Prometheus endpoint
-    Combines the readiness probe URL (i.e. checks that ledger is configured)
-    and the health probe (check power status for example)
+    This combines:
+    * the metrics from the signer, which themselves are a combination of the
+      prometheus node-expoter and custom probes (power status, etc)
+    * the `unhealthy_signers_total` metric exported by this script, verifying
+      whether the signer URL configured upstream returns a 200 OK
     '''
 
     try:

From f8a670bb089fa0831864907f2805fbf280bf7b29 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 13:42:08 -0700
Subject: [PATCH 15/35] remove namespace from serviceMonitor (bc it's not set
 anywhere else)

---
 charts/tezos-signer-forwarder/templates/servicemonitor.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index be54a6a04..c0e54aea8 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -6,7 +6,6 @@ metadata:
   labels:
     app.kubernetes.io/name: tezos-signer-forwarder
   name: tezos-remote-signer-monitoring-{{ .name }}
-  namespace: {{ $.Release.Namespace }}
 spec:
   endpoints:
   - port: metrics

From 5eae0941a07b06880e176874247b1b27a53b775b Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 13:47:58 -0700
Subject: [PATCH 16/35] midl => tezos

---
 .../templates/alertmanagerconfig.yaml                |  2 +-
 .../templates/prometheusrule.yaml                    | 12 ++++++------
 charts/tezos-signer-forwarder/templates/service.yaml |  4 ++--
 .../templates/servicemonitor.yaml                    |  6 +++---
 .../templates/statefulset.yaml                       |  4 ++--
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
index 4cd6bff15..5a8701837 100644
--- a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
+++ b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml
@@ -24,7 +24,7 @@ spec:
     - name: alertType
       value: tezos-remote-signer-alert
       regex: false
-    - name: midl_endpoint_name
+    - name: tezos_endpoint_name
       value: {{ .alias }}
       regex: false
     continue: false
diff --git a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
index fe4773822..f0090d558 100644
--- a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
+++ b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml
@@ -11,8 +11,8 @@ spec:
     rules:
     - alert: SignerPowerLoss
       annotations:
-        description: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost power'
-        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost power'
+        description: 'Remote signer "{{`{{ $labels.tezos_endpoint_name }}`}}" for baker "{{`{{ $labels.tezos_baker_name }}`}}" has lost power'
+        summary: 'Remote signer "{{`{{ $labels.tezos_endpoint_name }}`}}" for baker "{{`{{ $labels.tezos_baker_name }}`}}" has lost power'
       expr: power{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
@@ -20,8 +20,8 @@ spec:
         alertType: tezos-remote-signer-alert
     - alert: SignerWiredNetworkLoss
       annotations:
-        description: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost wired internet connection'
-        summary: 'Tezos remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" has lost wired internet connection'
+        description: 'Remote signer "{{`{{ $labels.tezos_endpoint_name }}`}}" for baker "{{`{{ $labels.tezos_baker_name }}`}}" has lost wired internet connection'
+        summary: 'Tezos remote signer "{{`{{ $labels.tezos_endpoint_name }}`}}" for baker "{{`{{ $labels.tezos_baker_name }}`}}" has lost wired internet connection'
       expr: wired_network{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
@@ -40,8 +40,8 @@ spec:
     rules:
     - alert: NoRemoteSigner
       annotations:
-        description: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" is down'
-        summary: 'Remote signer "{{`{{ $labels.midl_endpoint_name }}`}}" for baker "{{`{{ $labels.midl_baker_name }}`}}" is down or unable to sign.'
+        description: 'Remote signer "{{`{{ $labels.tezos_endpoint_name }}`}}" for baker "{{`{{ $labels.tezos_baker_name }}`}}" is down'
+        summary: 'Remote signer "{{`{{ $labels.tezos_endpoint_name }}`}}" for baker "{{`{{ $labels.tezos_baker_name }}`}}" is down or unable to sign.'
       expr: unhealthy_signers_total{namespace="{{ .Release.Namespace }}"} != 0
       for: 1m
       labels:
diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index 1c2ff017a..ed86c6a9d 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -36,11 +36,11 @@ metadata:
   name: tezos-remote-signer-{{ .name }}
   labels:
     app.kubernetes.io/name: tezos-signer-forwarder
-    midl_baker_name: {{ .name }}
+    tezos_baker_name: {{ .name }}
 spec:
   selector:
     app.kubernetes.io/name: tezos-signer-forwarder
-    midl_baker_name: {{ .name }}
+    tezos_baker_name: {{ .name }}
   ports:
   - port: {{ .signer_port }}
     name: signer
diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
index c0e54aea8..759156e09 100644
--- a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
+++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml
@@ -15,11 +15,11 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: tezos-signer-forwarder
-      midl_baker_name: {{ .name }}
+      tezos_baker_name: {{ .name }}
   targetLabels:
-  - midl_baker_name
+  - tezos_baker_name
   podTargetLabels:
-  - midl_endpoint_name
+  - tezos_endpoint_name
 ---
 {{- end }}
 {{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 1d5d0d21c..b2900d0da 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -29,8 +29,8 @@ spec:
       {{- end }}
       labels:
         app.kubernetes.io/name: tezos-signer-forwarder
-        midl_baker_name: {{ $name }}
-        midl_endpoint_name: {{ $endpoint.alias }}
+        tezos_baker_name: {{ $name }}
+        tezos_endpoint_name: {{ $endpoint.alias }}
     spec:
       volumes:
       - name: config-volume

From 104a3eadec3f97e800e67ad68faeaedf7b9d4c54 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 13:49:03 -0700
Subject: [PATCH 17/35] pin alpine to more stable

---
 signerForwarder/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/signerForwarder/Dockerfile b/signerForwarder/Dockerfile
index d1a692b4a..da11306a8 100644
--- a/signerForwarder/Dockerfile
+++ b/signerForwarder/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:edge
+FROM alpine:3.18.4
 
 # add openssh and clean
 RUN apk add --no-cache openssh shadow

From 3b978d0640fcc384be102718bdb94282aba04e29 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 13:53:52 -0700
Subject: [PATCH 18/35] add -D and -e to CMD in signerForwarder dockerfile

does not do anything since we use entrypoint in chart
---
 signerForwarder/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/signerForwarder/Dockerfile b/signerForwarder/Dockerfile
index da11306a8..ab244e41e 100644
--- a/signerForwarder/Dockerfile
+++ b/signerForwarder/Dockerfile
@@ -20,4 +20,4 @@ RUN cat /etc/ssh/sshd_config
 
 RUN mkdir /home/signer/.ssh && chown -R signer /home/signer
 
-CMD ["/usr/sbin/sshd"]
+CMD ["/usr/sbin/sshd", "-D", "-e"]

From bd66fd1320c68f0fdb91e1d8027f3112f6fe7190 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 13:57:47 -0700
Subject: [PATCH 19/35] move signer forwarder image into tezos_k8s_images

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 2 +-
 charts/tezos-signer-forwarder/values.yaml                | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index b2900d0da..73cdf827c 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -42,7 +42,7 @@ spec:
            defaultMode: 0400
       containers:
       - name: tezos-signer-forwarder
-        image: {{ $.Values.images.tezos_signer_forwarder }}
+        image: {{ $.Values.tezos_k8s_images.tezos_signer_forwarder }}
         imagePullPolicy: IfNotPresent
         command:
           - /bin/sh
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 4b6912267..f00d55d62 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -1,7 +1,6 @@
-images:
-  tezos_signer_forwarder: localhost/tezos-k8s-signerforwarder:dev
 tezos_k8s_images:
   utils: ghcr.io/oxheadalpha/tezos-k8s-utils:master
+  tezos_signer_forwarder: ghcr.io/oxheadalpha/tezos-k8s-signerforwarder:dev
 
 # List the endpoints below.
 # Each endpoint represents a ssh server.

From 6817c1a5641481eccbc81347c263eb3991e451bd Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 13:58:25 -0700
Subject: [PATCH 20/35] values: uncomment and make ""

---
 charts/tezos-signer-forwarder/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index f00d55d62..4674897f2 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -54,7 +54,7 @@ signers:
   signer_metrics: false
 
   # Enter email address to send alerts to.
-  # monitoring_email:
+  monitoring_email: ""
 
 # Name that goes into the service
 # e.g tezos-signer-mybaker

From 8835dbaee1dfde082043698e0a2c3b5c4fe46e3a Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:01:34 -0700
Subject: [PATCH 21/35] load balancer ip: uncomment and set to ""

---
 charts/tezos-signer-forwarder/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index 4674897f2..aba7c658d 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -71,7 +71,7 @@ service_annotations: {}
 
 # Load Balancer IP to set the ssh service.
 # In some cloud providers, it is used to assign static ip.
-#load_balancer_ip:
+load_balancer_ip: ""
 
 secrets:
   # The ssh host key must be passed as input.

From ec63d9c31d86588e5ff275c8781c4f14dcffbed4 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:08:20 -0700
Subject: [PATCH 22/35] Update
 charts/tezos-signer-forwarder/templates/statefulset.yaml

Co-authored-by: Aryeh Harris <harryttd@users.noreply.github.com>
---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 73cdf827c..52f857107 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -3,9 +3,7 @@
 {{- $readiness_probe_path := .readiness_probe_path }}
 {{- $signer_port := .signer_port }}
 {{- $signer_metrics := .signer_metrics }}
-{{- $signers := . }}
-{{- range $i := until (len .endpoints) }}
-{{- $endpoint := index $signers.endpoints $i }}
+{{- range $i, $endpoint := .endpoints }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

From 0ef9f7dd3abd8557e2c7468a9f1667e7499c56f0 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:15:57 -0700
Subject: [PATCH 23/35] simplify enumeration

---
 charts/tezos-signer-forwarder/templates/config.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/config.yaml b/charts/tezos-signer-forwarder/templates/config.yaml
index c64c86773..9566c7b72 100644
--- a/charts/tezos-signer-forwarder/templates/config.yaml
+++ b/charts/tezos-signer-forwarder/templates/config.yaml
@@ -1,8 +1,6 @@
 {{- range .Values.signers }}
 {{- $name := .name }}
-{{- $signers := . }}
-{{- range $i := until (len .endpoints) }}
-{{- $endpoint := index $signers.endpoints $i }}
+{{- range $i, $endpoint := .endpoints }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

From 0aeda6ada5b237a6c8c65a66cf86f2078cfba0da Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:17:20 -0700
Subject: [PATCH 24/35] Update
 charts/tezos-signer-forwarder/scripts/signer_exporter.py

Co-authored-by: Aryeh Harris <harryttd@users.noreply.github.com>
---
 charts/tezos-signer-forwarder/scripts/signer_exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index 1e511060b..d991fbad2 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -20,7 +20,7 @@ def prometheus_metrics():
     Prometheus endpoint
     This combines:
     * the metrics from the signer, which themselves are a combination of the
-      prometheus node-expoter and custom probes (power status, etc)
+      prometheus node-exporter and custom probes (power status, etc)
     * the `unhealthy_signers_total` metric exported by this script, verifying
       whether the signer URL configured upstream returns a 200 OK
     '''

From 826cef42ce0c01e1655edced57f007d9af135352 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:18:47 -0700
Subject: [PATCH 25/35] add readonly for the ssh secrets

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 52f857107..62a3bd3a0 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -38,6 +38,7 @@ spec:
         secret:
            secretName: tezos-signer-forwarder-secret-{{ $.Values.name }}
            defaultMode: 0400
+           readOnly: true
       containers:
       - name: tezos-signer-forwarder
         image: {{ $.Values.tezos_k8s_images.tezos_signer_forwarder }}
@@ -52,9 +53,11 @@ spec:
         - name: config-volume
           mountPath: /home/signer/.ssh/authorized_keys
           subPath: authorized_keys
+          readOnly: true
         - name: secret-volume
           mountPath: /etc/ssh/ssh_host_ecdsa_key
           subPath: ssh_host_ecdsa_key
+          readOnly: true
         env:
         - name: TUNNEL_ENDPOINT_PORT
           valueFrom:

From ca0376dfe39c27d9d9bbd9d9742af4dea3948bdd Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:20:28 -0700
Subject: [PATCH 26/35] default mode 400 for more config files

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 62a3bd3a0..56f5a8db3 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -34,6 +34,7 @@ spec:
       - name: config-volume
         configMap:
            name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
+           defaultMode: 0400
       - name: secret-volume
         secret:
            secretName: tezos-signer-forwarder-secret-{{ $.Values.name }}
@@ -63,6 +64,7 @@ spec:
           valueFrom:
             configMapKeyRef:
               name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
+              defaultMode: 0400
               key: tunnel_endpoint_port
         ports:
         - name: signer

From a915821b6d12f5a29b2ea6a5546b79db61534d16 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:27:45 -0700
Subject: [PATCH 27/35] remove range and add enumeration in service

---
 charts/tezos-signer-forwarder/templates/service.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index ed86c6a9d..32d5f4363 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -11,13 +11,11 @@ spec:
   ports:
 {{- range .Values.signers }}
 {{- $name := .name }}
-{{- $signers := . }}
   # undocumented k8s feature to make a service route to different pods
   # based on the port - allows to reuse the same public ip in all cloud
   # providers. For it to work, ports need to have names.
   # https://github.com/kubernetes/kubernetes/issues/24875#issuecomment-794596576
-{{- range $i := until (len .endpoints) }}
-{{- $endpoint := index $signers.endpoints $i }}
+{{- range $i, $endpoint := .endpoints }}
   - port: {{ $endpoint.tunnel_endpoint_port }}
     name: ssh-{{ trunc 9 $name }}-{{ $i }}
     targetPort: ssh-{{ trunc 9 $name }}-{{ $i }}

From 49cf3a95765aec5c4f53d475ee42fd46ed544419 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:31:19 -0700
Subject: [PATCH 28/35] only expose metrics port in service when enabled in
 values

---
 charts/tezos-signer-forwarder/templates/service.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index 32d5f4363..c2d0f9302 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -42,8 +42,10 @@ spec:
   ports:
   - port: {{ .signer_port }}
     name: signer
+{{ if .signer_metrics }}
   - port: 31732
     name: metrics
+{{ end }}
   # make sure that the service always targets the same signer, when HA is in use.
   sessionAffinity: ClientIP
 ---

From e28f727dabc36e67128211ee46beee6027d401f3 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:41:16 -0700
Subject: [PATCH 29/35] Revert "only expose metrics port in service when
 enabled in values"

This reverts commit 49cf3a95765aec5c4f53d475ee42fd46ed544419.
---
 charts/tezos-signer-forwarder/templates/service.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml
index c2d0f9302..32d5f4363 100644
--- a/charts/tezos-signer-forwarder/templates/service.yaml
+++ b/charts/tezos-signer-forwarder/templates/service.yaml
@@ -42,10 +42,8 @@ spec:
   ports:
   - port: {{ .signer_port }}
     name: signer
-{{ if .signer_metrics }}
   - port: 31732
     name: metrics
-{{ end }}
   # make sure that the service always targets the same signer, when HA is in use.
   sessionAffinity: ClientIP
 ---

From 92accfeeaa7379268951e2b8925fa2c3bf03b314 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 14:46:41 -0700
Subject: [PATCH 30/35] grab endpoint port straight from values.yaml instead of
 going thru a cm

---
 charts/tezos-signer-forwarder/templates/config.yaml      | 1 -
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 6 +-----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/config.yaml b/charts/tezos-signer-forwarder/templates/config.yaml
index 9566c7b72..975712d18 100644
--- a/charts/tezos-signer-forwarder/templates/config.yaml
+++ b/charts/tezos-signer-forwarder/templates/config.yaml
@@ -7,7 +7,6 @@ metadata:
   name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
 data:
   authorized_keys: "{{ $endpoint.ssh_pubkey }} signer"
-  tunnel_endpoint_port: "{{ $endpoint.tunnel_endpoint_port }}"
 ---
 {{- end }}
 {{- end }}
diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 56f5a8db3..cc3e4ed5b 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -61,11 +61,7 @@ spec:
           readOnly: true
         env:
         - name: TUNNEL_ENDPOINT_PORT
-          valueFrom:
-            configMapKeyRef:
-              name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
-              defaultMode: 0400
-              key: tunnel_endpoint_port
+          value: {{ $endpoint.tunnel_endpoint_port }}
         ports:
         - name: signer
           containerPort: {{ $signer_port }}

From db803424a484caf8ee5c8b409b1d2fd92cc0da3a Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 15:54:49 -0700
Subject: [PATCH 31/35] re-add missing quotes

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index cc3e4ed5b..639912d01 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -61,7 +61,7 @@ spec:
           readOnly: true
         env:
         - name: TUNNEL_ENDPOINT_PORT
-          value: {{ $endpoint.tunnel_endpoint_port }}
+          value: "{{ $endpoint.tunnel_endpoint_port }}"
         ports:
         - name: signer
           containerPort: {{ $signer_port }}

From b9a58c93f581ab1cdae5d91a80cdcd9f9bda58c7 Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 16:45:48 -0700
Subject: [PATCH 32/35] revert some of the perm changes to make it work

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 639912d01..5b50770c9 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -34,7 +34,7 @@ spec:
       - name: config-volume
         configMap:
            name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
-           defaultMode: 0400
+           defaultMode: 0444
       - name: secret-volume
         secret:
            secretName: tezos-signer-forwarder-secret-{{ $.Values.name }}
@@ -54,11 +54,9 @@ spec:
         - name: config-volume
           mountPath: /home/signer/.ssh/authorized_keys
           subPath: authorized_keys
-          readOnly: true
         - name: secret-volume
           mountPath: /etc/ssh/ssh_host_ecdsa_key
           subPath: ssh_host_ecdsa_key
-          readOnly: true
         env:
         - name: TUNNEL_ENDPOINT_PORT
           value: "{{ $endpoint.tunnel_endpoint_port }}"

From 75d05495e10a089bc0a0584bca2830420171777f Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 18:56:46 -0700
Subject: [PATCH 33/35] add comment why container runs as root

---
 signerForwarder/Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/signerForwarder/Dockerfile b/signerForwarder/Dockerfile
index ab244e41e..7a873468c 100644
--- a/signerForwarder/Dockerfile
+++ b/signerForwarder/Dockerfile
@@ -20,4 +20,6 @@ RUN cat /etc/ssh/sshd_config
 
 RUN mkdir /home/signer/.ssh && chown -R signer /home/signer
 
+# Note that the container must run as root. see
+#  https://superuser.com/a/1548482
 CMD ["/usr/sbin/sshd", "-D", "-e"]

From 133cff2853a0647fa8efd454fcdaacca93d8256f Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Tue, 24 Oct 2023 19:10:55 -0700
Subject: [PATCH 34/35] handle readiness probe timeout just like for the node

---
 .../scripts/signer_exporter.py                      | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
index d991fbad2..b81362024 100644
--- a/charts/tezos-signer-forwarder/scripts/signer_exporter.py
+++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py
@@ -2,7 +2,6 @@
 import os
 from flask import Flask, request, jsonify
 import requests
-import re
 
 import logging
 log = logging.getLogger('werkzeug')
@@ -14,6 +13,10 @@
 signer_port = os.getenv("SIGNER_PORT")
 signer_metrics = os.getenv("SIGNER_METRICS") == "true"
 
+# https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+# Configured readiness probe timeoutSeconds is 5s, timeout sync request before that.
+SIGNER_CONNECT_TIMEOUT = 4.5
+
 @application.route('/metrics', methods=['GET'])
 def prometheus_metrics():
     '''
@@ -26,7 +29,13 @@ def prometheus_metrics():
     '''
 
     try:
-        probe = requests.get(f"http://localhost:{signer_port}{readiness_probe_path}")
+        probe = requests.get(f"http://localhost:{signer_port}{readiness_probe_path}", timeout=SIGNER_CONNECT_TIMEOUT)
+    except requests.exceptions.ConnectTimeout:
+        #Timeout connect to node
+        probe = None
+    except requests.exceptions.ReadTimeout:
+        #Timeout read from node
+        probe = None
     except requests.exceptions.RequestException:
         probe = None
     if probe and signer_metrics:

From 00d66e41c7bd2461c1bdea07b522feba51228f5d Mon Sep 17 00:00:00 2001
From: Nicolas Ochem <nicolas.ochem@gmail.com>
Date: Fri, 10 Nov 2023 14:38:19 -0800
Subject: [PATCH 35/35] do not hardcode pulumi annotation

---
 charts/tezos-signer-forwarder/templates/statefulset.yaml | 5 ++++-
 charts/tezos-signer-forwarder/values.yaml                | 5 +++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml
index 5b50770c9..84bcef2d5 100644
--- a/charts/tezos-signer-forwarder/templates/statefulset.yaml
+++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml
@@ -9,7 +9,9 @@ kind: StatefulSet
 metadata:
   name: tezos-signer-forwarder-{{ $name}}-{{ $i }}
   annotations:
-    "pulumi.com/skipAwait": "true"
+  {{- with $.Values.stateful_set_annotations }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   replicas: 1
   serviceName: tezos-remote-signer-{{ $name }}
@@ -35,6 +37,7 @@ spec:
         configMap:
            name: tezos-signer-forwarder-config-{{ $name }}-{{ $i }}
            defaultMode: 0444
+           readOnly: true
       - name: secret-volume
         secret:
            secretName: tezos-signer-forwarder-secret-{{ $.Values.name }}
diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml
index aba7c658d..55d5bc66a 100644
--- a/charts/tezos-signer-forwarder/values.yaml
+++ b/charts/tezos-signer-forwarder/values.yaml
@@ -69,6 +69,11 @@ node_selector: {}
 # be used to assign a static ip address.
 service_annotations: {}
 
+
+stateful_set_annotations: {}
+# example: this will cause pulumi to not wait for tunnel to be up
+# pulumi/skipAwait: true
+
 # Load Balancer IP to set the ssh service.
 # In some cloud providers, it is used to assign static ip.
 load_balancer_ip: ""