feat: enable HTTPS in OpenShift clusters (#595)

This change adds an end-to-end test ensuring that the operator's
/metrics endpoint works as expected.

When the OpenShift flag isn't enabled, metrics are still served over
HTTP.

Signed-off-by: Simon Pasquier <[email protected]>

bundle/manifests/observability-operator.clusterserviceversion.yaml

@@ -42,7 +42,7 @@ metadata:
     categories: Monitoring
     certified: "false"
     containerImage: observability-operator:0.4.2
-    createdAt: "2024-10-29T09:23:51Z"
+    createdAt: "2024-10-29T11:40:05Z"
     description: A Go based Kubernetes operator to setup and manage highly available
       Monitoring Stack using Prometheus, Alertmanager and Thanos Querier.
     operators.operatorframework.io/builder: operator-sdk-v1.37.0
@@ -753,6 +753,10 @@ spec:
                   capabilities:
                     drop:
                     - ALL
+                volumeMounts:
+                - mountPath: /etc/tls/private
+                  name: observability-operator-tls
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: observability-operator-sa
@@ -761,6 +765,11 @@ spec:
               - effect: NoSchedule
                 key: node-role.kubernetes.io/infra
                 operator: Exists
+              volumes:
+              - name: observability-operator-tls
+                secret:
+                  optional: true
+                  secretName: observability-operator-tls
     strategy: deployment
   installModes:
   - supported: false

bundle/manifests/observability-operator_v1_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: observability-operator-tls
   creationTimestamp: null
   labels:
     app.kubernetes.io/component: operator

cmd/operator/main.go

@@ -115,7 +115,10 @@ func main() {
 		os.Exit(1)
 	}
 
+	ctx := ctrl.SetupSignalHandler()
+
 	op, err := operator.New(
+		ctx,
 		operator.NewOperatorConfiguration(
 			operator.WithMetricsAddr(metricsAddr),
 			operator.WithHealthProbeAddr(healthProbeAddr),
@@ -135,7 +138,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	setupLog.Info("starting manager")
 	if err := op.Start(ctx); err != nil {
 		setupLog.Error(err, "terminating")

deploy/operator/observability-operator-deployment.yaml

@@ -53,5 +53,14 @@ spec:
              httpGet:
                path: /healthz
                port: 8081
+          volumeMounts:
+          - mountPath: /etc/tls/private
+            name: observability-operator-tls
+            readOnly: true
       serviceAccountName: observability-operator-sa
+      volumes:
+      - name: observability-operator-tls
+        secret:
+          secretName: observability-operator-tls
+          optional: true
       terminationGracePeriodSeconds: 30

deploy/operator/observability-operator-service.yaml

@@ -6,6 +6,8 @@ metadata:
     app.kubernetes.io/component: operator
     app.kubernetes.io/name: observability-operator
     app.kubernetes.io/part-of: observability-operator
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: observability-operator-tls
 spec:
   selector:
     app.kubernetes.io/name: observability-operator

go.mod

@@ -19,8 +19,9 @@ require (
 	k8s.io/api v0.31.2
 	k8s.io/apiextensions-apiserver v0.31.1
 	k8s.io/apimachinery v0.31.2
-	k8s.io/client-go v0.31.1
-	k8s.io/component-base v0.31.1
+	k8s.io/apiserver v0.31.2
+	k8s.io/client-go v0.31.2
+	k8s.io/component-base v0.31.2
 	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3
 	sigs.k8s.io/controller-runtime v0.19.0
 )

go.sum

@@ -45,8 +45,8 @@ github.com/efficientgo/core v1.0.0-rc.2 h1:7j62qHLnrZqO3V3UA0AqOGd5d5aXV3AX6m/NZ
 github.com/efficientgo/core v1.0.0-rc.2/go.mod h1:FfGdkzWarkuzOlY04VY+bGfb1lWrjaL6x/GLcQ4vJps=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
-github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
+github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
@@ -91,6 +91,7 @@ github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ
 github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
 github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -359,10 +360,12 @@ k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/
 k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ=
 k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
 k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
-k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
-k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
-k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8=
-k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w=
+k8s.io/apiserver v0.31.2 h1:VUzOEUGRCDi6kX1OyQ801m4A7AUPglpsmGvdsekmcI4=
+k8s.io/apiserver v0.31.2/go.mod h1:o3nKZR7lPlJqkU5I3Ove+Zx3JuoFjQobGX1Gctw6XuE=
+k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
+k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
+k8s.io/component-base v0.31.2 h1:Z1J1LIaC0AV+nzcPRFqfK09af6bZ4D1nAOpWsy9owlA=
+k8s.io/component-base v0.31.2/go.mod h1:9PeyyFN/drHjtJZMCTkSpQJS3U9OXORnHQqMLDz0sUQ=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo=

pkg/operator/operator.go

@@ -2,8 +2,16 @@ package operator
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
-
+	"os"
+	"path/filepath"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -15,15 +23,23 @@ import (
 	uictrl "github.com/rhobs/observability-operator/pkg/controllers/uiplugin"
 )
 
-// NOTE: The instance selector label is hardcoded in static assets.
-// Any change to that must be reflected here as well
-const instanceSelector = "app.kubernetes.io/managed-by=observability-operator"
+const (
+	// NOTE: The instance selector label is hardcoded in static assets.
+	// Any change to that must be reflected here as well
+	instanceSelector = "app.kubernetes.io/managed-by=observability-operator"
+
+	ObservabilityOperatorName = "observability-operator"
 
-const ObservabilityOperatorName = "observability-operator"
+	// The mount path for the serving certificate seret is hardcoded in the
+	// static assets.
+	tlsMountPath = "/etc/tls/private"
+)
 
-// Operator embedds manager and exposes only the minimal set of functions
+// Operator embeds a manager and a serving certificate controller (for
+// OpenShift installations).
 type Operator struct {
-	manager manager.Manager
+	manager               manager.Manager
+	servingCertController *dynamiccertificates.DynamicServingCertificateController
 }
 
 type OpenShiftFeatureGates struct {
@@ -102,14 +118,90 @@ func NewOperatorConfiguration(opts ...func(*OperatorConfiguration)) *OperatorCon
 	return cfg
 }
 
-func New(cfg *OperatorConfiguration) (*Operator, error) {
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme: NewScheme(cfg),
-		Metrics: metricsserver.Options{
-			BindAddress: cfg.MetricsAddr,
-		},
-		HealthProbeBindAddress: cfg.HealthProbeAddr,
-	})
+func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) {
+	restConfig := ctrl.GetConfigOrDie()
+
+	metricsOpts := metricsserver.Options{
+		BindAddress: cfg.MetricsAddr,
+	}
+
+	var servingCertController *dynamiccertificates.DynamicServingCertificateController
+	if cfg.FeatureGates.OpenShift.Enabled {
+		// When running in OpenShift, the server uses HTTPS thanks to the
+		// service CA operator.
+		certFile := filepath.Join(tlsMountPath, "tls.crt")
+		keyFile := filepath.Join(tlsMountPath, "tls.key")
+
+		// Wait for the files to be mounted into the container.
+		var pollErr error
+		err := wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
+			for _, f := range []string{certFile, keyFile} {
+				if _, err := os.Stat(f); err != nil {
+					pollErr = err
+					return false, nil
+				}
+			}
+
+			return true, nil
+		})
+		if err != nil {
+			return nil, fmt.Errorf("%w: %w", err, pollErr)
+		}
+
+		// DynamicCertKeyPairContent automatically reloads the certificate and key from disk.
+		certKeyProvider, err := dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", certFile, keyFile)
+		if err != nil {
+			return nil, err
+		}
+		if err := certKeyProvider.RunOnce(ctx); err != nil {
+			return nil, fmt.Errorf("failed to initialize cert/key content: %w", err)
+		}
+
+		kubeClient, err := kubernetes.NewForConfig(restConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		// ConfigMapCAController automatically reloads the client CA.
+		clientCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
+			"client-ca",
+			metav1.NamespaceSystem,
+			"extension-apiserver-authentication",
+			"client-ca-file",
+			kubeClient,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize client CA controller: %w", err)
+		}
+
+		servingCertController = dynamiccertificates.NewDynamicServingCertificateController(
+			&tls.Config{
+				ClientAuth: tls.NoClientCert,
+			},
+			clientCAProvider,
+			certKeyProvider,
+			nil,
+			nil,
+		)
+		if err := servingCertController.RunOnce(); err != nil {
+			return nil, fmt.Errorf("failed to initialize serving certificate controller: %w", err)
+		}
+
+		metricsOpts.SecureServing = true
+		metricsOpts.TLSOpts = []func(*tls.Config){
+			func(c *tls.Config) {
+				c.GetConfigForClient = servingCertController.GetConfigForClient
+			},
+		}
+	}
+
+	mgr, err := ctrl.NewManager(
+		restConfig,
+		ctrl.Options{
+			Scheme:                 NewScheme(cfg),
+			Metrics:                metricsOpts,
+			HealthProbeBindAddress: cfg.HealthProbeAddr,
+		})
 	if err != nil {
 		return nil, fmt.Errorf("unable to create manager: %w", err)
 	}
@@ -141,11 +233,16 @@ func New(cfg *OperatorConfiguration) (*Operator, error) {
 	}
 
 	return &Operator{
-		manager: mgr,
+		manager:               mgr,
+		servingCertController: servingCertController,
 	}, nil
 }
 
 func (o *Operator) Start(ctx context.Context) error {
+	if o.servingCertController != nil {
+		go o.servingCertController.Run(1, ctx.Done())
+	}
+
 	if err := o.manager.Start(ctx); err != nil {
 		return fmt.Errorf("unable to start manager: %w", err)
 	}