feat: enable HTTPS in OpenShift clusters

This change adds an end-to-end test ensuring that the operator's
/metrics endpoint works as expected.

When the OpenShift flag isn't enabled, metrics are still served over
HTTP.

Signed-off-by: Simon Pasquier <[email protected]>

.github/e2e-tests-olm/action.yaml

@@ -33,7 +33,7 @@ runs:
 
     - name: Install OLM
       shell: bash
-      run: ./tmp/bin/operator-sdk olm install
+      run: ./tmp/bin/operator-sdk olm install --timeout 4m
 
     - name: Create a local docker registry in the Kind cluster
       shell: bash

Makefile

@@ -310,7 +310,7 @@ initiate-release-as: $(STANDARD_VERSION)
 .PHONY: kind-cluster
 kind-cluster: $(OPERATOR_SDK)
 	kind create cluster --config hack/kind/config.yaml
-	$(OPERATOR_SDK) olm install
+	$(OPERATOR_SDK) olm install --timeout 4m
 	kubectl apply -f hack/kind/registry.yaml -n operators
 	kubectl create -k deploy/crds/kubernetes/
 	kubectl create -k deploy/dependencies

bundle/manifests/observability-operator.clusterserviceversion.yaml

@@ -42,7 +42,7 @@ metadata:
     categories: Monitoring
     certified: "false"
     containerImage: observability-operator:0.4.2
-    createdAt: "2024-10-08T12:44:05Z"
+    createdAt: "2024-10-17T13:23:55Z"
     description: A Go based Kubernetes operator to setup and manage highly available
       Monitoring Stack using Prometheus, Alertmanager and Thanos Querier.
     operators.operatorframework.io/builder: operator-sdk-v1.36.1
@@ -745,6 +745,10 @@ spec:
                   capabilities:
                     drop:
                     - ALL
+                volumeMounts:
+                - mountPath: /etc/tls/private
+                  name: observability-operator-tls
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: observability-operator-sa
@@ -753,6 +757,11 @@ spec:
               - effect: NoSchedule
                 key: node-role.kubernetes.io/infra
                 operator: Exists
+              volumes:
+              - name: observability-operator-tls
+                secret:
+                  optional: true
+                  secretName: observability-operator-tls
     strategy: deployment
   installModes:
   - supported: false

bundle/manifests/observability-operator_v1_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: observability-operator-tls
   creationTimestamp: null
   labels:
     app.kubernetes.io/component: operator

cmd/operator/main.go

@@ -114,7 +114,10 @@ func main() {
 		os.Exit(1)
 	}
 
+	ctx := ctrl.SetupSignalHandler()
+
 	op, err := operator.New(
+		ctx,
 		operator.NewOperatorConfiguration(
 			operator.WithMetricsAddr(metricsAddr),
 			operator.WithHealthProbeAddr(healthProbeAddr),
@@ -134,7 +137,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	setupLog.Info("starting manager")
 	if err := op.Start(ctx); err != nil {
 		setupLog.Error(err, "terminating")

deploy/operator/observability-operator-deployment.yaml

@@ -53,5 +53,14 @@ spec:
              httpGet:
                path: /healthz
                port: 8081
+          volumeMounts:
+          - mountPath: /etc/tls/private
+            name: observability-operator-tls
+            readOnly: true
       serviceAccountName: observability-operator-sa
+      volumes:
+      - name: observability-operator-tls
+        secret:
+          secretName: observability-operator-tls
+          optional: true
       terminationGracePeriodSeconds: 30

deploy/operator/observability-operator-service.yaml

@@ -6,6 +6,8 @@ metadata:
     app.kubernetes.io/component: operator
     app.kubernetes.io/name: observability-operator
     app.kubernetes.io/part-of: observability-operator
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: observability-operator-tls
 spec:
   selector:
     app.kubernetes.io/name: observability-operator

go.mod

@@ -18,6 +18,7 @@ require (
 	k8s.io/api v0.31.1
 	k8s.io/apiextensions-apiserver v0.31.1
 	k8s.io/apimachinery v0.31.1
+	k8s.io/apiserver v0.31.1
 	k8s.io/client-go v0.31.1
 	k8s.io/component-base v0.31.1
 	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3

go.sum

@@ -357,6 +357,8 @@ k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/
 k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ=
 k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=
 k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c=
+k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM=
 k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
 k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
 k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8=

pkg/operator/operator.go

@@ -2,8 +2,16 @@ package operator
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
-
+	"os"
+	"path/filepath"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -15,15 +23,23 @@ import (
 	uictrl "github.com/rhobs/observability-operator/pkg/controllers/uiplugin"
 )
 
-// NOTE: The instance selector label is hardcoded in static assets.
-// Any change to that must be reflected here as well
-const instanceSelector = "app.kubernetes.io/managed-by=observability-operator"
+const (
+	// NOTE: The instance selector label is hardcoded in static assets.
+	// Any change to that must be reflected here as well
+	instanceSelector = "app.kubernetes.io/managed-by=observability-operator"
+
+	ObservabilityOperatorName = "observability-operator"
 
-const ObservabilityOperatorName = "observability-operator"
+	// The mount path for the serving certificate seret is hardcoded in the
+	// static assets.
+	tlsMountPath = "/etc/tls/private"
+)
 
-// Operator embedds manager and exposes only the minimal set of functions
+// Operator embeds a manager and a serving certificate controller (for
+// OpenShift installations).
 type Operator struct {
-	manager manager.Manager
+	manager               manager.Manager
+	servingCertController *dynamiccertificates.DynamicServingCertificateController
 }
 
 type OpenShiftFeatureGates struct {
@@ -102,14 +118,90 @@ func NewOperatorConfiguration(opts ...func(*OperatorConfiguration)) *OperatorCon
 	return cfg
 }
 
-func New(cfg *OperatorConfiguration) (*Operator, error) {
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme: NewScheme(cfg),
-		Metrics: metricsserver.Options{
-			BindAddress: cfg.MetricsAddr,
-		},
-		HealthProbeBindAddress: cfg.HealthProbeAddr,
-	})
+func New(ctx context.Context, cfg *OperatorConfiguration) (*Operator, error) {
+	restConfig := ctrl.GetConfigOrDie()
+
+	metricsOpts := metricsserver.Options{
+		BindAddress: cfg.MetricsAddr,
+	}
+
+	var servingCertController *dynamiccertificates.DynamicServingCertificateController
+	if cfg.FeatureGates.OpenShift.Enabled {
+		// When running in OpenShift, the server uses HTTPS thanks to the
+		// service CA operator.
+		certFile := filepath.Join(tlsMountPath, "tls.crt")
+		keyFile := filepath.Join(tlsMountPath, "tls.key")
+
+		// Wait for the files to be mounted into the container.
+		var pollErr error
+		err := wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
+			for _, f := range []string{certFile, keyFile} {
+				if _, err := os.Stat(f); err != nil {
+					pollErr = err
+					return false, nil
+				}
+			}
+
+			return true, nil
+		})
+		if err != nil {
+			return nil, fmt.Errorf("%w: %w", err, pollErr)
+		}
+
+		// DynamicCertKeyPairContent automatically reloads the certificate and key from disk.
+		certKeyProvider, err := dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", certFile, keyFile)
+		if err != nil {
+			return nil, err
+		}
+		if err := certKeyProvider.RunOnce(ctx); err != nil {
+			return nil, fmt.Errorf("failed to initialize cert/key content: %w", err)
+		}
+
+		kubeClient, err := kubernetes.NewForConfig(restConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		// ConfigMapCAController automatically reloads the client CA.
+		clientCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
+			"client-ca",
+			metav1.NamespaceSystem,
+			"extension-apiserver-authentication",
+			"client-ca-file",
+			kubeClient,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize client CA controller: %w", err)
+		}
+
+		servingCertController = dynamiccertificates.NewDynamicServingCertificateController(
+			&tls.Config{
+				ClientAuth: tls.NoClientCert,
+			},
+			clientCAProvider,
+			certKeyProvider,
+			nil,
+			nil,
+		)
+		if err := servingCertController.RunOnce(); err != nil {
+			return nil, fmt.Errorf("failed to initialize serving certificate controller: %w", err)
+		}
+
+		metricsOpts.SecureServing = true
+		metricsOpts.TLSOpts = []func(*tls.Config){
+			func(c *tls.Config) {
+				c.GetConfigForClient = servingCertController.GetConfigForClient
+			},
+		}
+	}
+
+	mgr, err := ctrl.NewManager(
+		restConfig,
+		ctrl.Options{
+			Scheme:                 NewScheme(cfg),
+			Metrics:                metricsOpts,
+			HealthProbeBindAddress: cfg.HealthProbeAddr,
+		})
 	if err != nil {
 		return nil, fmt.Errorf("unable to create manager: %w", err)
 	}
@@ -141,11 +233,16 @@ func New(cfg *OperatorConfiguration) (*Operator, error) {
 	}
 
 	return &Operator{
-		manager: mgr,
+		manager:               mgr,
+		servingCertController: servingCertController,
 	}, nil
 }
 
 func (o *Operator) Start(ctx context.Context) error {
+	if o.servingCertController != nil {
+		go o.servingCertController.Run(1, ctx.Done())
+	}
+
 	if err := o.manager.Start(ctx); err != nil {
 		return fmt.Errorf("unable to start manager: %w", err)
 	}