Ensure that metrics is protected with TLS for Prometheus integration

This change replaces controllers-runtime's self-signed certificates for metrics with cert-manager-managed certificates, ensuring security. We should not use `insecureSkipVerify: true`. Also, it is not a good practice for production env relay on the certs self-signed generated by controller-runtime and/or the old kube-rbac-proxy according to sig-security.

Key updates:
- Configured `metricsServerOptions` to use cert-manager-managed certificates (`/var/metrics/certs`).
- Added `ClusterIssuer` (`olmv1-metrics-ca`) and `Certificate` resources for automated certificate management.
- Updated `ServiceMonitor` to enable secure TLS scraping by Prometheus using certificates issued by `olmv1-metrics-ca`.
- The deployment was pushed to mount metrics certificates as secrets.

Benefits:
- Enhanced security with automated certificate lifecycle management.
- Production-ready TLS setup for Prometheus metrics scraping.

cmd/manager/main.go

@@ -198,6 +198,10 @@ func main() {
 		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
 		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
 		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		metricsServerOptions.CertDir = "/var/metrics/certs"
+		metricsServerOptions.CertName = "tls.crt"
+		metricsServerOptions.KeyName = "tls.key"
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{

config/base/prometheus/kustomization.yaml

@@ -1,2 +1,11 @@
 resources:
 - monitor.yaml
+
+# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# to securely reference certificates created and managed by cert-manager.
+# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
+# to mount the "metrics-server-cert" secret in the Manager Deployment.
+patches:
+  - path: monitor_tls_patch.yaml
+    target:
+      kind: ServiceMonitor

config/base/prometheus/monitor.yaml

@@ -17,3 +17,4 @@ spec:
   selector:
     matchLabels:
       control-plane: operator-controller-controller-manager
+

config/base/prometheus/monitor_tls_patch.yaml

@@ -0,0 +1,22 @@
+# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
+# using certificates managed by cert-manager
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: controller-manager-metrics-monitor
+  namespace: system
+spec:
+  endpoints:
+    - tlsConfig:
+        insecureSkipVerify: false
+        ca:
+          secret:
+            name: olmv1-metrics-ca
+            key: ca.crt
+        cert:
+          secret:
+            name: olmv1-metrics-ca
+            key: tls.crt
+        keySecret:
+          name: olmv1-metrics-ca
+          key: tls.key

config/components/ca/issuers.yaml

@@ -30,3 +30,28 @@ metadata:
 spec:
   ca:
     secretName: olmv1-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: olmv1-metrics-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: olmv1-metrics-ca
+  secretName: olmv1-metrics-ca
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: self-sign-issuer
+    kind: Issuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: olmv1-metrics-ca
+spec:
+  ca:
+    secretName: olmv1-metrics-ca

config/components/tls/kustomization.yaml

@@ -3,8 +3,13 @@ kind: Component
 namespace: olmv1-system
 resources:
 - resources/manager_cert.yaml
+- resources/manager_metrics_cert.yaml
 patches:
 - target:
     kind: Deployment
     name: controller-manager
   path: patches/manager_deployment_cert.yaml
+- target:
+    kind: Deployment
+    name: controller-manager
+  path: patches/manager_metrics_deployment_cert.yaml

config/components/tls/patches/manager_metrics_deployment_cert.yaml

@@ -0,0 +1,12 @@
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: olmv1-metrics-ca
+    secret:
+      secretName: olmv1-metrics-ca
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: olmv1-metrics-ca
+    mountPath: /var/metrics/certs
+    readOnly: true

config/components/tls/resources/manager_metrics_cert.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: olmv1-metrics-ca
+spec:
+  secretName: olmv1-metrics-ca
+  dnsNames:
+    - controller-manager-metrics-service.olmv1-system.svc
+    - controller-manager-metrics-service.olmv1-system.svc.cluster.local
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: olmv1-metrics-ca
+    kind: ClusterIssuer
+    group: cert-manager.io