diff --git a/acm/Makefile b/acm/Makefile index ac5182de8..46cf37998 100644 --- a/acm/Makefile +++ b/acm/Makefile @@ -9,6 +9,7 @@ REGISTRY ?= ${ARO_HCP_IMAGE_ACR}.azurecr.io MCE_OPERATOR_BUNDLE_FILE = mce-operator-bundle.tgz HELM_BASE_DIR = deploy/helm MCE_CHART_DIR = ${HELM_BASE_DIR}/multicluster-engine +CLC_CHART_DIR = ${HELM_BASE_DIR}/clc-state-metrics MCE_CONFIG_DIR = ${HELM_BASE_DIR}/multicluster-engine-config MCE_NS = multicluster-engine POLICY_HELM_REPO = https://github.com/stolostron/mce-install-kube.git @@ -24,6 +25,12 @@ deploy: mce-config ${MCE_CONFIG_DIR} \ --namespace ${MCE_NS} \ --set global.registryOverride=${REGISTRY} + helm upgrade --install \ + clc-state-metrics ${CLC_CHART_DIR} \ + --namespace ${MCE_NS} \ + --set global.imageOverrides.clusterlifecycle_state_metrics=${REGISTRY}/multicluster-engine/clusterlifecycle-state-metrics-rhel9@sha256:${CLC_STATE_METRICS_IMAGE_DIGEST} \ + --set global.namespace=${MCE_NS} + helm-chart: @podman pull --arch amd64 ${MCE_OPERATOR_BUNDLE_IMAGE} diff --git a/acm/config.tmpl.mk b/acm/config.tmpl.mk index 200f6bd2a..c8c48a94e 100644 --- a/acm/config.tmpl.mk +++ b/acm/config.tmpl.mk @@ -1 +1,2 @@ ARO_HCP_IMAGE_ACR ?= {{ .svcAcrName }} +CLC_STATE_METRICS_IMAGE_DIGEST ?= {{ .mce.clcStateMetrics.imageDigest }} \ No newline at end of file diff --git a/acm/deploy/helm/clc-state-metrics/Chart.yaml b/acm/deploy/helm/clc-state-metrics/Chart.yaml new file mode 100644 index 000000000..bc3f139ab --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/Chart.yaml @@ -0,0 +1,7 @@ +# Original source https://github.com/stolostron/mce-install-kube/tree/main/clc-state-metrics +# TODO: Remove this chart and enable via MCE once MCE 2.8.0 is released (Feb 2025) +apiVersion: v2 +name: clusterlifecycle-state-metrics +description: A Helm chart for clusterlifecycle-state-metrics +version: 2.7.0 +appVersion: 2.7.0 diff --git a/acm/deploy/helm/clc-state-metrics/templates/_helpers.tpl b/acm/deploy/helm/clc-state-metrics/templates/_helpers.tpl new file mode 100644 index 000000000..303f97c33 --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/_helpers.tpl @@ -0,0 +1,12 @@ +{{- define "installNamespace" }} +{{- if .Values.global.namespace }} +{{- printf "%s" .Values.global.namespace }} +{{- else }} +{{- printf "multicluster-engine" }} +{{- end }} +{{- end }} + +{{- define "commonCN" }} +{{- printf "clusterlifecycle-state-metrics-v2.%s.svc" .Values.global.namespace }} +{{- end }} + diff --git a/acm/deploy/helm/clc-state-metrics/templates/cert-secret.yaml b/acm/deploy/helm/clc-state-metrics/templates/cert-secret.yaml new file mode 100644 index 000000000..b2189db07 --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/cert-secret.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: clusterlifecycle-state-metrics-certs + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-weight": "0" +type: kubernetes.io/tls +{{- if .Release.IsUpgrade }} +data: {{ (lookup "v1" "Secret" (include "installNamespace" .) "clusterlifecycle-state-metrics-certs").data }} +{{ else }} +{{ $ca := genCA (include "commonCN" .) 365 -}} +data: + tls.crt: {{ $ca.Cert | b64enc }} + tls.key: {{ $ca.Key | b64enc }} +{{- end }} diff --git a/acm/deploy/helm/clc-state-metrics/templates/clc-servicemonitor.yaml b/acm/deploy/helm/clc-state-metrics/templates/clc-servicemonitor.yaml new file mode 100644 index 000000000..c510a413e --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/clc-servicemonitor.yaml @@ -0,0 +1,19 @@ +apiVersion: azmonitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: clusterlifecycle-state-metrics-v2 + namespace: {{ .Values.global.namespace }} + labels: + clc-app: clusterlifecycle-state-metrics-v2 +spec: + selector: + matchLabels: + clc-app: clusterlifecycle-state-metrics-v2 + namespaceSelector: + matchNames: + - {{ .Values.global.namespace }} + endpoints: + - port: https + scheme: https + tlsConfig: + insecureSkipVerify: true \ No newline at end of file diff --git a/acm/deploy/helm/clc-state-metrics/templates/metrics-clusterrole.yaml b/acm/deploy/helm/clc-state-metrics/templates/metrics-clusterrole.yaml new file mode 100644 index 000000000..e0e99e797 --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/metrics-clusterrole.yaml @@ -0,0 +1,55 @@ +# Copyright Contributors to the Open Cluster Management project + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Values.org }}:cluster-lifecycle:clusterlifecycle-state-metrics-v2 +rules: +# Allow hub to manage configmap for leader selection +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - update + - get + - delete +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +# Allow hub to monitor and update status of csr +- apiGroups: ["hive.openshift.io"] + resources: ["clusterdeployments"] + verbs: ["get"] +- apiGroups: ["internal.open-cluster-management.io"] + resources: ["managedclusterinfos"] + verbs: ["get","list","watch"] +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters"] + verbs: ["get","list","watch"] +# Allow to query the CVO on the Hub Cluster to get the ClusterId +- apiGroups: ["config.openshift.io"] + resources: ["clusterversions"] + verbs: ["get"] +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods","services","endpoints"] + verbs: ["get","list","watch"] +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +# Allow hub to monitor add-ons & manifestworks +- apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons"] + verbs: ["get","list","watch"] +- apiGroups: ["work.open-cluster-management.io"] + resources: ["manifestworks"] + verbs: ["get","list","watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/acm/deploy/helm/clc-state-metrics/templates/metrics-clusterrole_binding.yaml b/acm/deploy/helm/clc-state-metrics/templates/metrics-clusterrole_binding.yaml new file mode 100644 index 000000000..e81d979ac --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/metrics-clusterrole_binding.yaml @@ -0,0 +1,14 @@ +# Copyright Contributors to the Open Cluster Management project + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Values.org }}:cluster-lifecycle:clusterlifecycle-state-metrics-v2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Values.org }}:cluster-lifecycle:clusterlifecycle-state-metrics-v2 +subjects: + - kind: ServiceAccount + name: clusterlifecycle-state-metrics-v2 + namespace: {{ .Values.global.namespace }} diff --git a/acm/deploy/helm/clc-state-metrics/templates/metrics-deployment.yaml b/acm/deploy/helm/clc-state-metrics/templates/metrics-deployment.yaml new file mode 100644 index 000000000..00ee419fa --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/metrics-deployment.yaml @@ -0,0 +1,128 @@ +# Copyright Contributors to the Open Cluster Management project + +kind: Deployment +apiVersion: apps/v1 +metadata: + name: clusterlifecycle-state-metrics-v2 + namespace: {{ .Values.global.namespace }} +spec: + replicas: 2 + selector: + matchLabels: + app: clusterlifecycle-state-metrics-v2 + template: + metadata: + labels: + app: clusterlifecycle-state-metrics-v2 + ocm-antiaffinity-selector: "clusterlifecycle-state-metrics-v2" + spec: + {{- if .Values.global.pullSecret }} + imagePullSecrets: + - name: {{ .Values.global.pullSecret }} + {{- end }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: ocm-antiaffinity-selector + operator: In + values: + - clusterlifecycle-state-metrics-v2 + topologyKey: topology.kubernetes.io/zone + weight: 70 + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: ocm-antiaffinity-selector + operator: In + values: + - clusterlifecycle-state-metrics-v2 + topologyKey: kubernetes.io/hostname + weight: 35 + serviceAccountName: clusterlifecycle-state-metrics-v2 + containers: + - name: clusterlifecycle-state-metrics + image: "{{ .Values.global.imageOverrides.clusterlifecycle_state_metrics }}" + imagePullPolicy: {{ .Values.global.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + resources: + requests: + memory: "32Mi" + cpu: "25m" + limits: + memory: "2Gi" + cpu: "500m" + args: + - "--http-port=8080" + - "--http-telemetry-port=8081" + - "--https-port=8443" + - "--https-telemetry-port=8444" + - "--tls-crt-file=/var/run/clusterlifecycle-state-metrics/tls.crt" + - "--tls-key-file=/var/run/clusterlifecycle-state-metrics/tls.key" +{{- if .Values.hubconfig.hubType }} + - "--hub-type={{ .Values.hubconfig.hubType }}" +{{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: GOMEMLIMIT + value: "1892MiB" +{{- if .Values.hubconfig.proxyConfigs }} + - name: HTTP_PROXY + value: {{ .Values.hubconfig.proxyConfigs.HTTP_PROXY }} + - name: HTTPS_PROXY + value: {{ .Values.hubconfig.proxyConfigs.HTTPS_PROXY }} + - name: NO_PROXY + value: {{ .Values.hubconfig.proxyConfigs.NO_PROXY }} +{{- end }} + readinessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 5 + volumeMounts: + - mountPath: "/var/run/clusterlifecycle-state-metrics" + name: certs + readOnly: true + securityContext: + {{- if .Values.global.deployOnOCP }} + {{- if semverCompare ">=4.11.0" .Values.hubconfig.ocpVersion }} + seccompProfile: + type: RuntimeDefault + {{- end }} + {{- end }} + volumes: + - name: certs + secret: + secretName: clusterlifecycle-state-metrics-certs +{{- with .Values.hubconfig.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.hubconfig.tolerations }} + tolerations: + {{- range . }} + - {{ if .Key }} key: {{ .Key }} {{- end }} + {{ if .Operator }} operator: {{ .Operator }} {{- end }} + {{ if .Value }} value: {{ .Value }} {{- end }} + {{ if .Effect }} effect: {{ .Effect }} {{- end }} + {{ if .TolerationSeconds }} tolerationSeconds: {{ .TolerationSeconds }} {{- end }} + {{- end }} +{{- end }} diff --git a/acm/deploy/helm/clc-state-metrics/templates/metrics-service.yaml b/acm/deploy/helm/clc-state-metrics/templates/metrics-service.yaml new file mode 100644 index 000000000..f158ce20c --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/metrics-service.yaml @@ -0,0 +1,20 @@ +# Copyright Contributors to the Open Cluster Management project + +apiVersion: v1 +kind: Service +metadata: + name: clusterlifecycle-state-metrics-v2 + namespace: {{ .Values.global.namespace }} + labels: + clc-app: clusterlifecycle-state-metrics-v2 + annotations: + service.beta.openshift.io/serving-cert-secret-name: clusterlifecycle-state-metrics-certs +spec: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + selector: + app: clusterlifecycle-state-metrics-v2 diff --git a/acm/deploy/helm/clc-state-metrics/templates/metrics-service_account.yaml b/acm/deploy/helm/clc-state-metrics/templates/metrics-service_account.yaml new file mode 100644 index 000000000..120e1ebdd --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/templates/metrics-service_account.yaml @@ -0,0 +1,7 @@ +# Copyright Contributors to the Open Cluster Management project + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clusterlifecycle-state-metrics-v2 + namespace: {{ .Values.global.namespace }} diff --git a/acm/deploy/helm/clc-state-metrics/values.yaml b/acm/deploy/helm/clc-state-metrics/values.yaml new file mode 100644 index 000000000..05c956a2a --- /dev/null +++ b/acm/deploy/helm/clc-state-metrics/values.yaml @@ -0,0 +1,15 @@ +global: + imageOverrides: + clusterlifecycle_state_metrics: "" + templateOverrides: {} + pullSecret: "" + namespace: multicluster-engine + hubSize: Small + deployOnOCP: "false" +hubconfig: + nodeSelector: {} + proxyConfigs: {} + replicaCount: 1 + tolerations: [] + ocpVersion: "4.17.0" +org: open-cluster-management diff --git a/config/config.schema.json b/config/config.schema.json index 071ec9dab..1679370e2 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -416,6 +416,19 @@ "restrictIstioIngress" ] }, + "mce": { + "type": "object", + "properties": { + "clcStateMetrics": { + "type": "object", + "properties": { + "imageDigest": { + "type": "string" + } + } + } + } + }, "mgmt": { "type": "object", "properties": { diff --git a/config/config.yaml b/config/config.yaml index 52fd074f3..0e744cf02 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -99,6 +99,11 @@ defaults: private: false softDelete: false + # MCE + mce: + clcStateMetrics: + imageDigest: bf5bb514e4d8af5e38317c3727d4cd9f90c22b293fe3e2367f9f0e179e0ee0c7 + serviceKeyVault: name: arohcp-svc-{{ .ctx.regionShort }} rg: hcp-underlay-{{ .ctx.regionShort }} diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index 2b70860ec..a7e9f5884 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -100,6 +100,11 @@ "restrictIstioIngress": false, "serverMqttClientName": "maestro-server-cspr-cs" }, + "mce": { + "clcStateMetrics": { + "imageDigest": "bf5bb514e4d8af5e38317c3727d4cd9f90c22b293fe3e2367f9f0e179e0ee0c7" + } + }, "mgmt": { "clusterServiceResourceId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-cspr-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service", "etcd": { diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index 0220889b0..1318c3d8b 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -100,6 +100,11 @@ "restrictIstioIngress": true, "serverMqttClientName": "maestro-server-dev-dev" }, + "mce": { + "clcStateMetrics": { + "imageDigest": "bf5bb514e4d8af5e38317c3727d4cd9f90c22b293fe3e2367f9f0e179e0ee0c7" + } + }, "mgmt": { "clusterServiceResourceId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-dev-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service", "etcd": { diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index 894ec7ad8..ea5f8bf85 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -100,6 +100,11 @@ "restrictIstioIngress": true, "serverMqttClientName": "maestro-server-usw3tst" }, + "mce": { + "clcStateMetrics": { + "imageDigest": "bf5bb514e4d8af5e38317c3727d4cd9f90c22b293fe3e2367f9f0e179e0ee0c7" + } + }, "mgmt": { "clusterServiceResourceId": "/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourcegroups/hcp-underlay-usw3tst-svc/providers/Microsoft.ManagedIdentity/userAssignedIdentities/clusters-service", "etcd": {