From 28205a791d48ee6102bf1a609c6a3b4b5cc7232a Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:09:03 -0500 Subject: [PATCH 1/9] feat(rook-ceph): enable collecting RBD per-image IO statistics --- .../main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index eefddbe979633..ea405fd8760af 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -89,6 +89,8 @@ spec: - name: ceph-blockpool spec: failureDomain: host + deviceClass: nvme + enableRBDStats: true replicated: size: 3 storageClass: From 195f9fe1d3fb32621dcd6cd54d708c40ee4f3404 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:10:45 -0500 Subject: [PATCH 2/9] Update helmrelease.yaml --- .../main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index ea405fd8760af..54fafaa83d617 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -120,6 +120,7 @@ spec: - name: &cephFileSystemName ceph-filesystem spec: metadataPool: + deviceClass: nvme replicated: size: 3 dataPools: From 7a775f7ae5b21729edcfcc7e771e2a3596cc7cde Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:11:15 -0500 Subject: [PATCH 3/9] Update helmrelease.yaml --- .../main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index 54fafaa83d617..f5ba190f913d3 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -173,6 +173,7 @@ spec: # spec: # metadataPool: # failureDomain: host + # deviceClass: nvme # replicated: # size: 3 # dataPool: From 8b29b6432c3f636de40d80ab6347cc390a6c0e9a Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:14:27 -0500 Subject: [PATCH 4/9] Update helmrelease.yaml --- .../rook-ceph/rook-ceph/cluster/helmrelease.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index f5ba190f913d3..a13d59a2ec147 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -105,11 +105,11 @@ spec: imageFormat: "2" imageFeatures: layering,fast-diff,object-map,deep-flatten,exclusive-lock csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}" csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}" csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}" csi.storage.k8s.io/fstype: ext4 cephBlockPoolsVolumeSnapshotClass: enabled: true @@ -157,11 +157,11 @@ spec: volumeBindingMode: Immediate parameters: csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}" csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}" csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node - csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}" csi.storage.k8s.io/fstype: ext4 cephFileSystemVolumeSnapshotClass: enabled: true From 319e6f9459cf3deb00673396388cafee656cad16 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:18:57 -0500 Subject: [PATCH 5/9] Update helmrelease.yaml --- .../rook-ceph/rook-ceph/cluster/helmrelease.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index a13d59a2ec147..90f8428455af5 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -59,6 +59,7 @@ spec: ssl: false prometheusEndpoint: http://prometheus-operated.observability.svc.cluster.local:9090 mgr: + allowMultiplePerNode: false modules: - name: insights enabled: true @@ -66,6 +67,8 @@ spec: enabled: true - name: rook enabled: true + mon: + allowMultiplePerNode: false network: provider: host connections: @@ -76,15 +79,6 @@ spec: devicePathFilter: /dev/disk/by-id/nvme-Micron_7450_MTFDKBA800TFS_.* config: osdsPerDevice: "1" - placement: - mgr: &placement - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - mon: *placement cephBlockPools: - name: ceph-blockpool spec: From a442279c3aced72f2772b5fb763d3722834c5741 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:24:56 -0500 Subject: [PATCH 6/9] Update flux-diff.yaml --- .github/workflows/flux-diff.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flux-diff.yaml b/.github/workflows/flux-diff.yaml index 61ed623755a30..b8139dc800b5a 100644 --- a/.github/workflows/flux-diff.yaml +++ b/.github/workflows/flux-diff.yaml @@ -95,6 +95,6 @@ jobs: name: Flux Diff Successful runs-on: ubuntu-latest steps: - - if: ${{ !contains(needs.*.result, 'success') }} + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} name: Check matrix status run: exit 1 From d59df295f5b0135cb4f99092145b4694aec106e9 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:25:33 -0500 Subject: [PATCH 7/9] Update pre-pull-images.yaml --- .github/workflows/pre-pull-images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-pull-images.yaml b/.github/workflows/pre-pull-images.yaml index 7396e003e0ba9..acbec1561ad9c 100644 --- a/.github/workflows/pre-pull-images.yaml +++ b/.github/workflows/pre-pull-images.yaml @@ -104,6 +104,6 @@ jobs: name: Pre-pull Images Successful runs-on: ubuntu-latest steps: - - if: ${{ !contains(needs.*.result, 'success') }} + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} name: Check matrix status run: exit 1 From 3f832635f53432ee90529b567de0fea8cb7c9924 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:39:29 -0500 Subject: [PATCH 8/9] Update helmrelease.yaml --- .../main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index 90f8428455af5..8442cbabc768a 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -59,7 +59,6 @@ spec: ssl: false prometheusEndpoint: http://prometheus-operated.observability.svc.cluster.local:9090 mgr: - allowMultiplePerNode: false modules: - name: insights enabled: true @@ -67,8 +66,6 @@ spec: enabled: true - name: rook enabled: true - mon: - allowMultiplePerNode: false network: provider: host connections: From f19f244c375b138c7956f3ee2d77c914c4640131 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 1 Jan 2025 22:47:14 -0500 Subject: [PATCH 9/9] Update helmrelease.yaml --- .../main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index 8442cbabc768a..0b687b8d83fc0 100644 --- a/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/main/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -80,7 +80,6 @@ spec: - name: ceph-blockpool spec: failureDomain: host - deviceClass: nvme enableRBDStats: true replicated: size: 3 @@ -111,7 +110,6 @@ spec: - name: &cephFileSystemName ceph-filesystem spec: metadataPool: - deviceClass: nvme replicated: size: 3 dataPools: @@ -164,7 +162,6 @@ spec: # spec: # metadataPool: # failureDomain: host - # deviceClass: nvme # replicated: # size: 3 # dataPool: