From c1e6e347f65a8b42defde651723743051dbb4fd5 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 15 Aug 2024 14:06:33 -0600 Subject: [PATCH] core: add annotations and labels to detect version jobs The jobs to detect the ceph and csi versions now can have custom annotationso and labels added to them. They are very short-lived jobs, but they may need custom annotations in some environments to run. Signed-off-by: Travis Nielsen --- Documentation/CRDs/specification.md | 2 ++ deploy/examples/cluster.yaml | 5 +++++ deploy/examples/operator.yaml | 4 ++-- pkg/apis/ceph.rook.io/v1/annotations.go | 5 +++++ pkg/apis/ceph.rook.io/v1/annotations_test.go | 9 +++++++-- pkg/apis/ceph.rook.io/v1/keys.go | 1 + pkg/apis/ceph.rook.io/v1/labels.go | 4 ++++ pkg/apis/ceph.rook.io/v1/labels_test.go | 10 ++++++++-- pkg/operator/ceph/controller/network.go | 2 ++ pkg/operator/ceph/controller/version.go | 3 +++ pkg/operator/ceph/csi/controller.go | 6 ++++++ pkg/operator/ceph/csi/spec.go | 5 +++++ 12 files changed, 50 insertions(+), 6 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index fc32110fceb79..fd4346f7344db 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7482,6 +7482,8 @@ string

"clusterMetadata"

+

"cmdreporter"

+

"crashcollector"

"dashboard"

diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 6e9619774ed90..6127c79d4feed 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -194,9 +194,14 @@ spec: annotations: # all: # mon: + # mgr: # osd: + # exporter: + # crashcollector: # cleanup: # prepareosd: + # cmdreporter is for jobs to detect ceph and csi versions, and check network status + # cmdreporter: # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets. # And clusterMetadata annotations will not be merged with `all` annotations. # clusterMetadata: diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 95d6f57b00387..c5dae100e1ab2 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -32,9 +32,9 @@ data: ROOK_USE_CSI_OPERATOR: "false" # Enable the CSI driver. # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml - ROOK_CSI_ENABLE_CEPHFS: "true" + ROOK_CSI_ENABLE_CEPHFS: "false" # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. - ROOK_CSI_ENABLE_RBD: "true" + ROOK_CSI_ENABLE_RBD: "false" # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_NFS: "false" # Disable the CSI driver. diff --git a/pkg/apis/ceph.rook.io/v1/annotations.go b/pkg/apis/ceph.rook.io/v1/annotations.go index b2d77dc2d06ab..9610420be28c3 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations.go +++ b/pkg/apis/ceph.rook.io/v1/annotations.go @@ -72,6 +72,11 @@ func GetCephExporterAnnotations(a AnnotationsSpec) Annotations { return mergeAllAnnotationsWithKey(a, KeyCephExporter) } +// GetCmdReporterAnnotations returns the Annotations for jobs detecting versions +func GetCmdReporterAnnotations(a AnnotationsSpec) Annotations { + return mergeAllAnnotationsWithKey(a, KeyCmdReporter) +} + func GetClusterMetadataAnnotations(a AnnotationsSpec) Annotations { return a[KeyClusterMetadata] } diff --git a/pkg/apis/ceph.rook.io/v1/annotations_test.go b/pkg/apis/ceph.rook.io/v1/annotations_test.go index e3012a482d549..b832690c35679 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations_test.go +++ b/pkg/apis/ceph.rook.io/v1/annotations_test.go @@ -58,8 +58,9 @@ func TestCephAnnotationsMerge(t *testing.T) { // Merge with "all" testAnnotations = AnnotationsSpec{ - "all": {"allkey1": "allval1", "allkey2": "allval2"}, - "mgr": {"mgrkey": "mgrval"}, + "all": {"allkey1": "allval1", "allkey2": "allval2"}, + "mgr": {"mgrkey": "mgrval"}, + "cmdreporter": {"myversions": "detect"}, } a = GetMonAnnotations(testAnnotations) assert.Equal(t, "allval1", a["allkey1"]) @@ -70,6 +71,10 @@ func TestCephAnnotationsMerge(t *testing.T) { assert.Equal(t, "allval1", a["allkey1"]) assert.Equal(t, "allval2", a["allkey2"]) assert.Equal(t, 3, len(a)) + b := GetCmdReporterAnnotations(testAnnotations) + assert.Equal(t, "detect", b["myversions"]) + assert.Equal(t, "allval1", b["allkey1"]) + assert.Equal(t, "allval2", b["allkey2"]) } func TestAnnotationsSpec(t *testing.T) { diff --git a/pkg/apis/ceph.rook.io/v1/keys.go b/pkg/apis/ceph.rook.io/v1/keys.go index 05395f1e3423b..88467d31f3d5d 100644 --- a/pkg/apis/ceph.rook.io/v1/keys.go +++ b/pkg/apis/ceph.rook.io/v1/keys.go @@ -32,4 +32,5 @@ const ( KeyCrashCollector KeyType = "crashcollector" KeyClusterMetadata KeyType = "clusterMetadata" KeyCephExporter KeyType = "exporter" + KeyCmdReporter KeyType = "cmdreporter" ) diff --git a/pkg/apis/ceph.rook.io/v1/labels.go b/pkg/apis/ceph.rook.io/v1/labels.go index ad4bb14660d41..6825838f83541 100644 --- a/pkg/apis/ceph.rook.io/v1/labels.go +++ b/pkg/apis/ceph.rook.io/v1/labels.go @@ -87,6 +87,10 @@ func GetCephExporterLabels(a LabelsSpec) Labels { return mergeAllLabelsWithKey(a, KeyCephExporter) } +func GetCmdReporterLabels(a LabelsSpec) Labels { + return mergeAllLabelsWithKey(a, KeyCmdReporter) +} + func mergeAllLabelsWithKey(a LabelsSpec, name KeyType) Labels { all := a.All() if all != nil { diff --git a/pkg/apis/ceph.rook.io/v1/labels_test.go b/pkg/apis/ceph.rook.io/v1/labels_test.go index c4810045310ef..86668fb25f664 100644 --- a/pkg/apis/ceph.rook.io/v1/labels_test.go +++ b/pkg/apis/ceph.rook.io/v1/labels_test.go @@ -58,8 +58,9 @@ func TestCephLabelsMerge(t *testing.T) { // Merge with "all" testLabels = LabelsSpec{ - "all": {"allkey1": "allval1", "allkey2": "allval2"}, - "mgr": {"mgrkey": "mgrval"}, + "all": {"allkey1": "allval1", "allkey2": "allval2"}, + "mgr": {"mgrkey": "mgrval"}, + "cmdreporter": {"detect": "myversion"}, } a = GetMonLabels(testLabels) assert.Equal(t, "allval1", a["allkey1"]) @@ -70,6 +71,11 @@ func TestCephLabelsMerge(t *testing.T) { assert.Equal(t, "allval1", a["allkey1"]) assert.Equal(t, "allval2", a["allkey2"]) assert.Equal(t, 3, len(a)) + a = GetCmdReporterLabels(testLabels) + assert.Equal(t, "myversion", a["detect"]) + assert.Equal(t, "allval1", a["allkey1"]) + assert.Equal(t, "allval2", a["allkey2"]) + assert.Equal(t, 3, len(a)) } func TestLabelsSpec(t *testing.T) { diff --git a/pkg/operator/ceph/controller/network.go b/pkg/operator/ceph/controller/network.go index 4c64d37ab03ff..b97cb454af1f8 100644 --- a/pkg/operator/ceph/controller/network.go +++ b/pkg/operator/ceph/controller/network.go @@ -264,6 +264,8 @@ func discoverAddressRanges( job.Spec.Template.Annotations = map[string]string{ nadv1.NetworkAttachmentAnnot: netSelectionValue, } + cephv1.GetCmdReporterAnnotations(clusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(clusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) // use osd placement for net canaries b/c osd pods are present on both public and cluster nets cephv1.GetOSDPlacement(clusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) diff --git a/pkg/operator/ceph/controller/version.go b/pkg/operator/ceph/controller/version.go index 1585cef77695a..915672fda7d23 100644 --- a/pkg/operator/ceph/controller/version.go +++ b/pkg/operator/ceph/controller/version.go @@ -86,6 +86,9 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil + cephv1.GetCmdReporterAnnotations(cephClusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(cephClusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + stdout, stderr, retcode, err := versionReporter.Run(ctx, detectCephVersionTimeout) if err != nil { return nil, errors.Wrap(err, "failed to complete ceph version job") diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index bd9912f9e5950..db55853597da7 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -58,6 +58,8 @@ type ReconcileCSI struct { opManagerContext context.Context opConfig opcontroller.OperatorConfig clustersWithHolder []ClusterDetail + // the first cluster CR which will determine some settings for the csi driver + firstCephCluster *cephv1.ClusterSpec } // ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like @@ -276,6 +278,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, nil } + if r.firstCephCluster == nil { + r.firstCephCluster = &cephClusters.Items[i].Spec + } + // Load cluster info for later use in updating the ceph-csi configmap clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec) if err != nil { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index a4eef9cc0faf5..3d9924c313f71 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -25,6 +25,7 @@ import ( "strings" "time" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/cluster/telemetry" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" @@ -874,6 +875,10 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS job.Spec.Template.Spec.Affinity = &corev1.Affinity{ NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), } + if r.firstCephCluster != nil { + cephv1.GetCmdReporterAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + } stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout) if err != nil {