core: add annotations and labels to detect version jobs

The jobs to detect the ceph and csi versions now can have
custom annotationso and labels added to them. They are
very short-lived jobs, but they may need custom annotations
in some environments to run.

Signed-off-by: Travis Nielsen <[email protected]>

deploy/examples/cluster.yaml

@@ -194,7 +194,11 @@ spec:
   annotations:
   #   all:
   #   mon:
+  #   mgr:
   #   osd:
+  #   exporter:
+  #   detectversion:
+  #   crashcollector:
   #   cleanup:
   #   prepareosd:
   # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets.

pkg/apis/ceph.rook.io/v1/annotations.go

@@ -72,6 +72,11 @@ func GetCephExporterAnnotations(a AnnotationsSpec) Annotations {
 	return mergeAllAnnotationsWithKey(a, KeyCephExporter)
 }
 
+// GetDetectVersionAnnotations returns the Annotations for jobs detecting versions
+func GetDetectVersionAnnotations(a AnnotationsSpec) Annotations {
+	return mergeAllAnnotationsWithKey(a, KeyDetectVersions)
+}
+
 func GetClusterMetadataAnnotations(a AnnotationsSpec) Annotations {
 	return a[KeyClusterMetadata]
 }

pkg/apis/ceph.rook.io/v1/annotations_test.go

@@ -58,8 +58,9 @@ func TestCephAnnotationsMerge(t *testing.T) {
 
 	// Merge with "all"
 	testAnnotations = AnnotationsSpec{
-		"all": {"allkey1": "allval1", "allkey2": "allval2"},
-		"mgr": {"mgrkey": "mgrval"},
+		"all":           {"allkey1": "allval1", "allkey2": "allval2"},
+		"mgr":           {"mgrkey": "mgrval"},
+		"detectversion": {"myversions": "detect"},
 	}
 	a = GetMonAnnotations(testAnnotations)
 	assert.Equal(t, "allval1", a["allkey1"])
@@ -70,6 +71,10 @@ func TestCephAnnotationsMerge(t *testing.T) {
 	assert.Equal(t, "allval1", a["allkey1"])
 	assert.Equal(t, "allval2", a["allkey2"])
 	assert.Equal(t, 3, len(a))
+	b := GetDetectVersionAnnotations(testAnnotations)
+	assert.Equal(t, "detect", b["myversions"])
+	assert.Equal(t, "allval1", b["allkey1"])
+	assert.Equal(t, "allval2", b["allkey2"])
 }
 
 func TestAnnotationsSpec(t *testing.T) {

pkg/apis/ceph.rook.io/v1/keys.go

@@ -32,4 +32,5 @@ const (
 	KeyCrashCollector  KeyType = "crashcollector"
 	KeyClusterMetadata KeyType = "clusterMetadata"
 	KeyCephExporter    KeyType = "exporter"
+	KeyDetectVersions  KeyType = "detectversion"
 )

pkg/apis/ceph.rook.io/v1/labels.go

@@ -87,6 +87,10 @@ func GetCephExporterLabels(a LabelsSpec) Labels {
 	return mergeAllLabelsWithKey(a, KeyCephExporter)
 }
 
+func GetDetectVersionLabels(a LabelsSpec) Labels {
+	return mergeAllLabelsWithKey(a, KeyDetectVersions)
+}
+
 func mergeAllLabelsWithKey(a LabelsSpec, name KeyType) Labels {
 	all := a.All()
 	if all != nil {

pkg/apis/ceph.rook.io/v1/labels_test.go

@@ -58,8 +58,9 @@ func TestCephLabelsMerge(t *testing.T) {
 
 	// Merge with "all"
 	testLabels = LabelsSpec{
-		"all": {"allkey1": "allval1", "allkey2": "allval2"},
-		"mgr": {"mgrkey": "mgrval"},
+		"all":           {"allkey1": "allval1", "allkey2": "allval2"},
+		"mgr":           {"mgrkey": "mgrval"},
+		"detectversion": {"detect": "myversion"},
 	}
 	a = GetMonLabels(testLabels)
 	assert.Equal(t, "allval1", a["allkey1"])
@@ -70,6 +71,11 @@ func TestCephLabelsMerge(t *testing.T) {
 	assert.Equal(t, "allval1", a["allkey1"])
 	assert.Equal(t, "allval2", a["allkey2"])
 	assert.Equal(t, 3, len(a))
+	a = GetDetectVersionLabels(testLabels)
+	assert.Equal(t, "myversion", a["detect"])
+	assert.Equal(t, "allval1", a["allkey1"])
+	assert.Equal(t, "allval2", a["allkey2"])
+	assert.Equal(t, 3, len(a))
 }
 
 func TestLabelsSpec(t *testing.T) {

pkg/operator/ceph/controller/version.go

@@ -86,6 +86,9 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string
 	cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)
 	job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil
 
+	cephv1.GetDetectVersionAnnotations(cephClusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	cephv1.GetDetectVersionLabels(cephClusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+
 	stdout, stderr, retcode, err := versionReporter.Run(ctx, detectCephVersionTimeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to complete ceph version job")

pkg/operator/ceph/csi/controller.go

@@ -58,6 +58,8 @@ type ReconcileCSI struct {
 	opManagerContext   context.Context
 	opConfig           opcontroller.OperatorConfig
 	clustersWithHolder []ClusterDetail
+	// the first cluster CR which will determine some settings for the csi driver
+	firstCephCluster *cephv1.ClusterSpec
 }
 
 // ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like
@@ -276,6 +278,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 			return reconcile.Result{}, nil
 		}
 
+		if r.firstCephCluster == nil {
+			r.firstCephCluster = &cluster.Spec
+		}
+
 		// Load cluster info for later use in updating the ceph-csi configmap
 		clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec)
 		if err != nil {

pkg/operator/ceph/csi/spec.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/telemetry"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -874,6 +875,10 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS
 	job.Spec.Template.Spec.Affinity = &corev1.Affinity{
 		NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}),
 	}
+	if r.firstCephCluster != nil {
+		cephv1.GetDetectVersionAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+		cephv1.GetDetectVersionLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	}
 
 	stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout)
 	if err != nil {