core: add annotations and labels to detect version jobs

The jobs to detect the ceph and csi versions now can have
custom annotationso and labels added to them. They are
very short-lived jobs, but they may need custom annotations
in some environments to run.

Signed-off-by: Travis Nielsen <[email protected]>

Documentation/CRDs/specification.md

@@ -7482,6 +7482,8 @@ string
 <td></td>
 </tr><tr><td><p>&#34;clusterMetadata&#34;</p></td>
 <td></td>
+</tr><tr><td><p>&#34;cmdreporter&#34;</p></td>
+<td></td>
 </tr><tr><td><p>&#34;crashcollector&#34;</p></td>
 <td></td>
 </tr><tr><td><p>&#34;dashboard&#34;</p></td>

deploy/examples/cluster.yaml

@@ -194,9 +194,14 @@ spec:
   annotations:
   #   all:
   #   mon:
+  #   mgr:
   #   osd:
+  #   exporter:
+  #   crashcollector:
   #   cleanup:
   #   prepareosd:
+  # cmdreporter is for jobs to detect ceph and csi versions, and check network status
+  #   cmdreporter:
   # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets.
   # And clusterMetadata annotations will not be merged with `all` annotations.
   #    clusterMetadata:

deploy/examples/operator.yaml

@@ -32,9 +32,9 @@ data:
   ROOK_USE_CSI_OPERATOR: "false"
   # Enable the CSI driver.
   # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml
-  ROOK_CSI_ENABLE_CEPHFS: "true"
+  ROOK_CSI_ENABLE_CEPHFS: "false"
   # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below.
-  ROOK_CSI_ENABLE_RBD: "true"
+  ROOK_CSI_ENABLE_RBD: "false"
   # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below.
   ROOK_CSI_ENABLE_NFS: "false"
   # Disable the CSI driver.

pkg/apis/ceph.rook.io/v1/annotations.go

@@ -72,6 +72,11 @@ func GetCephExporterAnnotations(a AnnotationsSpec) Annotations {
 	return mergeAllAnnotationsWithKey(a, KeyCephExporter)
 }
 
+// GetCmdReporterAnnotations returns the Annotations for jobs detecting versions
+func GetCmdReporterAnnotations(a AnnotationsSpec) Annotations {
+	return mergeAllAnnotationsWithKey(a, KeyCmdReporter)
+}
+
 func GetClusterMetadataAnnotations(a AnnotationsSpec) Annotations {
 	return a[KeyClusterMetadata]
 }

pkg/apis/ceph.rook.io/v1/annotations_test.go

@@ -58,8 +58,9 @@ func TestCephAnnotationsMerge(t *testing.T) {
 
 	// Merge with "all"
 	testAnnotations = AnnotationsSpec{
-		"all": {"allkey1": "allval1", "allkey2": "allval2"},
-		"mgr": {"mgrkey": "mgrval"},
+		"all":         {"allkey1": "allval1", "allkey2": "allval2"},
+		"mgr":         {"mgrkey": "mgrval"},
+		"cmdreporter": {"myversions": "detect"},
 	}
 	a = GetMonAnnotations(testAnnotations)
 	assert.Equal(t, "allval1", a["allkey1"])
@@ -70,6 +71,10 @@ func TestCephAnnotationsMerge(t *testing.T) {
 	assert.Equal(t, "allval1", a["allkey1"])
 	assert.Equal(t, "allval2", a["allkey2"])
 	assert.Equal(t, 3, len(a))
+	b := GetCmdReporterAnnotations(testAnnotations)
+	assert.Equal(t, "detect", b["myversions"])
+	assert.Equal(t, "allval1", b["allkey1"])
+	assert.Equal(t, "allval2", b["allkey2"])
 }
 
 func TestAnnotationsSpec(t *testing.T) {

pkg/apis/ceph.rook.io/v1/keys.go

@@ -32,4 +32,5 @@ const (
 	KeyCrashCollector  KeyType = "crashcollector"
 	KeyClusterMetadata KeyType = "clusterMetadata"
 	KeyCephExporter    KeyType = "exporter"
+	KeyCmdReporter     KeyType = "cmdreporter"
 )

pkg/apis/ceph.rook.io/v1/labels.go

@@ -87,6 +87,10 @@ func GetCephExporterLabels(a LabelsSpec) Labels {
 	return mergeAllLabelsWithKey(a, KeyCephExporter)
 }
 
+func GetCmdReporterLabels(a LabelsSpec) Labels {
+	return mergeAllLabelsWithKey(a, KeyCmdReporter)
+}
+
 func mergeAllLabelsWithKey(a LabelsSpec, name KeyType) Labels {
 	all := a.All()
 	if all != nil {

pkg/apis/ceph.rook.io/v1/labels_test.go

@@ -58,8 +58,9 @@ func TestCephLabelsMerge(t *testing.T) {
 
 	// Merge with "all"
 	testLabels = LabelsSpec{
-		"all": {"allkey1": "allval1", "allkey2": "allval2"},
-		"mgr": {"mgrkey": "mgrval"},
+		"all":         {"allkey1": "allval1", "allkey2": "allval2"},
+		"mgr":         {"mgrkey": "mgrval"},
+		"cmdreporter": {"detect": "myversion"},
 	}
 	a = GetMonLabels(testLabels)
 	assert.Equal(t, "allval1", a["allkey1"])
@@ -70,6 +71,11 @@ func TestCephLabelsMerge(t *testing.T) {
 	assert.Equal(t, "allval1", a["allkey1"])
 	assert.Equal(t, "allval2", a["allkey2"])
 	assert.Equal(t, 3, len(a))
+	a = GetCmdReporterLabels(testLabels)
+	assert.Equal(t, "myversion", a["detect"])
+	assert.Equal(t, "allval1", a["allkey1"])
+	assert.Equal(t, "allval2", a["allkey2"])
+	assert.Equal(t, 3, len(a))
 }
 
 func TestLabelsSpec(t *testing.T) {

pkg/operator/ceph/controller/network.go

@@ -264,6 +264,8 @@ func discoverAddressRanges(
 	job.Spec.Template.Annotations = map[string]string{
 		nadv1.NetworkAttachmentAnnot: netSelectionValue,
 	}
+	cephv1.GetCmdReporterAnnotations(clusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	cephv1.GetCmdReporterLabels(clusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
 
 	// use osd placement for net canaries b/c osd pods are present on both public and cluster nets
 	cephv1.GetOSDPlacement(clusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)

pkg/operator/ceph/controller/version.go

@@ -86,6 +86,9 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string
 	cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec)
 	job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil
 
+	cephv1.GetCmdReporterAnnotations(cephClusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	cephv1.GetCmdReporterLabels(cephClusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+
 	stdout, stderr, retcode, err := versionReporter.Run(ctx, detectCephVersionTimeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to complete ceph version job")

pkg/operator/ceph/csi/controller.go

@@ -58,6 +58,8 @@ type ReconcileCSI struct {
 	opManagerContext   context.Context
 	opConfig           opcontroller.OperatorConfig
 	clustersWithHolder []ClusterDetail
+	// the first cluster CR which will determine some settings for the csi driver
+	firstCephCluster *cephv1.ClusterSpec
 }
 
 // ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like
@@ -276,6 +278,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 			return reconcile.Result{}, nil
 		}
 
+		if r.firstCephCluster == nil {
+			r.firstCephCluster = &cephClusters.Items[i].Spec
+		}
+
 		// Load cluster info for later use in updating the ceph-csi configmap
 		clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec)
 		if err != nil {

pkg/operator/ceph/csi/spec.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/telemetry"
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -874,6 +875,10 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS
 	job.Spec.Template.Spec.Affinity = &corev1.Affinity{
 		NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}),
 	}
+	if r.firstCephCluster != nil {
+		cephv1.GetCmdReporterAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+		cephv1.GetCmdReporterLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta)
+	}
 
 	stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout)
 	if err != nil {