diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 48d3b8d34bcb..26f5c483b643 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -184,7 +184,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, }, Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), + controller.DaemonEnvVars(&c.spec), c.cephMgrOrchestratorModuleEnvs()..., ), Resources: cephv1.GetMgrResources(c.spec.Resources), @@ -246,7 +246,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount), - Env: append(controller.DaemonEnvVars(c.spec.CephVersion.Image), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), + Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), Resources: cephv1.GetMgrResources(c.spec.Resources), SecurityContext: controller.PodSecurityContext(), } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index a3a6f2f64ff3..af8db024e8bb 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -275,7 +275,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), SecurityContext: controller.PodSecurityContext(), // filesystem creation does not require ports to be exposed - Env: controller.DaemonEnvVars(c.spec.CephVersion.Image), + Env: controller.DaemonEnvVars(&c.spec), Resources: cephv1.GetMonResources(c.spec.Resources), } } @@ -314,7 +314,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container }, }, Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), + controller.DaemonEnvVars(&c.spec), k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go index 0e40b19125eb..83757853e7e6 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/crash.go +++ b/pkg/operator/ceph/cluster/nodedaemon/crash.go @@ -167,10 +167,9 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container } func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container { - cephImage := cephCluster.Spec.CephVersion.Image dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) crashEnvVar := generateCrashEnvVar() - envVars := append(controller.DaemonEnvVars(cephImage), crashEnvVar) + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), crashEnvVar) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector()) @@ -179,7 +178,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver Command: []string{ "ceph-crash", }, - Image: cephImage, + Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), Env: envVars, VolumeMounts: volumeMounts, diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index 791fbed84c4a..39d9a6d66618 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -171,7 +171,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin()) envVars := append( - controller.DaemonEnvVars(cephCluster.Spec.CephVersion.Image), + controller.DaemonEnvVars(&cephCluster.Spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}) args := []string{ diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go index cb033130ec77..d03b525b2069 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/pruner.go +++ b/pkg/operator/ceph/cluster/nodedaemon/pruner.go @@ -170,8 +170,7 @@ func (r *ReconcileNode) deletev1betaJob(objectMeta metav1.ObjectMeta) { } func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container { - cephImage := cephCluster.Spec.CephVersion.Image - envVars := append(controller.DaemonEnvVars(cephImage), generateCrashEnvVar()) + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), generateCrashEnvVar()) dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector()) @@ -188,7 +187,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver. Args: []string{ fmt.Sprintf("%d", cephCluster.Spec.CrashCollector.DaysToRetain), }, - Image: cephImage, + Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), Env: envVars, VolumeMounts: volumeMounts, diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 789514ef4e7e..610afea95e9b 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -583,6 +583,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC "", )) + envVars = append(envVars, controller.ApplyNetworkEnv(&c.spec)...) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: AppName, diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index 2fff871fa65b..35ba7bae831f 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -122,7 +122,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem Image: r.cephClusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: rbdMirror.Spec.Resources, SecurityContext: controller.PodSecurityContext(), WorkingDir: config.VarLogCephDir, diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 7e919e7b6720..573c8962850b 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -394,13 +394,41 @@ func ContainerEnvVarReference(envVarName string) string { } // DaemonEnvVars returns the container environment variables used by all Ceph daemons. -func DaemonEnvVars(image string) []v1.EnvVar { +func DaemonEnvVars(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar { + networkEnv := ApplyNetworkEnv(cephClusterSpec) + cephDaemonsEnvVars := append(k8sutil.ClusterDaemonEnvVars(cephClusterSpec.CephVersion.Image), networkEnv...) + return append( - k8sutil.ClusterDaemonEnvVars(image), + cephDaemonsEnvVars, config.StoredMonHostEnvVars()..., ) } +func ApplyNetworkEnv(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar { + if cephClusterSpec.Network.Connections != nil { + msgr2Required := false + encryptionEnabled := false + compressionEnabled := false + if cephClusterSpec.Network.Connections.RequireMsgr2 { + msgr2Required = true + } + if cephClusterSpec.Network.Connections.Encryption != nil && cephClusterSpec.Network.Connections.Encryption.Enabled { + encryptionEnabled = true + } + if cephClusterSpec.Network.Connections.Compression != nil && cephClusterSpec.Network.Connections.Compression.Enabled { + compressionEnabled = true + } + envVarValue := fmt.Sprintf("msgr2_%t_encryption_%t_compression_%t", msgr2Required, encryptionEnabled, compressionEnabled) + + rookMsgr2Env := []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: envVarValue, + }} + return rookMsgr2Env + } + return []v1.EnvVar{} +} + // AppLabels returns labels common for all Rook-Ceph applications which may be useful for admins. // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc. func AppLabels(appName, namespace string) map[string]string { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index a2697daaaf5a..fa82bd7d8ef3 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -463,3 +463,97 @@ func TestGetContainerImagePullPolicy(t *testing.T) { assert.Equal(t, exepctedImagePullPolicy, imagePullPolicy) }) } + +func TestDaemonEnvVars(t *testing.T) { + // No network settings specified + want := []v1.EnvVar{} + + clusterSpec := &cephv1.ClusterSpec{} + got := ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Encryption is enabled + connections := &cephv1.ConnectionsSpec{ + Encryption: &cephv1.EncryptionSpec{Enabled: true}, + } + + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_false_encryption_true_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Compression is enabled + connections = &cephv1.ConnectionsSpec{ + Compression: &cephv1.CompressionSpec{ + Enabled: true, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_false_encryption_false_compression_true", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2 is enabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_false_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2, Compression, Encryption are enabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + Encryption: &cephv1.EncryptionSpec{ + Enabled: true, + }, + Compression: &cephv1.CompressionSpec{ + Enabled: true, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_true_compression_true", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2 is enabled but Compression, Encryption are disabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + Encryption: &cephv1.EncryptionSpec{ + Enabled: false, + }, + Compression: &cephv1.CompressionSpec{ + Enabled: false, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_false_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 5d86644f1252..f1d25cc1ce9e 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -147,7 +147,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { Image: c.clusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), - Env: append(controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), k8sutil.PodIPEnvVar(podIPEnvVar)), + Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, SecurityContext: controller.PodSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID), diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index 55a850b62597..4b197e039956 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -119,7 +119,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig Image: r.cephClusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: fsMirror.Spec.Resources, SecurityContext: controller.PodSecurityContext(), // TODO: diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 55602bcaae67..8c857edf48ad 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -243,7 +243,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig nfsConfigMount, dbusMount, }, - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 4b30db44b032..95e3f81ea896 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -359,7 +359,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.mimeTypesVolumeMount(), ), - Env: controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(c.clusterSpec), Resources: c.store.Spec.Gateway.Resources, StartupProbe: startupProbe, LivenessProbe: noLivenessProbe(),