From 64fecc75643d13d29c32a95dd8a63bff6fe88dae Mon Sep 17 00:00:00 2001 From: subhamkrai Date: Fri, 25 Aug 2023 19:58:46 +0530 Subject: [PATCH] core: restart ceph daemons when network updated We need to restart all the ceph daemons whenever cephCluster network settings are modified like requiremsgr2, encryption and compression. This required for Ceph to consider the new settings it require new ceph daemons all over. Signed-off-by: subhamkrai (cherry picked from commit 29d2b6a0715881a19edcda381a68a4ca6895db87) --- pkg/operator/ceph/cluster/mgr/spec.go | 4 +- pkg/operator/ceph/cluster/mon/spec.go | 4 +- pkg/operator/ceph/cluster/nodedaemon/crash.go | 5 +- .../ceph/cluster/nodedaemon/exporter.go | 2 +- .../ceph/cluster/nodedaemon/pruner.go | 5 +- pkg/operator/ceph/cluster/osd/spec.go | 1 + pkg/operator/ceph/cluster/rbd/spec.go | 2 +- pkg/operator/ceph/controller/spec.go | 32 ++++++- pkg/operator/ceph/controller/spec_test.go | 94 +++++++++++++++++++ pkg/operator/ceph/file/mds/spec.go | 2 +- pkg/operator/ceph/file/mirror/spec.go | 2 +- pkg/operator/ceph/nfs/spec.go | 2 +- pkg/operator/ceph/object/spec.go | 2 +- 13 files changed, 139 insertions(+), 18 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index bf616e5eed45..d6f8d24f346b 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -184,7 +184,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { }, }, Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), + controller.DaemonEnvVars(&c.spec), c.cephMgrOrchestratorModuleEnvs()..., ), Resources: cephv1.GetMgrResources(c.spec.Resources), @@ -246,7 +246,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe Image: c.spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), VolumeMounts: append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount), - Env: append(controller.DaemonEnvVars(c.spec.CephVersion.Image), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), + Env: append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}), Resources: cephv1.GetMgrResources(c.spec.Resources), SecurityContext: controller.PodSecurityContext(), } diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 8d4c8ab81a36..b146914f4b33 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -269,7 +269,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container VolumeMounts: controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath), SecurityContext: controller.PodSecurityContext(), // filesystem creation does not require ports to be exposed - Env: controller.DaemonEnvVars(c.spec.CephVersion.Image), + Env: controller.DaemonEnvVars(&c.spec), Resources: cephv1.GetMonResources(c.spec.Resources), } } @@ -308,7 +308,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container }, }, Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), + controller.DaemonEnvVars(&c.spec), k8sutil.PodIPEnvVar(podIPEnvVar), ), Resources: cephv1.GetMonResources(c.spec.Resources), diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go index 0e40b19125eb..83757853e7e6 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/crash.go +++ b/pkg/operator/ceph/cluster/nodedaemon/crash.go @@ -167,10 +167,9 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container } func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container { - cephImage := cephCluster.Spec.CephVersion.Image dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) crashEnvVar := generateCrashEnvVar() - envVars := append(controller.DaemonEnvVars(cephImage), crashEnvVar) + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), crashEnvVar) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector()) @@ -179,7 +178,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver Command: []string{ "ceph-crash", }, - Image: cephImage, + Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), Env: envVars, VolumeMounts: volumeMounts, diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index e8d8573db77c..7f6134b41daf 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -171,7 +171,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin()) envVars := append( - controller.DaemonEnvVars(cephCluster.Spec.CephVersion.Image), + controller.DaemonEnvVars(&cephCluster.Spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}) container := corev1.Container{ diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go index cb033130ec77..d03b525b2069 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/pruner.go +++ b/pkg/operator/ceph/cluster/nodedaemon/pruner.go @@ -170,8 +170,7 @@ func (r *ReconcileNode) deletev1betaJob(objectMeta metav1.ObjectMeta) { } func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container { - cephImage := cephCluster.Spec.CephVersion.Image - envVars := append(controller.DaemonEnvVars(cephImage), generateCrashEnvVar()) + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), generateCrashEnvVar()) dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector()) @@ -188,7 +187,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver. Args: []string{ fmt.Sprintf("%d", cephCluster.Spec.CrashCollector.DaysToRetain), }, - Image: cephImage, + Image: cephCluster.Spec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy), Env: envVars, VolumeMounts: volumeMounts, diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 3f3f9fbe0263..0cf055427cb3 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -583,6 +583,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC "", )) + envVars = append(envVars, controller.ApplyNetworkEnv(&c.spec)...) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: AppName, diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index 72d1eaf59476..6abd327128d7 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -122,7 +122,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem Image: r.cephClusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: rbdMirror.Spec.Resources, SecurityContext: controller.PodSecurityContext(), WorkingDir: config.VarLogCephDir, diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 7e919e7b6720..573c8962850b 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -394,13 +394,41 @@ func ContainerEnvVarReference(envVarName string) string { } // DaemonEnvVars returns the container environment variables used by all Ceph daemons. -func DaemonEnvVars(image string) []v1.EnvVar { +func DaemonEnvVars(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar { + networkEnv := ApplyNetworkEnv(cephClusterSpec) + cephDaemonsEnvVars := append(k8sutil.ClusterDaemonEnvVars(cephClusterSpec.CephVersion.Image), networkEnv...) + return append( - k8sutil.ClusterDaemonEnvVars(image), + cephDaemonsEnvVars, config.StoredMonHostEnvVars()..., ) } +func ApplyNetworkEnv(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar { + if cephClusterSpec.Network.Connections != nil { + msgr2Required := false + encryptionEnabled := false + compressionEnabled := false + if cephClusterSpec.Network.Connections.RequireMsgr2 { + msgr2Required = true + } + if cephClusterSpec.Network.Connections.Encryption != nil && cephClusterSpec.Network.Connections.Encryption.Enabled { + encryptionEnabled = true + } + if cephClusterSpec.Network.Connections.Compression != nil && cephClusterSpec.Network.Connections.Compression.Enabled { + compressionEnabled = true + } + envVarValue := fmt.Sprintf("msgr2_%t_encryption_%t_compression_%t", msgr2Required, encryptionEnabled, compressionEnabled) + + rookMsgr2Env := []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: envVarValue, + }} + return rookMsgr2Env + } + return []v1.EnvVar{} +} + // AppLabels returns labels common for all Rook-Ceph applications which may be useful for admins. // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc. func AppLabels(appName, namespace string) map[string]string { diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index a2697daaaf5a..fa82bd7d8ef3 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -463,3 +463,97 @@ func TestGetContainerImagePullPolicy(t *testing.T) { assert.Equal(t, exepctedImagePullPolicy, imagePullPolicy) }) } + +func TestDaemonEnvVars(t *testing.T) { + // No network settings specified + want := []v1.EnvVar{} + + clusterSpec := &cephv1.ClusterSpec{} + got := ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Encryption is enabled + connections := &cephv1.ConnectionsSpec{ + Encryption: &cephv1.EncryptionSpec{Enabled: true}, + } + + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_false_encryption_true_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Compression is enabled + connections = &cephv1.ConnectionsSpec{ + Compression: &cephv1.CompressionSpec{ + Enabled: true, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_false_encryption_false_compression_true", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2 is enabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_false_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2, Compression, Encryption are enabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + Encryption: &cephv1.EncryptionSpec{ + Enabled: true, + }, + Compression: &cephv1.CompressionSpec{ + Enabled: true, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_true_compression_true", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) + + // When Msgr2 is enabled but Compression, Encryption are disabled + connections = &cephv1.ConnectionsSpec{ + RequireMsgr2: true, + Encryption: &cephv1.EncryptionSpec{ + Enabled: false, + }, + Compression: &cephv1.CompressionSpec{ + Enabled: false, + }, + } + clusterSpec.Network = cephv1.NetworkSpec{Connections: connections} + + want = []v1.EnvVar{{ + Name: "ROOK_MSGR2", + Value: "msgr2_true_encryption_false_compression_false", + }} + + got = ApplyNetworkEnv(clusterSpec) + assert.Equal(t, want, got) +} diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 2f4edfe02a10..2136fe46df23 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -147,7 +147,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { Image: c.clusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath), - Env: append(controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), k8sutil.PodIPEnvVar(podIPEnvVar)), + Env: append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)), Resources: c.fs.Spec.MetadataServer.Resources, SecurityContext: controller.PodSecurityContext(), StartupProbe: controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID), diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index e750d02d8675..72212573b7d7 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -119,7 +119,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig Image: r.cephClusterSpec.CephVersion.Image, ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy), VolumeMounts: controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath), - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: fsMirror.Spec.Resources, SecurityContext: controller.PodSecurityContext(), // TODO: diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 068b82ed3fc2..dd03819c8fd3 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -243,7 +243,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig nfsConfigMount, dbusMount, }, - Env: controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), } diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index ac33b717179e..b85e7233357d 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -359,7 +359,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), c.mimeTypesVolumeMount(), ), - Env: controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), + Env: controller.DaemonEnvVars(c.clusterSpec), Resources: c.store.Spec.Gateway.Resources, StartupProbe: startupProbe, LivenessProbe: noLivenessProbe(),