Merge pull request rook#12831 from rook/mergify/bp/release-1.12/pr-12791

core: restart ceph daemons when network updated (backport rook#12791)
travisn · Sep 12, 2023 · 41004d8 · 41004d8
2 parents 7e64b9c + 64fecc7
commit 41004d8
Show file tree

Hide file tree

Showing 13 changed files with 139 additions and 18 deletions.
diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go
@@ -184,7 +184,7 @@ func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
 			},
 		},
 		Env: append(
-			controller.DaemonEnvVars(c.spec.CephVersion.Image),
+			controller.DaemonEnvVars(&c.spec),
 			c.cephMgrOrchestratorModuleEnvs()...,
 		),
 		Resources:       cephv1.GetMgrResources(c.spec.Resources),
@@ -246,7 +246,7 @@ func (c *Cluster) makeCmdProxySidecarContainer(mgrConfig *mgrConfig) v1.Containe
 		Image:           c.spec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    append(controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName, c.spec.DataDirHostPath), adminKeyringVolMount),
-		Env:             append(controller.DaemonEnvVars(c.spec.CephVersion.Image), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}),
+		Env:             append(controller.DaemonEnvVars(&c.spec), v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}),
 		Resources:       cephv1.GetMgrResources(c.spec.Resources),
 		SecurityContext: controller.PodSecurityContext(),
 	}

diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go
@@ -275,7 +275,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) corev1.Container
 		VolumeMounts:    controller.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName, c.spec.DataDirHostPath),
 		SecurityContext: controller.PodSecurityContext(),
 		// filesystem creation does not require ports to be exposed
-		Env:       controller.DaemonEnvVars(c.spec.CephVersion.Image),
+		Env:       controller.DaemonEnvVars(&c.spec),
 		Resources: cephv1.GetMonResources(c.spec.Resources),
 	}
 }
@@ -314,7 +314,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) corev1.Container
 			},
 		},
 		Env: append(
-			controller.DaemonEnvVars(c.spec.CephVersion.Image),
+			controller.DaemonEnvVars(&c.spec),
 			k8sutil.PodIPEnvVar(podIPEnvVar),
 		),
 		Resources:     cephv1.GetMonResources(c.spec.Resources),

diff --git a/pkg/operator/ceph/cluster/nodedaemon/crash.go b/pkg/operator/ceph/cluster/nodedaemon/crash.go
@@ -167,10 +167,9 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container
 }
 
 func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container {
-	cephImage := cephCluster.Spec.CephVersion.Image
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
 	crashEnvVar := generateCrashEnvVar()
-	envVars := append(controller.DaemonEnvVars(cephImage), crashEnvVar)
+	envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), crashEnvVar)
 	volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
 	volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector())
 
@@ -179,7 +178,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephver
 		Command: []string{
 			"ceph-crash",
 		},
-		Image:           cephImage,
+		Image:           cephCluster.Spec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
 		Env:             envVars,
 		VolumeMounts:    volumeMounts,

diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go
@@ -171,7 +171,7 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
 	volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin())
 
 	envVars := append(
-		controller.DaemonEnvVars(cephCluster.Spec.CephVersion.Image),
+		controller.DaemonEnvVars(&cephCluster.Spec),
 		v1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())})
 
 	args := []string{

diff --git a/pkg/operator/ceph/cluster/nodedaemon/pruner.go b/pkg/operator/ceph/cluster/nodedaemon/pruner.go
@@ -170,8 +170,7 @@ func (r *ReconcileNode) deletev1betaJob(objectMeta metav1.ObjectMeta) {
 }
 
 func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.CephVersion) corev1.Container {
-	cephImage := cephCluster.Spec.CephVersion.Image
-	envVars := append(controller.DaemonEnvVars(cephImage), generateCrashEnvVar())
+	envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), generateCrashEnvVar())
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
 	volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
 	volumeMounts = append(volumeMounts, keyring.VolumeMount().CrashCollector())
@@ -188,7 +187,7 @@ func getCrashPruneContainer(cephCluster cephv1.CephCluster, cephVersion cephver.
 		Args: []string{
 			fmt.Sprintf("%d", cephCluster.Spec.CrashCollector.DaysToRetain),
 		},
-		Image:           cephImage,
+		Image:           cephCluster.Spec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(cephCluster.Spec.CephVersion.ImagePullPolicy),
 		Env:             envVars,
 		VolumeMounts:    volumeMounts,

diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go
@@ -583,6 +583,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd OSDInfo, provisionC
 			"",
 		))
 
+	envVars = append(envVars, controller.ApplyNetworkEnv(&c.spec)...)
 	podTemplateSpec := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   AppName,

diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go
@@ -122,7 +122,7 @@ func (r *ReconcileCephRBDMirror) makeMirroringDaemonContainer(daemonConfig *daem
 		Image:           r.cephClusterSpec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
-		Env:             controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(r.cephClusterSpec),
 		Resources:       rbdMirror.Spec.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 		WorkingDir:      config.VarLogCephDir,

diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go
@@ -394,13 +394,41 @@ func ContainerEnvVarReference(envVarName string) string {
 }
 
 // DaemonEnvVars returns the container environment variables used by all Ceph daemons.
-func DaemonEnvVars(image string) []v1.EnvVar {
+func DaemonEnvVars(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar {
+	networkEnv := ApplyNetworkEnv(cephClusterSpec)
+	cephDaemonsEnvVars := append(k8sutil.ClusterDaemonEnvVars(cephClusterSpec.CephVersion.Image), networkEnv...)
+
 	return append(
-		k8sutil.ClusterDaemonEnvVars(image),
+		cephDaemonsEnvVars,
 		config.StoredMonHostEnvVars()...,
 	)
 }
 
+func ApplyNetworkEnv(cephClusterSpec *cephv1.ClusterSpec) []v1.EnvVar {
+	if cephClusterSpec.Network.Connections != nil {
+		msgr2Required := false
+		encryptionEnabled := false
+		compressionEnabled := false
+		if cephClusterSpec.Network.Connections.RequireMsgr2 {
+			msgr2Required = true
+		}
+		if cephClusterSpec.Network.Connections.Encryption != nil && cephClusterSpec.Network.Connections.Encryption.Enabled {
+			encryptionEnabled = true
+		}
+		if cephClusterSpec.Network.Connections.Compression != nil && cephClusterSpec.Network.Connections.Compression.Enabled {
+			compressionEnabled = true
+		}
+		envVarValue := fmt.Sprintf("msgr2_%t_encryption_%t_compression_%t", msgr2Required, encryptionEnabled, compressionEnabled)
+
+		rookMsgr2Env := []v1.EnvVar{{
+			Name:  "ROOK_MSGR2",
+			Value: envVarValue,
+		}}
+		return rookMsgr2Env
+	}
+	return []v1.EnvVar{}
+}
+
 // AppLabels returns labels common for all Rook-Ceph applications which may be useful for admins.
 // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc.
 func AppLabels(appName, namespace string) map[string]string {

diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go
@@ -463,3 +463,97 @@ func TestGetContainerImagePullPolicy(t *testing.T) {
 		assert.Equal(t, exepctedImagePullPolicy, imagePullPolicy)
 	})
 }
+
+func TestDaemonEnvVars(t *testing.T) {
+	// No network settings specified
+	want := []v1.EnvVar{}
+
+	clusterSpec := &cephv1.ClusterSpec{}
+	got := ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Encryption is enabled
+	connections := &cephv1.ConnectionsSpec{
+		Encryption: &cephv1.EncryptionSpec{Enabled: true},
+	}
+
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_false_encryption_true_compression_false",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Compression is enabled
+	connections = &cephv1.ConnectionsSpec{
+		Compression: &cephv1.CompressionSpec{
+			Enabled: true,
+		},
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_false_encryption_false_compression_true",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Msgr2 is enabled
+	connections = &cephv1.ConnectionsSpec{
+		RequireMsgr2: true,
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_true_encryption_false_compression_false",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Msgr2, Compression, Encryption are enabled
+	connections = &cephv1.ConnectionsSpec{
+		RequireMsgr2: true,
+		Encryption: &cephv1.EncryptionSpec{
+			Enabled: true,
+		},
+		Compression: &cephv1.CompressionSpec{
+			Enabled: true,
+		},
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_true_encryption_true_compression_true",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+
+	// When Msgr2 is enabled but Compression, Encryption are disabled
+	connections = &cephv1.ConnectionsSpec{
+		RequireMsgr2: true,
+		Encryption: &cephv1.EncryptionSpec{
+			Enabled: false,
+		},
+		Compression: &cephv1.CompressionSpec{
+			Enabled: false,
+		},
+	}
+	clusterSpec.Network = cephv1.NetworkSpec{Connections: connections}
+
+	want = []v1.EnvVar{{
+		Name:  "ROOK_MSGR2",
+		Value: "msgr2_true_encryption_false_compression_false",
+	}}
+
+	got = ApplyNetworkEnv(clusterSpec)
+	assert.Equal(t, want, got)
+}
diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go
@@ -147,7 +147,7 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
 		Image:           c.clusterSpec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    controller.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName, c.clusterSpec.DataDirHostPath),
-		Env:             append(controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image), k8sutil.PodIPEnvVar(podIPEnvVar)),
+		Env:             append(controller.DaemonEnvVars(c.clusterSpec), k8sutil.PodIPEnvVar(podIPEnvVar)),
 		Resources:       c.fs.Spec.MetadataServer.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 		StartupProbe:    controller.GenerateStartupProbeExecDaemon(cephconfig.MdsType, mdsConfig.DaemonID),

diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go
@@ -119,7 +119,7 @@ func (r *ReconcileFilesystemMirror) makeFsMirroringDaemonContainer(daemonConfig
 		Image:           r.cephClusterSpec.CephVersion.Image,
 		ImagePullPolicy: controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy),
 		VolumeMounts:    controller.DaemonVolumeMounts(daemonConfig.DataPathMap, daemonConfig.ResourceName, r.cephClusterSpec.DataDirHostPath),
-		Env:             controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(r.cephClusterSpec),
 		Resources:       fsMirror.Spec.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 		// TODO:

diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go
@@ -243,7 +243,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig
 			nfsConfigMount,
 			dbusMount,
 		},
-		Env:             controller.DaemonEnvVars(r.cephClusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(r.cephClusterSpec),
 		Resources:       nfs.Spec.Server.Resources,
 		SecurityContext: controller.PodSecurityContext(),
 	}

diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go
@@ -359,7 +359,7 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
 			controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
 			c.mimeTypesVolumeMount(),
 		),
-		Env:             controller.DaemonEnvVars(c.clusterSpec.CephVersion.Image),
+		Env:             controller.DaemonEnvVars(c.clusterSpec),
 		Resources:       c.store.Spec.Gateway.Resources,
 		StartupProbe:    startupProbe,
 		LivenessProbe:   noLivenessProbe(),