diff --git a/api/datadoghq/v2alpha1/const.go b/api/datadoghq/v2alpha1/const.go index ef52fb674..8a6e09b3d 100644 --- a/api/datadoghq/v2alpha1/const.go +++ b/api/datadoghq/v2alpha1/const.go @@ -178,8 +178,9 @@ const ( CriSocketVolumeName = "runtimesocketdir" RuntimeDirVolumePath = "/var/run" - KubeletAgentCAPath = "/var/run/host-kubelet-ca.crt" - KubeletCAVolumeName = "kubelet-ca" + KubeletAgentCAPath = "/var/run/host-kubelet-ca.crt" + KubeletCAVolumeName = "kubelet-ca" + KubeletPodResourcesVolumeName = "kubelet-pod-resources" APMSocketName = "apm.socket" diff --git a/api/datadoghq/v2alpha1/datadogagent_types.go b/api/datadoghq/v2alpha1/datadogagent_types.go index 461f98fdb..f51302f98 100644 --- a/api/datadoghq/v2alpha1/datadogagent_types.go +++ b/api/datadoghq/v2alpha1/datadogagent_types.go @@ -1079,6 +1079,11 @@ type KubeletConfig struct { // Default: '/var/run/host-kubelet-ca.crt' if hostCAPath is set, else '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' // +optional AgentCAPath string `json:"agentCAPath,omitempty"` + + // PodResourcesSocket is the path to the pod resources socket, to be used to read pod resource assignments + // Default: `/var/lib/kubelet/pod-resources/kubelet.sock` + // +optional + PodResourcesSocket string `json:"podResourcesSocket,omitempty"` } // HostPortConfig contains host port configuration. diff --git a/api/datadoghq/v2alpha1/envvar.go b/api/datadoghq/v2alpha1/envvar.go index dee11f317..fb0e49d92 100644 --- a/api/datadoghq/v2alpha1/envvar.go +++ b/api/datadoghq/v2alpha1/envvar.go @@ -44,6 +44,7 @@ const ( DDKubeResourcesNamespace = "DD_KUBE_RESOURCES_NAMESPACE" DDKubernetesResourcesLabelsAsTags = "DD_KUBERNETES_RESOURCES_LABELS_AS_TAGS" DDKubernetesResourcesAnnotationsAsTags = "DD_KUBERNETES_RESOURCES_ANNOTATIONS_AS_TAGS" + DDKubernetesPodResourcesSocket = "DD_KUBERNETES_KUBELET_PODRESOURCES_SOCKET" DDLeaderElection = "DD_LEADER_ELECTION" DDLogsEnabled = "DD_LOGS_ENABLED" DDNamespaceLabelsAsTags = "DD_KUBERNETES_NAMESPACE_LABELS_AS_TAGS" diff --git a/config/crd/bases/v1/datadoghq.com_datadogagents.yaml b/config/crd/bases/v1/datadoghq.com_datadogagents.yaml index 383dc4188..95f8f413a 100644 --- a/config/crd/bases/v1/datadoghq.com_datadogagents.yaml +++ b/config/crd/bases/v1/datadoghq.com_datadogagents.yaml @@ -2149,6 +2149,11 @@ spec: hostCAPath: description: HostCAPath is the host path where the kubelet CA certificate is stored. type: string + podResourcesSocket: + description: |- + PodResourcesSocket is the path to the pod resources socket, to be used to read pod resource assignments + Default: `/var/lib/kubelet/pod-resources/kubelet.sock` + type: string tlsVerify: description: |- TLSVerify toggles kubelet TLS verification. diff --git a/config/crd/bases/v1/datadoghq.com_datadogagents_v2alpha1.json b/config/crd/bases/v1/datadoghq.com_datadogagents_v2alpha1.json index 324d0267b..0257e69b9 100644 --- a/config/crd/bases/v1/datadoghq.com_datadogagents_v2alpha1.json +++ b/config/crd/bases/v1/datadoghq.com_datadogagents_v2alpha1.json @@ -2300,6 +2300,10 @@ "description": "HostCAPath is the host path where the kubelet CA certificate is stored.", "type": "string" }, + "podResourcesSocket": { + "description": "PodResourcesSocket is the path to the pod resources socket, to be used to read pod resource assignments\nDefault: `/var/lib/kubelet/pod-resources/kubelet.sock`", + "type": "string" + }, "tlsVerify": { "description": "TLSVerify toggles kubelet TLS verification.\nDefault: true", "type": "boolean" diff --git a/docs/configuration.v2alpha1.md b/docs/configuration.v2alpha1.md index 2932d7d5c..286853990 100644 --- a/docs/configuration.v2alpha1.md +++ b/docs/configuration.v2alpha1.md @@ -228,6 +228,7 @@ spec: | global.kubelet.host.secretKeyRef.name | Of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names | | global.kubelet.host.secretKeyRef.optional | Specify whether the Secret or its key must be defined | | global.kubelet.hostCAPath | HostCAPath is the host path where the kubelet CA certificate is stored. | +| global.kubelet.podResourcesSocket | PodResourcesSocket is the path to the pod resources socket, to be used to read pod resource assignments Default: `/var/lib/kubelet/pod-resources/kubelet.sock` | | global.kubelet.tlsVerify | TLSVerify toggles kubelet TLS verification. Default: true | | global.kubernetesResourcesAnnotationsAsTags | Provide a mapping of Kubernetes Resource Groups to annotations mapping to Datadog Tags. : : KUBERNETES_RESOURCE_GROUP should be in the form `{resource}.{group}` or `{resource}` (example: deployments.apps, pods) | | global.kubernetesResourcesLabelsAsTags | Provide a mapping of Kubernetes Resource Groups to labels mapping to Datadog Tags. : : KUBERNETES_RESOURCE_GROUP should be in the form `{resource}.{group}` or `{resource}` (example: deployments.apps, pods) | diff --git a/internal/controller/datadogagent/defaults/datadogagent_default.go b/internal/controller/datadogagent/defaults/datadogagent_default.go index 26e118105..6c0aa655f 100644 --- a/internal/controller/datadogagent/defaults/datadogagent_default.go +++ b/internal/controller/datadogagent/defaults/datadogagent_default.go @@ -115,6 +115,7 @@ const ( // defaultKubeletAgentCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" // defaultKubeletAgentCAPathHostPathSet = "/var/run/host-kubelet-ca.crt" + defaultKubeletPodResourcesSocket = "/var/lib/kubelet/pod-resources/kubelet.sock" defaultContainerStrategy = v2alpha1.OptimizedContainerStrategy @@ -192,6 +193,14 @@ func defaultGlobalConfig(ddaSpec *v2alpha1.DatadogAgentSpec) { apiutils.DefaultBooleanIfUnset(&ddaSpec.Global.FIPS.UseHTTPS, defaultFIPSUseHTTPS) } + if ddaSpec.Global.Kubelet == nil { + ddaSpec.Global.Kubelet = &v2alpha1.KubeletConfig{} + } + + if ddaSpec.Global.Kubelet.PodResourcesSocket == "" { + ddaSpec.Global.Kubelet.PodResourcesSocket = defaultKubeletPodResourcesSocket + } + apiutils.DefaultBooleanIfUnset(&ddaSpec.Global.RunProcessChecksInCoreAgent, defaultRunProcessChecksInCoreAgent) } diff --git a/internal/controller/datadogagent/override/global.go b/internal/controller/datadogagent/override/global.go index 3a1f43734..ea436440e 100644 --- a/internal/controller/datadogagent/override/global.go +++ b/internal/controller/datadogagent/override/global.go @@ -285,6 +285,36 @@ func applyGlobalSettings(logger logr.Logger, manager feature.PodTemplateManagers Value: agentCAPath, }) } + if config.Kubelet.PodResourcesSocket != "" { + manager.EnvVar().AddEnvVar(&corev1.EnvVar{ + Name: v2alpha1.DDKubernetesPodResourcesSocket, + Value: config.Kubelet.PodResourcesSocket, + }) + + podResourcesVol, podResourcesMount := volume.GetVolumes(v2alpha1.KubeletPodResourcesVolumeName, config.Kubelet.PodResourcesSocket, config.Kubelet.PodResourcesSocket, false) + if singleContainerStrategyEnabled { + manager.VolumeMount().AddVolumeMountToContainers( + &podResourcesMount, + []apicommon.AgentContainerName{ + apicommon.UnprivilegedSingleAgentContainerName, + }, + ) + manager.Volume().AddVolume(&podResourcesVol) + } else { + manager.VolumeMount().AddVolumeMountToContainers( + &podResourcesMount, + []apicommon.AgentContainerName{ + apicommon.CoreAgentContainerName, + apicommon.ProcessAgentContainerName, + apicommon.TraceAgentContainerName, + apicommon.SecurityAgentContainerName, + apicommon.AgentDataPlaneContainerName, + apicommon.SystemProbeContainerName, + }, + ) + manager.Volume().AddVolume(&podResourcesVol) + } + } } var runtimeVol corev1.Volume diff --git a/internal/controller/datadogagent/override/global_test.go b/internal/controller/datadogagent/override/global_test.go index 7137e91fa..8db296a66 100644 --- a/internal/controller/datadogagent/override/global_test.go +++ b/internal/controller/datadogagent/override/global_test.go @@ -7,6 +7,7 @@ package override import ( "fmt" + "slices" "testing" "github.com/DataDog/datadog-operator/pkg/constants" @@ -32,6 +33,7 @@ import ( const ( hostCAPath = "/host/ca/path/ca.crt" agentCAPath = "/agent/ca/path/ca.crt" + podResourcesSocket = "/var/lib/kubelet/pod-resources/kubelet.sock" dockerSocketPath = "/docker/socket/path/docker.sock" secretBackendCommand = "foo.sh" secretBackendArgs = "bar baz" @@ -52,9 +54,6 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Scheme: testScheme, } - var emptyVolumeMounts []*corev1.VolumeMount - emptyVolumes := []*corev1.Volume{} - tests := []struct { name string dda *v2alpha1.DatadogAgent @@ -69,7 +68,7 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { name: "Kubelet volume configured", singleContainerStrategyEnabled: false, dda: testutils.NewDatadogAgentBuilder(). - WithGlobalKubeletConfig(hostCAPath, agentCAPath, true). + WithGlobalKubeletConfig(hostCAPath, agentCAPath, true, podResourcesSocket). WithGlobalDockerSocketPath(dockerSocketPath). BuildWithDefaults(), wantEnvVars: getExpectedEnvVars([]*corev1.EnvVar{ @@ -81,20 +80,24 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Name: v2alpha1.DDKubeletCAPath, Value: agentCAPath, }, + { + Name: v2alpha1.DDKubernetesPodResourcesSocket, + Value: podResourcesSocket, + }, { Name: v2alpha1.DockerHost, Value: "unix:///host" + dockerSocketPath, }, }...), - wantVolumeMounts: getExpectedVolumeMounts(), - wantVolumes: getExpectedVolumes(), + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes, kubeletCAVolumes, criSocketVolume), + wantVolumes: getExpectedVolumes(defaultVolumes, kubeletCAVolumes, criSocketVolume), want: assertAll, }, { name: "Kubelet volume configured", singleContainerStrategyEnabled: true, dda: testutils.NewDatadogAgentBuilder(). - WithGlobalKubeletConfig(hostCAPath, agentCAPath, true). + WithGlobalKubeletConfig(hostCAPath, agentCAPath, true, podResourcesSocket). WithGlobalDockerSocketPath(dockerSocketPath). BuildWithDefaults(), wantEnvVars: getExpectedEnvVars([]*corev1.EnvVar{ @@ -106,13 +109,17 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Name: v2alpha1.DDKubeletCAPath, Value: agentCAPath, }, + { + Name: v2alpha1.DDKubernetesPodResourcesSocket, + Value: podResourcesSocket, + }, { Name: v2alpha1.DockerHost, Value: "unix:///host" + dockerSocketPath, }, }...), - wantVolumeMounts: getExpectedVolumeMounts(), - wantVolumes: getExpectedVolumes(), + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes, kubeletCAVolumes, criSocketVolume), + wantVolumes: getExpectedVolumes(defaultVolumes, kubeletCAVolumes, criSocketVolume), want: assertAllAgentSingleContainer, }, { @@ -125,8 +132,8 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Name: v2alpha1.DDChecksTagCardinality, Value: "orchestrator", }), - wantVolumeMounts: emptyVolumeMounts, - wantVolumes: emptyVolumes, + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes), + wantVolumes: getExpectedVolumes(defaultVolumes), want: assertAll, }, { @@ -139,8 +146,8 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Name: v2alpha1.DDOriginDetectionUnified, Value: "true", }), - wantVolumeMounts: emptyVolumeMounts, - wantVolumes: emptyVolumes, + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes), + wantVolumes: getExpectedVolumes(defaultVolumes), want: assertAll, }, { @@ -168,8 +175,8 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Value: "valueB", }, }...), - wantVolumeMounts: emptyVolumeMounts, - wantVolumes: emptyVolumes, + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes), + wantVolumes: getExpectedVolumes(defaultVolumes), want: assertAll, }, { @@ -196,8 +203,8 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Value: "60", }, }...), - wantVolumeMounts: emptyVolumeMounts, - wantVolumes: emptyVolumes, + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes), + wantVolumes: getExpectedVolumes(defaultVolumes), want: assertAll, wantDependency: assertSecretBackendGlobalPerms, }, @@ -225,8 +232,8 @@ func TestNodeAgentComponenGlobalSettings(t *testing.T) { Value: "60", }, }...), - wantVolumeMounts: emptyVolumeMounts, - wantVolumes: emptyVolumes, + wantVolumeMounts: getExpectedVolumeMounts(defaultVolumes), + wantVolumes: getExpectedVolumes(defaultVolumes), want: assertAll, wantDependency: assertSecretBackendSpecificPerms, }, @@ -256,15 +263,15 @@ func assertAll(t testing.TB, mgrInterface feature.PodTemplateManagers, expectedE traceAgentVolumeMounts := mgr.VolumeMountMgr.VolumeMountsByC[apicommon.TraceAgentContainerName] processAgentVolumeMounts := mgr.VolumeMountMgr.VolumeMountsByC[apicommon.ProcessAgentContainerName] - assert.True(t, apiutils.IsEqualStruct(coreAgentVolumeMounts, expectedVolumeMounts), "Volume mounts \ndiff = %s", cmp.Diff(coreAgentVolumeMounts, []*corev1.VolumeMount(nil))) - assert.True(t, apiutils.IsEqualStruct(traceAgentVolumeMounts, expectedVolumeMounts), "Volume mounts \ndiff = %s", cmp.Diff(traceAgentVolumeMounts, []*corev1.VolumeMount(nil))) - assert.True(t, apiutils.IsEqualStruct(processAgentVolumeMounts, expectedVolumeMounts), "Volume mounts \ndiff = %s", cmp.Diff(processAgentVolumeMounts, []*corev1.VolumeMount(nil))) + assert.ElementsMatch(t, coreAgentVolumeMounts, expectedVolumeMounts, "core-agent volume mounts \ndiff = %s", cmp.Diff(coreAgentVolumeMounts, expectedVolumeMounts)) + assert.ElementsMatch(t, traceAgentVolumeMounts, expectedVolumeMounts, "trace-agent volume mounts \ndiff = %s", cmp.Diff(traceAgentVolumeMounts, expectedVolumeMounts)) + assert.ElementsMatch(t, processAgentVolumeMounts, expectedVolumeMounts, "process-agent volume mounts \ndiff = %s", cmp.Diff(processAgentVolumeMounts, expectedVolumeMounts)) volumes := mgr.VolumeMgr.Volumes - assert.True(t, apiutils.IsEqualStruct(volumes, expectedVolumes), "Volumes \ndiff = %s", cmp.Diff(volumes, []*corev1.Volume{})) + assert.ElementsMatch(t, volumes, expectedVolumes, "Volumes \ndiff = %s", cmp.Diff(volumes, []*corev1.Volume{})) agentEnvVars := mgr.EnvVarMgr.EnvVarsByC[apicommon.AllContainers] - assert.True(t, apiutils.IsEqualStruct(agentEnvVars, expectedEnvVars), "Agent envvars \ndiff = %s", cmp.Diff(agentEnvVars, expectedEnvVars)) + assert.ElementsMatch(t, agentEnvVars, expectedEnvVars, "Agent envvars \ndiff = %s", cmp.Diff(agentEnvVars, expectedEnvVars)) } func assertAllAgentSingleContainer(t testing.TB, mgrInterface feature.PodTemplateManagers, expectedEnvVars []*corev1.EnvVar, expectedVolumes []*corev1.Volume, expectedVolumeMounts []*corev1.VolumeMount) { @@ -272,7 +279,7 @@ func assertAllAgentSingleContainer(t testing.TB, mgrInterface feature.PodTemplat agentSingleContainerVolumeMounts := mgr.VolumeMountMgr.VolumeMountsByC[apicommon.UnprivilegedSingleAgentContainerName] - assert.True(t, apiutils.IsEqualStruct(agentSingleContainerVolumeMounts, expectedVolumeMounts), "Volume mounts \ndiff = %s", cmp.Diff(agentSingleContainerVolumeMounts, []*corev1.VolumeMount(nil))) + assert.True(t, apiutils.IsEqualStruct(agentSingleContainerVolumeMounts, expectedVolumeMounts), "Volume mounts \ndiff = %s", cmp.Diff(agentSingleContainerVolumeMounts, expectedVolumeMounts)) volumes := mgr.VolumeMgr.Volumes assert.True(t, apiutils.IsEqualStruct(volumes, expectedVolumes), "Volumes \ndiff = %s", cmp.Diff(volumes, []*corev1.Volume{})) @@ -293,43 +300,104 @@ func getExpectedEnvVars(addedEnvVars ...*corev1.EnvVar) []*corev1.EnvVar { }, } + containsPodResourcesEnvVar := slices.ContainsFunc(addedEnvVars, func(envVar *corev1.EnvVar) bool { + return envVar.Name == v2alpha1.DDKubernetesPodResourcesSocket + }) + + if !containsPodResourcesEnvVar { + defaultEnvVars = append(defaultEnvVars, &corev1.EnvVar{ + Name: v2alpha1.DDKubernetesPodResourcesSocket, + Value: podResourcesSocket, + }) + } + return append(defaultEnvVars, addedEnvVars...) } -func getExpectedVolumes() []*corev1.Volume { - return []*corev1.Volume{ - { +type volumeConfig string + +const defaultVolumes volumeConfig = "default" +const kubeletCAVolumes volumeConfig = "kubeletCA" +const criSocketVolume volumeConfig = "criSocket" + +func getExpectedVolumes(configs ...volumeConfig) []*corev1.Volume { + volumes := []*corev1.Volume{} + + // Order is important for the comparisons in the assertion, so respect that + if slices.Contains(configs, kubeletCAVolumes) { + volumes = append(volumes, &corev1.Volume{ Name: v2alpha1.KubeletCAVolumeName, VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: hostCAPath, }, }, - }, - { + }) + } + + if slices.Contains(configs, defaultVolumes) { + volumes = append(volumes, &corev1.Volume{ + Name: v2alpha1.KubeletPodResourcesVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: podResourcesSocket, + }, + }, + }) + } + + if slices.Contains(configs, criSocketVolume) { + volumes = append(volumes, &corev1.Volume{ Name: v2alpha1.CriSocketVolumeName, VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ Path: dockerSocketPath, }, }, - }, + }) } + + return volumes } -func getExpectedVolumeMounts() []*corev1.VolumeMount { +func getDefaultVolumeMounts() []*corev1.VolumeMount { return []*corev1.VolumeMount{ { + Name: v2alpha1.KubeletPodResourcesVolumeName, + MountPath: podResourcesSocket, + ReadOnly: false, + }, + } +} + +func getExpectedVolumeMounts(configs ...volumeConfig) []*corev1.VolumeMount { + mounts := []*corev1.VolumeMount{} + + if slices.Contains(configs, kubeletCAVolumes) { + mounts = append(mounts, &corev1.VolumeMount{ Name: v2alpha1.KubeletCAVolumeName, MountPath: agentCAPath, ReadOnly: true, - }, - { + }) + } + + if slices.Contains(configs, defaultVolumes) { + mounts = append(mounts, &corev1.VolumeMount{ + Name: v2alpha1.KubeletPodResourcesVolumeName, + MountPath: podResourcesSocket, + ReadOnly: false, + }) + } + + if slices.Contains(configs, criSocketVolume) { + mounts = append(mounts, &corev1.VolumeMount{ Name: v2alpha1.CriSocketVolumeName, MountPath: "/host" + dockerSocketPath, ReadOnly: true, - }, + }) } + + return mounts } func addNameNamespaceToDDA(name string, namespace string, dda *v2alpha1.DatadogAgent) *v2alpha1.DatadogAgent { diff --git a/pkg/testutils/builder.go b/pkg/testutils/builder.go index e049886a2..303e3cf14 100644 --- a/pkg/testutils/builder.go +++ b/pkg/testutils/builder.go @@ -834,11 +834,12 @@ func (builder *DatadogAgentBuilder) WithHelmCheckValuesAsTags(valuesAsTags map[s // Global Kubelet -func (builder *DatadogAgentBuilder) WithGlobalKubeletConfig(hostCAPath, agentCAPath string, tlsVerify bool) *DatadogAgentBuilder { +func (builder *DatadogAgentBuilder) WithGlobalKubeletConfig(hostCAPath, agentCAPath string, tlsVerify bool, podResourcesSocket string) *DatadogAgentBuilder { builder.datadogAgent.Spec.Global.Kubelet = &v2alpha1.KubeletConfig{ - TLSVerify: apiutils.NewBoolPointer(tlsVerify), - HostCAPath: hostCAPath, - AgentCAPath: agentCAPath, + TLSVerify: apiutils.NewBoolPointer(tlsVerify), + HostCAPath: hostCAPath, + AgentCAPath: agentCAPath, + PodResourcesSocket: podResourcesSocket, } return builder }