Merge branch 'main' of github.com:DataDog/datadog-agent into misteria…

…ud/ASCII-2591-migrate-agent-ipc-clients-to-use-certificate

.github/dependabot.yaml

@@ -24,7 +24,6 @@ updates:
       - dependency-name: github.com/mholt/archiver
       - dependency-name: github.com/prometheus/client_golang
       - dependency-name: github.com/spf13/cast
-      - dependency-name: github.com/ugorji/go
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
       # Ignore golang.org/x/... deps to avoid noise, they are updated together, pretty regularly

.github/workflows/backport-pr.yml

@@ -24,7 +24,7 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0
+      - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
         id: app-token
         with:
           app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }}

.github/workflows/create_release_schedule.yml

@@ -7,8 +7,8 @@ on:
         description: 'Full version of the release to schedule (e.g. 7.31.0)'
         required: true
         type: string
-      freeze_date:
-        description: 'Isoformat date when we freeze CI (e.g. 2023-12-31)'
+      cutoff_date:
+        description: 'Isoformat date when we the cut-off happens (e.g. 2023-12-31)'
         required: true
         type: string
 
@@ -38,6 +38,6 @@ jobs:
         ATLASSIAN_USERNAME : ${{ secrets.ATLASSIAN_USERNAME }}
         ATLASSIAN_PASSWORD : ${{ secrets.ATLASSIAN_PASSWORD }}
         INPUT_VERSION: ${{ github.event.inputs.version }}
-        FREEZE_DATE: ${{ github.event.inputs.freeze_date }}
+        CUTOFF_DATE: ${{ github.event.inputs.cutoff_date }}
       run: |
-        inv -e release.create-schedule --version "$INPUT_VERSION" --freeze-date "$FREEZE_DATE"
+        inv -e release.create-schedule --version "$INPUT_VERSION" --cutoff-date "CUTOFF_DATE"

.github/workflows/label-analysis.yml

@@ -140,7 +140,7 @@ jobs:
         run: inv -e github.agenttelemetry-list-change-ack-check --pr-id=${{ github.event.pull_request.number }}
 
   ask-reviews:
-    if: github.triggering_actor != 'dd-devflow[bot]' && github.event.action != 'synchronize'
+    if: github.triggering_actor != 'dd-devflow[bot]' && github.event.action == 'labeled' && github.event.label.name == 'ask-review'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository

.github/workflows/update_dependencies.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions: {} # the workflow uses the GitHub App token to create the PR so no specific permissions needed here
     steps:
-      - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0
+      - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
         id: app-token
         with:
           app-id: ${{ vars.DD_GITHUB_TOKEN_GENERATOR_APP_ID }}

.gitlab/common/test_infra_version.yml

@@ -5,5 +5,5 @@
 
 ---
 variables:
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 36860d3c29b4
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: f61405297d57
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ''

.gitlab/container_build/docker_windows.yml

@@ -54,6 +54,7 @@
   variables:
     AGENT_ZIP: "datadog-agent-7*-x86_64.zip"
     BUILD_ARG: "--build-arg BASE_IMAGE=mcr.microsoft.com/powershell:lts-nanoserver-${VARIANT} --build-arg WITH_JMX=${WITH_JMX} --build-arg VARIANT=${VARIANT} --build-arg INSTALL_INFO=nano-${VARIANT}"
+  retry: 2
 
 .docker_build_agent7_windows_servercore_common:
   extends:

.gitlab/dev_container_deploy/docker_linux.yml

@@ -31,6 +31,8 @@ dev_branch_multiarch-a7:
         IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3
       - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64
         IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-py3-jmx
+      - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64
+        IMG_DESTINATIONS: agent-dev:${CI_COMMIT_SHORT_SHA}-py3-jmx
 
 dev_branch_multiarch-fips:
   extends: .docker_publish_job_definition

.gitlab/e2e/e2e.yml

@@ -642,6 +642,7 @@ new-e2e-gpu:
   variables:
     TARGETS: ./tests/gpu                          # the target path where tests are
     TEAM: ebpf-platform
+    E2E_PULUMI_LOG_LEVEL: 8 # incident-33572
   needs:                                         # list of required jobs. By default gitlab waits for any previous jobs.
     - !reference [.needs_new_e2e_template]
     - deploy_deb_testing-a7_x64                  # agent 7 debian package

.gitlab/e2e_install_packages/centos.yml

@@ -64,6 +64,16 @@ new-e2e-agent-platform-install-script-centos-dogstatsd-a7-x86_64:
   variables:
     FLAVOR: datadog-dogstatsd
 
+new-e2e-agent-platform-install-script-centos-datadog-fips-agent-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_install_script
+    - .new-e2e_os_centos
+    - .new-e2e_centos_a7_x86_64
+    - .new-e2e_agent_a7
+  variables:
+    FLAVOR: datadog-fips-agent
+
 new-e2e-agent-platform-install-script-centos-fips-a7-x86_64:
   extends:
     - .new_e2e_template
@@ -96,6 +106,16 @@ new-e2e-agent-platform-install-script-centos-fips-dogstatsd-a7-x86_64:
   variables:
     FLAVOR: datadog-dogstatsd
 
+new-e2e-agent-platform-install-script-centos-fips-datadog-fips-agent-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_install_script
+    - .new-e2e_os_centos
+    - .new-e2e_centos-fips_a7_x86_64
+    - .new-e2e_agent_a7
+  variables:
+    FLAVOR: datadog-fips-agent
+
 new-e2e-agent-platform-step-by-step-centos-a7-x86_64:
   extends:
     - .new_e2e_template

.gitlab/e2e_install_packages/debian.yml

@@ -75,6 +75,16 @@ new-e2e-agent-platform-install-script-debian-dogstatsd-a7-x86_64:
   variables:
     FLAVOR: datadog-dogstatsd
 
+new-e2e-agent-platform-install-script-debian-fips-agent-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_install_script
+    - .new-e2e_os_debian
+    - .new-e2e_debian_a7_x86_64
+    - .new-e2e_agent_a7
+  variables:
+    FLAVOR: datadog-fips-agent
+
 new-e2e-agent-platform-install-script-debian-heroku-agent-a7-x86_64:
   extends:
     - .new_e2e_template

.gitlab/e2e_install_packages/suse.yml

@@ -71,6 +71,16 @@ new-e2e-agent-platform-install-script-suse-dogstatsd-a7-x86_64:
   variables:
     FLAVOR: datadog-dogstatsd
 
+new-e2e-agent-platform-install-script-suse-fips-agent-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_install_script
+    - .new-e2e_os_suse
+    - .new-e2e_suse_a7_x86_64
+    - .new-e2e_agent_a7
+  variables:
+    FLAVOR: datadog-fips-agent
+
 new-e2e-agent-platform-step-by-step-suse-a7-x86_64:
   extends:
     - .new_e2e_template

.gitlab/e2e_install_packages/ubuntu.yml

@@ -99,6 +99,16 @@ new-e2e-agent-platform-install-script-ubuntu-heroku-agent-a7-x86_64:
   variables:
     FLAVOR: datadog-heroku-agent
 
+new-e2e-agent-platform-install-script-ubuntu-fips-agent-a7-x86_64:
+  extends:
+    - .new_e2e_template
+    - .new-e2e_install_script
+    - .new-e2e_os_ubuntu
+    - .new-e2e_ubuntu_a7_x86_64
+    - .new-e2e_agent_a7
+  variables:
+    FLAVOR: datadog-fips-agent
+
 new-e2e-agent-platform-step-by-step-ubuntu-a7-x86_64:
   extends:
     - .new_e2e_template

.gitlab/e2e_testing_deploy/e2e_deploy.yml

@@ -52,6 +52,7 @@ deploy_deb_testing-a7_x64:
     [
       "installer_deb-amd64",
       "agent_deb-x64-a7",
+      "agent_deb-x64-a7-fips",
       "agent_heroku_deb-x64-a7",
       "iot_agent_deb-x64",
       "dogstatsd_deb-x64",
@@ -75,7 +76,7 @@ deploy_deb_testing-a7_arm64:
     - !reference [.manual]
   extends:
     - .deploy_deb_testing-a7
-  needs: ["installer_deb-arm64", "agent_deb-arm64-a7", "lint_linux-arm64"]
+  needs: ["installer_deb-arm64", "agent_deb-arm64-a7", "agent_deb-arm64-a7-fips", "lint_linux-arm64"]
   script:
     - *setup_apt_signing_key
     - set +x # make sure we don't output the creds to the build log
@@ -105,6 +106,7 @@ deploy_rpm_testing-a7_x64:
     [
       "installer_rpm-amd64",
       "agent_rpm-x64-a7",
+      "agent_rpm-x64-a7-fips",
       "iot_agent_rpm-x64",
       "dogstatsd_rpm-x64",
       "lint_linux-x64",
@@ -121,7 +123,7 @@ deploy_rpm_testing-a7_arm64:
     - !reference [.manual]
   extends:
     - .deploy_rpm_testing-a7
-  needs: ["installer_rpm-arm64", "agent_rpm-arm64-a7", "lint_linux-arm64"]
+  needs: ["installer_rpm-arm64", "agent_rpm-arm64-a7", "agent_rpm-arm64-a7-fips", "lint_linux-arm64"]
   script:
     - *setup_rpm_signing_key
     - set +x
@@ -141,6 +143,7 @@ deploy_suse_rpm_testing_x64-a7:
       "agent_suse-x64-a7",
       "iot_agent_suse-x64",
       "dogstatsd_suse-x64",
+      "agent_suse-x64-a7-fips",
       "lint_linux-x64",
     ]
   variables:
@@ -160,7 +163,7 @@ deploy_suse_rpm_testing_arm64-a7:
   stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
-  needs: ["installer_suse_rpm-arm64", "agent_suse-arm64-a7", "lint_linux-arm64"]
+  needs: ["installer_suse_rpm-arm64", "agent_suse-arm64-a7", "agent_suse-arm64-a7-fips", "lint_linux-arm64"]
   variables:
     DD_PIPELINE_ID: $CI_PIPELINE_ID-a7
   before_script:

.gitlab/integration_test/windows.yml

@@ -30,7 +30,7 @@
       -e VCPKG_BINARY_SOURCES="clear;x-azblob,${vcpkgBlobSaSUrl}"
       -e PIP_INDEX_URL=${PIP_INDEX_URL}
       registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
-      c:\mnt\tasks\winbuildscripts\integrationtests.bat
+      powershell.exe -c "c:\mnt\tasks\winbuildscripts\Invoke-IntegrationTests.ps1 -BuildOutOfSource 1 -CheckGoVersion 1 -InstallDeps 1"
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
 
 integration_tests_windows-x64:

.gitlab/lint/windows.yml

@@ -20,7 +20,8 @@
       -e CI_PIPELINE_ID=${CI_PIPELINE_ID}
       -e CI_PROJECT_NAME=${CI_PROJECT_NAME}
       -e GOMODCACHE="c:\modcache"
-      registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\lint.bat
+      registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:$Env:DATADOG_AGENT_WINBUILDIMAGES
+      powershell.exe -c "c:\mnt\tasks\winbuildscripts\Invoke-Linters.ps1 -BuildOutOfSource 1 -CheckGoVersion 1 -InstallDeps 1"
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
 
 lint_windows-x64:

.gitlab/notify/notify.yml

@@ -28,23 +28,8 @@ notify:
     - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN
     - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
     - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt
-    - |
-      # Do not send notifications if this is a child pipeline of another repo
-      # The triggering repo should already have its own notification system
-      if [ "$CI_PIPELINE_SOURCE" != "pipeline" ]; then
-        if [ "$DEPLOY_AGENT" = "true" ]; then
-          invoke -e notify.send-message --notification-type "deploy"
-        elif [ "$CI_PIPELINE_SOURCE" != "push" ]; then
-          invoke -e notify.send-message --notification-type "trigger"
-        else
-          invoke -e notify.send-message --notification-type "merge"
-        fi
-        if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
-          invoke notify.check-consistent-failures
-        fi
-      else
-        echo "This pipeline was triggered by another repository, skipping notification."
-      fi
+    - invoke -e notify.send-message -p $CI_PIPELINE_ID
+    - invoke -e notify.check-consistent-failures -p $CI_PIPELINE_ID
 
 send_pipeline_stats:
   stage: notify

.gitlab/packaging/rpm.yml

@@ -111,14 +111,14 @@ agent_suse-x64-a7-fips:
   extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_x86]
   needs: ["datadog-agent-7-x64-fips"]
   variables:
-    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    OMNIBUS_EXTRA_ARGS: "--host-distribution=suse --flavor fips"
     DD_PROJECT: agent
 
 agent_suse-arm64-a7-fips:
   extends: [.package_suse_rpm_common, .package_rpm_agent_7, .package_rpm_arm64]
   needs: ["datadog-agent-7-arm64-fips"]
   variables:
-    OMNIBUS_EXTRA_ARGS: "--flavor fips"
+    OMNIBUS_EXTRA_ARGS: "--host-distribution=suse --flavor fips"
     DD_PROJECT: agent
 
 installer_rpm-arm64:

.gitlab/source_test/linux.yml

@@ -174,4 +174,4 @@ new-e2e-unit-tests:
     KUBERNETES_CPU_REQUEST: 6
     # Not using the entrypoint script for the e2e runner image
     FF_KUBERNETES_HONOR_ENTRYPOINT: false
-  timeout: 10m
+  timeout: 20m  # Not less than 20m because job startup can take time.

.gitlab/source_test/windows.yml

@@ -44,7 +44,7 @@
       -e S3_PERMANENT_ARTIFACTS_URI="${S3_PERMANENT_ARTIFACTS_URI}"
       -e COVERAGE_CACHE_FLAG="${COVERAGE_CACHE_FLAG}"
       registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
-      c:\mnt\tasks\winbuildscripts\unittests.bat
+      powershell.exe -c "c:\mnt\tasks\winbuildscripts\Invoke-UnitTests.ps1 -BuildOutOfSource 1 -CheckGoVersion 1 -InstallDeps 1 -UploadCoverage 1 -UploadTestResults 1"
     - If ($lastExitCode -ne "0") { exit "$lastExitCode" }
   variables:
     TEST_OUTPUT_FILE: test_output.json

LICENSE-3rdparty.csv

@@ -2922,6 +2922,8 @@ core,google.golang.org/grpc/balancer,Apache-2.0,Copyright 2014 gRPC authors.
 core,google.golang.org/grpc/balancer/base,Apache-2.0,Copyright 2014 gRPC authors.
 core,google.golang.org/grpc/balancer/grpclb/state,Apache-2.0,Copyright 2014 gRPC authors.
 core,google.golang.org/grpc/balancer/pickfirst,Apache-2.0,Copyright 2014 gRPC authors.
+core,google.golang.org/grpc/balancer/pickfirst/internal,Apache-2.0,Copyright 2014 gRPC authors.
+core,google.golang.org/grpc/balancer/pickfirst/pickfirstleaf,Apache-2.0,Copyright 2014 gRPC authors.
 core,google.golang.org/grpc/balancer/roundrobin,Apache-2.0,Copyright 2014 gRPC authors.
 core,google.golang.org/grpc/binarylog/grpc_binarylog_v1,Apache-2.0,Copyright 2014 gRPC authors.
 core,google.golang.org/grpc/channelz,Apache-2.0,Copyright 2014 gRPC authors.
@@ -3000,6 +3002,7 @@ core,google.golang.org/protobuf/internal/impl,BSD-3-Clause,Copyright (c) 2018 Th
 core,google.golang.org/protobuf/internal/msgfmt,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
 core,google.golang.org/protobuf/internal/order,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
 core,google.golang.org/protobuf/internal/pragma,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
+core,google.golang.org/protobuf/internal/protolazy,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
 core,google.golang.org/protobuf/internal/set,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
 core,google.golang.org/protobuf/internal/strs,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved
 core,google.golang.org/protobuf/internal/version,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved

cmd/agent/common/loader.go

@@ -16,7 +16,7 @@ import (
 	workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
 	"github.com/DataDog/datadog-agent/pkg/sbom/scanner"
 	"github.com/DataDog/datadog-agent/pkg/util/defaultpaths"
-	"github.com/DataDog/datadog-agent/pkg/util/optional"
+	"github.com/DataDog/datadog-agent/pkg/util/option"
 )
 
 // GetWorkloadmetaInit provides the InitHelper for workloadmeta so it can be injected as a Param
@@ -25,7 +25,7 @@ func GetWorkloadmetaInit() workloadmeta.InitHelper {
 	return func(ctx context.Context, wm workloadmeta.Component, cfg config.Component) error {
 		// SBOM scanner needs to be called here as initialization is required prior to the
 		// catalog getting instantiated and initialized.
-		sbomScanner, err := scanner.CreateGlobalScanner(cfg, optional.NewOption(wm))
+		sbomScanner, err := scanner.CreateGlobalScanner(cfg, option.New(wm))
 		if err != nil {
 			return fmt.Errorf("failed to create SBOM scanner: %s", err)
 		} else if sbomScanner != nil {

cmd/agent/subcommands/diagnose/command.go

@@ -35,8 +35,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/diagnose"
 	"github.com/DataDog/datadog-agent/pkg/diagnose/diagnosis"
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
-	"github.com/DataDog/datadog-agent/pkg/util/optional"
-
+	"github.com/DataDog/datadog-agent/pkg/util/option"
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 )
@@ -102,7 +101,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 					AgentType:  workloadmeta.NodeAgent,
 					InitHelper: common.GetWorkloadmetaInit(),
 				}),
-				fx.Supply(optional.NewNoneOption[collector.Component]()),
+				fx.Supply(option.None[collector.Component]()),
 				dualTaggerfx.Module(common.DualTaggerParams()),
 				autodiscoveryimpl.Module(),
 				compressionfx.Module(),
@@ -301,7 +300,7 @@ This command print the security-agent metadata payload. This payload is used by
 
 func cmdDiagnose(cliParams *cliParams,
 	senderManager diagnosesendermanager.Component,
-	wmeta optional.Option[workloadmeta.Component],
+	wmeta option.Option[workloadmeta.Component],
 	ac autodiscovery.Component,
 	secretResolver secrets.Component,
 	_ log.Component,

cmd/agent/subcommands/flare/command.go

@@ -59,7 +59,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/util/defaultpaths"
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 	"github.com/DataDog/datadog-agent/pkg/util/input"
-	"github.com/DataDog/datadog-agent/pkg/util/optional"
+	"github.com/DataDog/datadog-agent/pkg/util/option"
 )
 
 // cliParams are the command-line arguments for this subcommand
@@ -130,7 +130,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 				}),
 				localTaggerfx.Module(tagger.Params{}),
 				autodiscoveryimpl.Module(),
-				fx.Supply(optional.NewNoneOption[collector.Component]()),
+				fx.Supply(option.None[collector.Component]()),
 				compressionfx.Module(),
 				diagnosesendermanagerimpl.Module(),
 				// We need inventoryagent to fill the status page generated by the flare.
@@ -304,7 +304,7 @@ func makeFlare(flareComp flare.Component,
 	config config.Component,
 	_ sysprobeconfig.Component,
 	cliParams *cliParams,
-	_ optional.Option[workloadmeta.Component],
+	_ option.Option[workloadmeta.Component],
 	_ tagger.Component) error {
 	var (
 		profile flare.ProfileData