Merge branch 'main' into elasticsearch-fix-dashboard-formulas

.buildkite/hooks/pre-command

@@ -73,7 +73,7 @@ if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
 
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
+if [[ ( "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ) &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
     # Get the commit from target branch in the first step (reference-target-branch).
     # This step MUST be the first one and not run in parallel with any other step to ensure
     # that there is just one value for this variable
@@ -91,7 +91,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
     fi
 fi
 
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
+if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN

.buildkite/hooks/pre-exit

@@ -4,7 +4,7 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
     # FIXME: update condition depending on the pipeline steps triggered
     if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY

.buildkite/pipeline.schedule-daily.yml

@@ -26,38 +26,38 @@ steps:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.18"
+  - label: "Check integrations local stacks - Stack Version v8.19"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.18.0-SNAPSHOT
+        STACK_VERSION: 8.19.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.18 - LogsDB"
+  - label: "Check integrations local stacks - Stack Version v8.19 - LogsDB"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.18.0-SNAPSHOT
+        STACK_VERSION: 8.19.0-SNAPSHOT
         STACK_LOGSDB_ENABLED: "true"
         PUBLISH_COVERAGE_REPORTS: "false"
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v9.0"
+  - label: "Check integrations local stacks - Stack Version v9.1"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 9.0.0-SNAPSHOT
+        STACK_VERSION: 9.1.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "false"
     depends_on:
       - step: "check"

.buildkite/pipeline.schedule-weekly.yml

@@ -15,26 +15,26 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v8.18"
+  - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v8.19"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.18.0-SNAPSHOT
+        STACK_VERSION: 8.19.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "false"
         ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.0"
+  - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.1"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 9.0.0-SNAPSHOT
+        STACK_VERSION: 9.1.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "false"
         ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
     depends_on:

.buildkite/pipeline.yml

@@ -70,6 +70,9 @@ steps:
       image: "${LINUX_AGENT_IMAGE}"
       cpu: "8"
       memory: "4G"
+    if: |
+      build.env('BUILDKITE_PULL_REQUEST') != "false" &&
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations"
 
   - label: ":sonarqube: Continuous Code Inspection"
     soft_fail: true # FIXME: Coverage is failing, remove this after solving the issue
@@ -83,6 +86,8 @@ steps:
     command: ".buildkite/scripts/run_sonar_scanner.sh"
     artifact_paths:
       - build/test-coverage/coverage_merged.xml
+    if: |
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations"
 
   - label: ":junit: Junit annotate"
     plugins:
@@ -108,4 +113,5 @@ steps:
     # run this step when if it is triggered by the daily job
     if: |
       build.source == "trigger_job" &&
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" &&
       build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

.buildkite/pull-requests.json

@@ -13,7 +13,23 @@
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$|^/test benchmark fullreport$",
       "skip_ci_labels": [],
       "skip_target_branches": [],
-      "skip_ci_on_only_changed": ["^.github/", "^docs/"],
+      "skip_ci_on_only_changed": ["^.github/workflows/", "^.github/dependabot.yml", "^.github/ISSUE_TEMPLATE/", "^docs/"],
+      "always_require_ci_on_changed": []
+    },
+    {
+      "enabled": true,
+      "pipelineSlug": "integrations-test-stack",
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": [],
+      "set_commit_status": true,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^/test stack (7|8|9)\\.\\d+\\.\\d+(-SNAPSHOT)?$",
+      "always_trigger_comment_regex": "^/test stack (7|8|9)\\.\\d+\\.\\d+(-SNAPSHOT)?$",
+      "skip_ci_labels": [],
+      "skip_target_branches": [],
+      "skip_ci_on_only_changed": [],
       "always_require_ci_on_changed": []
     },
     {

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -29,6 +29,16 @@ to="$(get_to_changeset)"
 
 echo "[DEBUG] Checking with commits: from: '${from}' to: '${to}'"
 
+# This variable does not exist in builds triggered automatically
+GITHUB_PR_TRIGGER_COMMENT="${GITHUB_PR_TRIGGER_COMMENT:-""}"
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-test-stack" && "${GITHUB_PR_TRIGGER_COMMENT}" =~ ^/test\ stack ]]; then
+    echo "--- Stack version set from Github comment"
+    STACK_VERSION=$(echo "$GITHUB_PR_TRIGGER_COMMENT" | cut -d " " -f 3)
+    export STACK_VERSION
+    echo "Use Elastic stack version from Github comment: ${STACK_VERSION}"
+fi
+
 packages_to_test=0
 
 for package in ${PACKAGE_LIST}; do

.github/CODEOWNERS

@@ -10,6 +10,7 @@
 /packages/1password @elastic/security-service-integrations
 /packages/abnormal_security @elastic/security-service-integrations
 /packages/activemq @elastic/obs-infraobs-integrations
+/packages/admin_by_request_epm @elastic/security-service-integrations
 /packages/airflow @elastic/obs-infraobs-integrations
 /packages/akamai @elastic/security-service-integrations
 /packages/amazon_security_lake @elastic/security-service-integrations
@@ -76,6 +77,7 @@
 /packages/aws_bedrock/data_stream/runtime @elastic/obs-infraobs-integrations
 /packages/aws_bedrock/data_stream/guardrails @elastic/obs-infraobs-integrations
 /packages/aws_logs @elastic/obs-ds-hosted-services
+/packages/aws_mq @elastic/obs-infraobs-integrations
 /packages/awsfargate @elastic/obs-infraobs-integrations
 /packages/awsfirehose @elastic/obs-ds-hosted-services
 /packages/azure @elastic/obs-infraobs-integrations @elastic/obs-ds-hosted-services @elastic/security-service-integrations
@@ -121,6 +123,7 @@
 /packages/bbot @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml
 /packages/beat @elastic/stack-monitoring
+/packages/beyondinsight_password_safe @elastic/security-service-integrations
 /packages/bitdefender @elastic/security-service-integrations
 /packages/bitwarden @elastic/security-service-integrations
 /packages/blacklens @elastic/security-service-integrations
@@ -163,7 +166,7 @@
 /packages/cloudflare @elastic/security-service-integrations
 /packages/cloudflare_logpush @elastic/security-service-integrations
 /packages/cockroachdb @elastic/obs-infraobs-integrations
-/packages/containerd @elastic/obs-cloudnative-monitoring
+/packages/containerd @elastic/obs-ds-hosted-services
 /packages/coredns @elastic/obs-infraobs-integrations
 /packages/corelight @elastic/security-service-integrations
 /packages/couchbase @elastic/obs-infraobs-integrations
@@ -179,7 +182,7 @@
 /packages/ded @elastic/ml-ui @elastic/sec-applied-ml
 /packages/dga @elastic/ml-ui @elastic/sec-applied-ml
 /packages/digital_guardian @elastic/security-service-integrations
-/packages/docker @elastic/obs-cloudnative-monitoring
+/packages/docker @elastic/obs-ds-hosted-services
 /packages/elastic_agent @elastic/elastic-agent
 /packages/elastic_connectors @elastic/search-extract-and-transform
 /packages/elastic_package_registry @elastic/ecosystem
@@ -253,7 +256,7 @@
 /packages/infoblox_bloxone_ddi @elastic/security-service-integrations
 /packages/infoblox_nios @elastic/security-service-integrations
 /packages/iptables @elastic/sec-deployment-and-devices
-/packages/istio @elastic/obs-cloudnative-monitoring
+/packages/istio @elastic/obs-ds-hosted-services
 /packages/jamf_compliance_reporter @elastic/security-service-integrations
 /packages/jamf_pro @elastic/security-service-integrations
 /packages/jamf_protect @elastic/security-service-integrations
@@ -267,14 +270,14 @@
 /packages/kafka_log @elastic/obs-infraobs-integrations
 /packages/keycloak @elastic/security-service-integrations
 /packages/kibana @elastic/stack-monitoring
-/packages/kubernetes @elastic/obs-cloudnative-monitoring
-/packages/kubernetes/kibana @elastic/obs-cloudnative-monitoring
-/packages/kubernetes_otel @elastic/obs-cloudnative-monitoring
+/packages/kubernetes @elastic/obs-ds-hosted-services
+/packages/kubernetes/kibana @elastic/obs-ds-hosted-services
+/packages/kubernetes_otel @elastic/obs-ds-hosted-services
 /packages/lastpass @elastic/security-service-integrations
 /packages/linux @elastic/elastic-agent-data-plane
 /packages/lmd @elastic/ml-ui @elastic/sec-applied-ml
 /packages/log @elastic/elastic-agent-data-plane
-/packages/logstash @elastic/stack-monitoring
+/packages/logstash @elastic/logstash
 /packages/lumos @elastic/security-service-integrations
 /packages/lyve_cloud @elastic/security-service-integrations
 /packages/m365_defender @elastic/security-service-integrations
@@ -302,10 +305,11 @@
 /packages/netskope @elastic/security-service-integrations
 /packages/network_traffic @elastic/sec-linux-platform
 /packages/nginx @elastic/obs-infraobs-integrations
-/packages/nginx_ingress_controller @elastic/obs-cloudnative-monitoring
+/packages/nginx_ingress_controller @elastic/obs-ds-hosted-services
 /packages/nginx_ingress_controller_otel @elastic/obs-infraobs-integrations
 /packages/o365 @elastic/security-service-integrations
 /packages/okta @elastic/security-service-integrations
+/packages/openai @elastic/obs-infraobs-integrations
 /packages/opencanary @elastic/security-service-integrations
 /packages/oracle @elastic/obs-infraobs-integrations
 /packages/oracle_weblogic @elastic/obs-infraobs-integrations
@@ -324,7 +328,7 @@
 /packages/prisma_cloud @elastic/security-service-integrations
 /packages/problemchild @elastic/ml-ui @elastic/sec-applied-ml
 /packages/prometheus @elastic/obs-infraobs-integrations
-/packages/prometheus/data_stream/remote_write @elastic/obs-cloudnative-monitoring
+/packages/prometheus/data_stream/remote_write @elastic/obs-ds-hosted-services
 /packages/prometheus/data_stream/collector @elastic/obs-infraobs-integrations
 /packages/prometheus/data_stream/query @elastic/obs-infraobs-integrations
 /packages/prometheus_input @elastic/obs-infraobs-integrations
@@ -340,6 +344,7 @@
 /packages/redis @elastic/obs-infraobs-integrations
 /packages/redisenterprise @elastic/obs-infraobs-integrations
 /packages/rubrik @elastic/obs-infraobs-integrations
+/packages/sailpoint_identity_sc @elastic/security-service-integrations
 /packages/salesforce @elastic/obs-infraobs-integrations
 /packages/santa @elastic/security-service-integrations
 /packages/security_detection_engine @elastic/protections
@@ -403,6 +408,7 @@
 /packages/ti_cif3 @elastic/security-service-integrations
 /packages/ti_crowdstrike @elastic/security-service-integrations
 /packages/ti_cybersixgill @elastic/security-service-integrations
+/packages/ti_domaintools @elastic/security-service-integrations
 /packages/ti_eclecticiq @elastic/security-service-integrations
 /packages/ti_eset @elastic/security-service-integrations
 /packages/ti_maltiverse @elastic/security-service-integrations
@@ -459,17 +465,25 @@
 /packages/cisco_meraki_metrics @elastic/obs-infraobs-integrations
 /packages/panw_metrics @elastic/obs-infraobs-integrations
 /packages/o365_metrics @elastic/obs-infraobs-integrations @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/active_users @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/active_users_services_user_counts @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/groups_activity_group_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/mailbox_usage_detail @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/mailbox_usage_quota_status @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/onedrive_usage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/onedrive_usage_account_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/onedrive_usage_file_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/onedrive_usage_storage @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/onedrive_usage_account_detail @elastic/security-service-integrations
 /packages/o365_metrics/data_stream/outlook_activity @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/outlook_app_usage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/sharepoint_site_usage_storage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/sharepoint_site_usage_detail @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/mailbox_usage_detail @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/mailbox_usage_quota_status @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/outlook_app_usage_version_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/sharepoint_site_usage @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/teams_device_usage_user_counts @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/teams_user_activity_user_counts @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/teams_user_activity_user_detail @elastic/security-service-integrations
 /packages/o365_metrics/data_stream/viva_engage_groups_activity_group_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/viva_engage_device_usage_user_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/yammer_device_usage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/service_health @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/viva_engage_device_usage_user_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/subscriptions @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/teams_call_quality @elastic/obs-infraobs-integrations

.github/ISSUE_TEMPLATE/integration_bug.yml

@@ -16,6 +16,7 @@ body:
         - 1Password [1password]
         - Abnormal Security [abnormal_security]
         - ActiveMQ [activemq]
+        - Admin By Request EPM [admin_by_request_epm]
         - Airflow [airflow]
         - Akamai [akamai]
         - Amazon Security Lake [amazon_security_lake]
@@ -34,6 +35,7 @@ body:
         - AWS [aws]
         - Amazon Bedrock [aws_bedrock]
         - Custom AWS Logs [aws_logs]
+        - Amazon MQ [aws_mq]
         - AWS Fargate (for ECS clusters) [awsfargate]
         - Amazon Data Firehose [awsfirehose]
         - Azure Logs [azure]
@@ -53,6 +55,7 @@ body:
         - BBOT (Bighuge BLS OSINT Tool) [bbot]
         - Network Beaconing Identification [beaconing]
         - Beat [beat]
+        - BeyondInsight and Password Safe [beyondinsight_password_safe]
         - BitDefender [bitdefender]
         - Bitwarden [bitwarden]
         - blacklens.io [blacklens]
@@ -214,6 +217,7 @@ body:
         - Microsoft Office 365 [o365]
         - Microsoft Office 365 Metrics [o365_metrics]
         - Okta [okta]
+        - OpenAI [openai]
         - OpenCanary [opencanary]
         - Oracle [oracle]
         - Oracle WebLogic [oracle_weblogic]
@@ -246,6 +250,7 @@ body:
         - Redis [redis]
         - Redis Enterprise [redisenterprise]
         - Rubrik RSC Metrics [rubrik]
+        - Sailpoint Identity Security Cloud [sailpoint_identity_sc]
         - Salesforce [salesforce]
         - Google Santa [santa]
         - Prebuilt Security Detection Rules [security_detection_engine]
@@ -291,6 +296,7 @@ body:
         - CrowdStrike Falcon Intelligence [ti_crowdstrike]
         - Custom Threat Intelligence [ti_custom]
         - Cybersixgill [ti_cybersixgill]
+        - DomainTools Real Time Unified Feeds [ti_domaintools]
         - EclecticIQ [ti_eclecticiq]
         - ESET Threat Intelligence [ti_eset]
         - Maltiverse [ti_maltiverse]