Merge branch 'main' into tbs-pebble-rebase-revert

.github/workflows/prepare-release/action.yml

@@ -49,6 +49,14 @@ runs:
         VERSION: ${{ inputs.version }}
       shell: 'bash'
 
+    - name: validate version format (major only)
+      if: ${{ inputs.type == 'major' && ! endsWith(inputs.version, '.0.0') }}
+      run: |-
+        FAILURE_MESSAGE='version is not a major one but a patch (only support for <major>.0.0)'
+        echo "FAILURE_MESSAGE=${FAILURE_MESSAGE}" >> "$GITHUB_ENV"
+        echo "::error::${FAILURE_MESSAGE}" ; exit 1
+      shell: 'bash'
+
     - name: validate version format (minor only)
       if: ${{ inputs.type == 'minor' && ! endsWith(inputs.version, '0') }}
       run: |-
@@ -65,6 +73,18 @@ runs:
         echo "::error::${FAILURE_MESSAGE}" ; exit 1
       shell: 'bash'
 
+    - name: validate if branch already exists (major only)
+      if: ${{ inputs.type == 'major' }}
+      run: |-
+        if git ls-remote --exit-code --heads "https://github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" > /dev/null ; then
+          echo "FAILURE_MESSAGE=${FAILURE_MESSAGE}" >> "$GITHUB_ENV"
+          echo "::error::${FAILURE_MESSAGE}" ; exit 1
+        fi
+      shell: 'bash'
+      env:
+        BRANCH: ${{ steps.generate.outputs.release-branch }}
+        FAILURE_MESSAGE: Branch already exists. This is not a major release.
+
     - name: validate if branch already exists (minor only)
       if: ${{ inputs.type == 'minor' }}
       run: |-

.github/workflows/run-major-release.yml

@@ -0,0 +1,114 @@
+---
+name: run-major-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The new version to be set, normally the version in the main branch (semver format: major.0.0)'
+        required: true
+        type: string
+
+# Avoid concurrency so we can watch the releases correctly
+concurrency:
+  group: ${{ github.workflow }}
+
+permissions:
+  contents: read
+
+env:
+  JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  SLACK_CHANNEL: "#apm-server"
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      release-branch: ${{ steps.prepare.outputs.release-branch }}
+      release-type: ${{ steps.prepare.outputs.release-type }}
+      release-version: ${{ steps.prepare.outputs.release-version }}
+      slack-thread: ${{ steps.prepare.outputs.slack-thread }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: prepare
+        uses: ./.github/workflows/prepare-release
+        with:
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          version: ${{ inputs.version }}
+          type: 'major'
+
+  run-major:
+    runs-on: ubuntu-latest
+    needs: [ prepare ]
+    env:
+      RELEASE_BRANCH: ${{ needs.prepare.outputs.release-branch }}
+      RELEASE_TYPE: ${{ needs.prepare.outputs.release-type }}
+      RELEASE_VERSION: ${{ needs.prepare.outputs.release-version }}
+    permissions:
+      contents: write
+    steps:
+
+      - uses: elastic/oblt-actions/slack/send@v1
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: ${{ env.SLACK_CHANNEL }}
+          message: |-
+            Feature freeze for `${{ github.repository }}@${{ env.RELEASE_VERSION }}` just started.
+            The `${{ github.repository }}@${{ env.RELEASE_BRANCH }}` branch will be created Today.
+          thread-timestamp: ${{ needs.prepare.outputs.slack-thread || '' }}
+
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "contents": "write",
+              "pull_requests": "write"
+            }
+
+      - uses: actions/checkout@v4
+        with:
+          # 0 indicates all history for all branches and tags.
+          fetch-depth: 0
+
+      # Required to use a service account, otherwise PRs created by
+      # GitHub bot won't trigger any CI builds.
+      # See https://github.com/peter-evans/create-pull-request/issues/48#issuecomment-537478081
+      - name: Configure git user
+        uses: elastic/oblt-actions/git/setup@v1
+        with:
+          github-token: ${{ steps.get_token.outputs.token }}
+
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@cb9bde2e2525e640591a934b1fd28eef1dcaf5e5  # v6.2.0
+        with:
+          gpg_private_key: ${{ secrets.APM_SERVER_RELEASE_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.APM_SERVER_RELEASE_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
+      - run: make major-release
+        env:
+          GH_TOKEN: ${{ steps.get_token.outputs.token }}
+
+      - if: success()
+        uses: elastic/oblt-actions/slack/send@v1
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: ${{ env.SLACK_CHANNEL }}
+          message: |-
+            `${{ github.repository }}@${{ env.RELEASE_BRANCH }}` is now available.
+            The docs and other references are updated. You can start using it.
+          thread-timestamp: ${{ needs.prepare.outputs.slack-thread || '' }}
+
+      - if: failure()
+        uses: elastic/oblt-actions/slack/send@v1
+        with:
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: ${{ env.SLACK_CHANNEL }}
+          message: |-
+            :fire: Something went wrong with the release. See <${{ env.JOB_URL }}|logs>.
+          thread-timestamp: ${{ needs.prepare.outputs.slack-thread || '' }}

dev_docs/RELEASES.md

@@ -28,6 +28,8 @@ For patch releases, only the version on the existing major and minor version bra
     * The PR should be merged the day before release.
   * For **minor releases**: run the [`run-minor-release`](https://github.com/elastic/apm-server/actions/workflows/run-minor-release.yml) workflow (In "Use workflow from", select `main` branch. Then in "The version", specify the minor release version the release is for).
     This workflow will: create a new release branch using the stack version (X.Y); update the changelog for the release branch and open a PR targeting the release branch titled `<major>.<minor>: update docs`; create a PR on `main` titled `<major>.<minor>: update docs, mergify, versions and changelogs`. Before merging them compare commits between latest minor and the new minor versions and ensure all relevant PRs have been included in the Changelog. If not, amend it in both PRs. Request and wait a PR review from the team before merging. After it's merged add your PR to the documentation release issue in the [`elastic/dev`](https://github.com/elastic/dev/issues?q=is%3Aissue%20state%3Aopen%20label%3Adocs) repo ([Sample Issue](https://github.com/elastic/dev/issues/2895)).
+  * For **major releases**: run the [`run-major-release`](https://github.com/elastic/apm-server/actions/workflows/run-major-release.yml) workflow (In "Use workflow from", select `main` branch. Then in "The version", specify the major release version the release is for).
+    This workflow will: create a new release branch using the stack version (X.Y); update the changelog for the release branch and open a PR targeting the release branch titled `<major>.<minor>: update docs`; create a PR on `main` titled `<major>.0: update docs, mergify, versions and changelogs`. Before merging them compare commits between latest minor and the new major versions and ensure all relevant PRs have been included in the Changelog. If not, amend it in both PRs. Request and wait a PR review from the team before merging. After it's merged add your PR to the documentation release issue in the [`elastic/dev`](https://github.com/elastic/dev/issues?q=is%3Aissue%20state%3Aopen%20label%3Adocs) repo ([Sample Issue](https://github.com/elastic/dev/issues/2895)).
 * The Release Manager will ping the team to align the release process
 
 * Update dependencies

docker-compose.yml

@@ -11,7 +11,7 @@ x-logging: &default-logging
 services:
   elasticsearch:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-310b7cb4-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-1870cc1a-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -43,7 +43,7 @@ services:
 
   kibana:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/kibana/kibana:9.0.0-310b7cb4-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:9.0.0-1870cc1a-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -63,7 +63,7 @@ services:
 
   metricbeat:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/beats/metricbeat:9.0.0-310b7cb4-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:9.0.0-1870cc1a-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

release.mk

@@ -58,13 +58,20 @@ ifeq ($(RELEASE_TYPE),minor)
 # NOTE: as long as 8.x is the branch to run releases, then we use base branch 8.x
 # when 8.x is not available then we use the main base branch.
 	BASE_BRANCH ?= 8.x
+	UPDATE_MERGIFY = true
 endif
 
 ifeq ($(RELEASE_TYPE),patch)
 	BASE_BRANCH ?= $(RELEASE_BRANCH)
 	LATEST_RELEASE ?= $(RELEASE_BRANCH).$(shell expr $(PROJECT_PATCH_VERSION) - 1)
 endif
 
+ifeq ($(RELEASE_TYPE),major)
+	BASE_BRANCH ?= main
+	CHANGELOG_BRANCH = main
+	UPDATE_MERGIFY = true
+endif
+
 #######################
 ## Templates
 #######################
@@ -135,8 +142,10 @@ minor-release:
 	$(MAKE) create-commit COMMIT_MESSAGE="docs: Update changelogs for $(RELEASE_BRANCH) release"
 
 # NOTE: as long as 8.x is the branch to run releases, then we update mergify
-# when 8.x is not available then this conditional and the update-mergify should be removed..
-ifeq ($(BASE_BRANCH),8.x)
+# when 8.x is not available then this conditional and the update-mergify should be removed.
+# We use a specific PR for mergify, it used to be part of the `update-` PR but it was separated.
+# to support 8.x and main releases.
+ifeq ($(UPDATE_MERGIFY),true)
 	@echo "INFO: Create feature branch for mergify changes. Target branch $(RELEASE_BRANCH)"
 	$(MAKE) create-branch NAME=mergify-$(RELEASE_BRANCH) BASE=main
 	$(MAKE) update-mergify VERSION=$(RELEASE_BRANCH)
@@ -146,26 +155,36 @@ endif
 	@echo "INFO: Create feature branch and update the versions. Target branch $(BASE_BRANCH)"
 	$(MAKE) create-branch NAME=update-$(RELEASE_VERSION) BASE=$(BASE_BRANCH)
 # NOTE: as long as main is the branch to run releases, then we update mergify
-# when 8.x is not available then this conditional should be removed and the update-mergify should be kept.
-ifeq ($(BASE_BRANCH),main)
-	$(MAKE) update-mergify VERSION=$(RELEASE_BRANCH)
-endif
+# TODO: when 8.x is not available then this conditional should be removed and the update-mergify should be kept.
+#ifeq ($(BASE_BRANCH),main)
+#	$(MAKE) update-mergify VERSION=$(RELEASE_BRANCH)
+#endif
 	$(MAKE) update-version VERSION=$(NEXT_PROJECT_MINOR_VERSION)
 	$(MAKE) create-commit COMMIT_MESSAGE="[Release] update version $(NEXT_PROJECT_MINOR_VERSION)"
 	$(MAKE) rename-changelog VERSION=$(RELEASE_BRANCH)
 	$(MAKE) create-commit COMMIT_MESSAGE="[Release] update changelogs for $(RELEASE_BRANCH) release"
 
 	@echo "INFO: Push changes to $(PROJECT_OWNER)/apm-server and create the relevant Pull Requests"
 	git push origin $(RELEASE_BRANCH)
-# NOTE: as long as 8.x is the branch to run releases, then we update mergify
-# when 8.x is not available then this conditional and its content should be removed.
-ifeq ($(BASE_BRANCH),8.x)
+ifeq ($(UPDATE_MERGIFY),true)
 	$(MAKE) create-pull-request BRANCH=mergify-$(RELEASE_BRANCH) TARGET_BRANCH=main TITLE="$(RELEASE_BRANCH): mergify" BODY="Merge as soon as the GitHub checks are green." BACKPORT_LABEL=backport-skip
 endif
 	$(MAKE) create-pull-request BRANCH=update-$(RELEASE_VERSION) TARGET_BRANCH=$(BASE_BRANCH) TITLE="$(RELEASE_BRANCH): update docs, versions and changelogs" BODY="Merge as soon as the GitHub checks are green" BACKPORT_LABEL=backport-skip
 # NOTE: as long as 8.x is the branch to run releases, then we use main as target with the backport label.
 # when 8.x is not available then we use TARGET_BRANCH=$(RELEASE_BRANCH)
+ifeq ($(BASE_BRANCH),8.x)
+	@echo "INFO: As long as 8.x is supported, we need to create a PR also in main"
 	$(MAKE) create-pull-request BRANCH=changelog-$(RELEASE_BRANCH) TARGET_BRANCH=main TITLE="$(RELEASE_BRANCH): update docs" BODY="Merge as soon as $(TARGET_BRANCH) branch is created and the GitHub checks are green. And the PR in main for the Mergify changes has been merged." BACKPORT_LABEL=backport-$(RELEASE_BRANCH)
+endif
+
+# This is the contract with the GitHub action .github/workflows/run-major-release.yml.
+# The GitHub action will provide the below environment variables:
+#  - RELEASE_VERSION
+#
+.PHONY: major-release
+major-release:
+# NOTE: major release uses minor-release with BASE_BRANCH=main and CHANGELOG_BRANCH=main
+	$(MAKE) minor-release
 
 # This is the contract with the GitHub action .github/workflows/run-patch-release.yml
 # The GitHub action will provide the below environment variables:

testing/benchmark/main.tf

@@ -156,7 +156,7 @@ module "standalone_apm_server" {
   aws_os              = "amzn2-ami-hvm-*-x86_64-ebs"
   apm_instance_type   = var.standalone_apm_server_instance_size
   apm_volume_type     = var.standalone_apm_server_volume_type
-  apm_volume_size     = var.standalone_apm_server_volume_size
+  apm_volume_size     = var.apm_server_tail_sampling ? coalesce(var.standalone_apm_server_volume_size, 60) : var.standalone_apm_server_volume_size
   apm_server_bin_path = var.apm_server_bin_path
   ea_managed          = false
 

testing/infra/k8s/base/stack/apm-server.yaml

@@ -4,7 +4,7 @@ metadata:
   name: apm-server
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-310b7cb4-SNAPSHOT
+  version: 9.0.0-1870cc1a-SNAPSHOT
   count: 1
   http:
     tls:

testing/infra/k8s/base/stack/elasticsearch.yaml

@@ -4,7 +4,7 @@ metadata:
   name: elasticsearch
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-310b7cb4-SNAPSHOT
+  version: 9.0.0-1870cc1a-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin

testing/infra/k8s/base/stack/kibana.yaml

@@ -4,7 +4,7 @@ metadata:
   name: kibana
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-310b7cb4-SNAPSHOT
+  version: 9.0.0-1870cc1a-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch