feat(spooler): Hash only own_key for partition routing #16538

Workflow file for this run

	name: CI

	on:
	push:
	branches:
	- master
	- release/**
	- release-library/**

	pull_request:
	types: [opened, synchronize, reopened, labeled]

	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	env:
	CARGO_TERM_COLOR: always
	RELAY_CARGO_ARGS: "--locked"

	jobs:
	lint:
	name: Lint
	runs-on: ubuntu-latest

	steps:
	- name: Install libcurl-dev
	run: \|
	sudo apt-get update
	sudo apt-get install -y libcurl4-openssl-dev

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"

	- name: Install Rust Toolchain
	run: rustup toolchain install stable --profile minimal --component clippy rustfmt rust-docs --no-self-update

	- uses: swatinem/rust-cache@v2
	with:
	key: ${{ github.job }}

	- run: make style lint

	- name: Check Docs
	run: cargo doc --workspace --all-features --no-deps --document-private-items
	env:
	RUSTDOCFLAGS: -Dwarnings

	lint_default:
	name: Lint Rust Default Features
	runs-on: ubuntu-latest

	steps:
	- name: Install libcurl-dev
	run: \|
	sudo apt-get update
	sudo apt-get install -y libcurl4-openssl-dev

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Install Rust Toolchain
	run: rustup toolchain install stable --profile minimal --component clippy --no-self-update

	- uses: swatinem/rust-cache@v2
	with:
	key: ${{ github.job }}

	- name: Run Clippy
	run: cargo clippy --workspace --all-targets --no-deps -- -D warnings

	test:
	timeout-minutes: 20
	strategy:
	fail-fast: false
	matrix:
	os: [macos-latest, windows-latest]

	name: Test (${{ matrix.os }})
	runs-on: ${{ matrix.os }}

	# Skip redundant checks for library releases
	if: "!startsWith(github.ref, 'refs/heads/release-library/')"

	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Install Rust Toolchain
	run: rustup toolchain install stable --profile minimal --no-self-update

	- uses: swatinem/rust-cache@v2
	with:
	key: ${{ github.job }}

	- name: Run Cargo Tests
	run: cargo test --workspace

	test_all:
	timeout-minutes: 15
	name: Test All Features (ubuntu-latest)
	runs-on: ubuntu-latest

	# Skip redundant checks for library releases
	if: "!startsWith(github.ref, 'refs/heads/release-library/')"

	# Testing all features requires Docker container operations that are only available on
	# `ubuntu-latest`. This `test-all` job is to be seen as complementary to the `test` job. If
	# services become available on other platforms, the jobs should be consolidated. See
	# https://docs.github.com/en/actions/guides/about-service-containers

	services:
	redis: # https://docs.github.com/en/actions/guides/creating-redis-service-containers
	image: redis
	ports:
	- 6379:6379
	redis_secondary:
	image: redis
	ports:
	- 6380:6379

	steps:
	- name: Install libcurl-dev
	run: \|
	sudo apt-get update
	sudo apt-get install -y libcurl4-openssl-dev

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Install Rust Toolchain
	run: rustup toolchain install stable --profile minimal --no-self-update

	- uses: swatinem/rust-cache@v2
	with:
	key: ${{ github.job }}

	- name: Run Cargo Tests
	run: cargo test --workspace --all-features

	test_py:
	# Skip redundant checks for binary releases
	if: "!startsWith(github.ref, 'refs/heads/release/')"

	name: Test Python
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Install Rust Toolchain
	run: rustup toolchain install stable --profile minimal --no-self-update

	- uses: actions/setup-python@v5
	with:
	python-version: "3.10"

	- name: Install Dependencies
	run: pip install -U pytest

	- uses: swatinem/rust-cache@v2
	with:
	key: ${{ github.job }}

	- name: Build and Install Library
	run: pip install -v --editable py
	env:
	RELAY_DEBUG: 1

	- name: Run Python Tests
	run: pytest -v py

	build-setup:
	name: Setup build metadata
	runs-on: ubuntu-latest

	if: "!startsWith(github.ref, 'refs/heads/release-library/')"

	env:
	FULL_CI: "${{
	github.ref == 'refs/heads/master'
	\|\| startsWith(github.ref, 'refs/heads/release/')
	\|\| contains(github.event.pull_request.labels.*.name, 'Trigger: Full-CI')
	}}"

	steps:
	- id: set-outputs
	run: \|
	echo "full_ci=$FULL_CI" >> $GITHUB_OUTPUT

	if [[ "$FULL_CI" == "true" ]]; then
	echo "Running full CI"
	echo 'image_names=["relay", "relay-pop"]' >> $GITHUB_OUTPUT
	echo 'targets=["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]' >> $GITHUB_OUTPUT
	echo 'platforms=["linux/amd64","linux/arm64"]' >> $GITHUB_OUTPUT
	else
	echo "Skipping some CI steps"
	echo 'image_names=["relay"]' >> $GITHUB_OUTPUT
	echo 'targets=["x86_64-unknown-linux-gnu"]' >> $GITHUB_OUTPUT
	echo 'platforms=["linux/amd64"]' >> $GITHUB_OUTPUT
	fi
	outputs:
	image_names: "${{ steps.set-outputs.outputs.image_names }}"
	targets: "${{ steps.set-outputs.outputs.targets }}"
	platforms: "${{ steps.set-outputs.outputs.platforms }}"
	full_ci: "${{ steps.set-outputs.outputs.full_ci }}"

	build:
	needs: build-setup
	timeout-minutes: 30

	strategy:
	matrix:
	image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }}
	target: ${{ fromJson(needs.build-setup.outputs.targets) }}

	name: Build Relay Binary
	runs-on: ubuntu-latest

	if: "!startsWith(github.ref, 'refs/heads/release-library/')"

	env:
	RELAY_BIN: "target/${{ matrix.target }}/release/relay"
	FEATURES: \|-
	${{fromJson('{
	"relay": "processing,crash-handler",
	"relay-pop": "crash-handler"
	}')[matrix.image_name] }}
	DOCKER_PLATFORM: \|-
	${{fromJson('{
	"x86_64-unknown-linux-gnu": "linux/amd64",
	"aarch64-unknown-linux-gnu": "linux/arm64"
	}')[matrix.target] }}
	# Fix editor: '

	steps:
	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y llvm

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- uses: dtolnay/rust-toolchain@stable
	- uses: swatinem/rust-cache@v2
	with:
	key: "${{ github.job }}-${{ matrix.target }}-${{ matrix.image_name }}"

	- name: Install Cross
	# We need a nightly version of cross for `cross-util`.
	run: cargo install cross --git https://github.com/cross-rs/cross --rev 085092c

	- name: Compile
	run: \|
	export PATH="/home/runner/.cargo/bin/:$PATH"

	cross build --release --locked --features "${FEATURES}" --target "${{ matrix.target }}"

	- name: Split debug info
	run: \|
	llvm-objcopy --only-keep-debug "${RELAY_BIN}"{,.debug}
	llvm-objcopy --strip-debug --strip-unneeded "${RELAY_BIN}"
	llvm-objcopy --add-gnu-debuglink "${RELAY_BIN}"{.debug,}

	cross-util run --target "${{ matrix.target }}" -- "sentry-cli difutil bundle-sources ${RELAY_BIN}.debug"
	zip "${RELAY_BIN}-debug.zip" "${RELAY_BIN}.debug"

	- name: Prepare Artifacts
	run: \|
	mkdir -p "artifacts/${DOCKER_PLATFORM}"
	cp "${RELAY_BIN}"{,-debug.zip,.src.zip} "artifacts/${DOCKER_PLATFORM}"

	- name: Upload Artifacts
	uses: actions/upload-artifact@v4
	with:
	retention-days: 1
	name: ${{ matrix.image_name }}@${{ matrix.target }}
	path: "./artifacts/*"

	build-docker:
	timeout-minutes: 5
	needs: [build-setup, build]

	name: Build Docker Image
	runs-on: ubuntu-latest

	strategy:
	matrix:
	image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }}

	env:
	PLATFORMS: "${{ join(fromJSON(needs.build-setup.outputs.platforms), ',') }}"
	DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}"
	REVISION: "${{ github.event.pull_request.head.sha \|\| github.sha }}"

	steps:
	- uses: actions/checkout@v4

	- uses: docker/setup-qemu-action@v3
	- uses: docker/setup-buildx-action@v3

	- uses: actions/download-artifact@v4
	with:
	pattern: "${{ matrix.image_name }}@*"
	merge-multiple: true

	- name: Build and push to ghcr.io
	if: "!github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'"
	run: \|
	docker login --username '${{ github.actor }}' --password '${{ secrets.GITHUB_TOKEN }}' ghcr.io

	docker buildx build \
	--platform "${PLATFORMS}" \
	--tag "${DOCKER_IMAGE}:${REVISION}" \
	$( [[ "${GITHUB_REF}" == "refs/heads/master" ]] && printf %s "--tag ${DOCKER_IMAGE}:nightly" ) \
	--file Dockerfile.release \
	--push \
	.

	- name: Build and publish docker artifact
	if: "github.event.pull_request.head.repo.fork \|\| github.actor == 'dependabot[bot]'"
	run: \|
	docker buildx build \
	--platform "${PLATFORMS}" \
	--tag "${DOCKER_IMAGE}:${REVISION}" \
	--file Dockerfile.release \
	--output type=docker,dest=${{ matrix.image_name }}-docker-image \
	.

	- name: Upload docker image
	if: "github.event.pull_request.head.repo.fork \|\| github.actor == 'dependabot[bot]'"
	uses: actions/upload-artifact@v4
	with:
	retention-days: 1
	name: ${{ matrix.image_name }}-docker-image
	path: "${{ matrix.image_name }}-docker-image"

	publish-to-dockerhub:
	needs: [build-setup, build-docker]

	runs-on: ubuntu-20.04
	name: Publish Relay to DockerHub

	strategy:
	matrix:
	image_name: [relay] # Don't publish relay-pop (for now)

	if: ${{ (github.ref_name == 'master') }}

	env:
	GHCR_DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}"
	DH_DOCKER_IMAGE: "getsentry/${{ matrix.image_name }}"
	REVISION: "${{ github.event.pull_request.head.sha \|\| github.sha }}"

	steps:
	- name: Login to DockerHub
	run: docker login --username=sentrybuilder --password ${{ secrets.DOCKER_HUB_RW_TOKEN }}

	- name: Copy Image from GHCR to DockerHub
	run: \|
	# We push 3 tags to Dockerhub:
	# 1) the full sha of the commit
	docker buildx imagetools create --tag "${DH_DOCKER_IMAGE}:${REVISION}" "${GHCR_DOCKER_IMAGE}:${REVISION}"

	# 2) the short sha
	SHORT_SHA=$(echo ${GITHUB_SHA} \| cut -c1-8)
	docker buildx imagetools create --tag "${DH_DOCKER_IMAGE}:${SHORT_SHA}" "${GHCR_DOCKER_IMAGE}:${REVISION}"

	# 3) nightly
	docker buildx imagetools create --tag "${DH_DOCKER_IMAGE}:nightly" "${GHCR_DOCKER_IMAGE}:${REVISION}"

	publish-to-gcr:
	timeout-minutes: 5
	needs: [build-setup, build-docker]

	name: Publish Relay to GCR
	runs-on: ubuntu-latest

	strategy:
	matrix:
	image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }}

	# required for google auth
	permissions:
	contents: "read"
	id-token: "write"

	env:
	GHCR_DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}"
	AR_DOCKER_IMAGE: "us-central1-docker.pkg.dev/sentryio/relay/${{ matrix.image_name }}"
	REVISION: "${{ github.event.pull_request.head.sha \|\| github.sha }}"

	# Skip redundant checks for library releases
	# Skip for dependabot and if run on a fork
	if: "!startsWith(github.ref, 'refs/heads/release-library/') && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.build-setup.outputs.full_ci == 'true'"

	steps:
	- name: Google Auth
	id: auth
	uses: google-github-actions/auth@v2
	with:
	workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool
	service_account: [email protected]

	- name: "Set up Cloud SDK"
	uses: "google-github-actions/setup-gcloud@v2"
	with:
	# https://github.com/google-github-actions/auth#authenticating-via-workload-identity-federation
	# You must use the Cloud SDK version 390.0.0 or later to authenticate the bq and gsutil tools.
	version: ">= 390.0.0"

	- name: Configure docker
	run: \|
	gcloud auth configure-docker us-central1-docker.pkg.dev

	- name: Copy Image from GHCR to AR
	run: docker buildx imagetools create --tag "${AR_DOCKER_IMAGE}:${REVISION}" "${GHCR_DOCKER_IMAGE}:${REVISION}"

	- name: Copy Nightly from GHCR to AR
	if: github.ref == 'refs/heads/master'
	run: docker buildx imagetools create --tag "${AR_DOCKER_IMAGE}:nightly" "${GHCR_DOCKER_IMAGE}:nightly"

	gocd-artifacts:
	timeout-minutes: 5
	needs: [build-setup, build-docker]

	name: Upload build artifacts to gocd
	runs-on: ubuntu-latest

	strategy:
	matrix:
	image_name: ${{ fromJson(needs.build-setup.outputs.image_names) }}

	# required for google auth
	permissions:
	contents: "read"
	id-token: "write"

	env:
	GHCR_DOCKER_IMAGE: "ghcr.io/getsentry/${{ matrix.image_name }}"
	REVISION: "${{ github.event.pull_request.head.sha \|\| github.sha }}"

	if: "!startsWith(github.ref, 'refs/heads/release-library/') && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && needs.build-setup.outputs.full_ci == 'true'"

	steps:
	- name: Google Auth
	id: auth
	uses: google-github-actions/auth@v2
	with:
	workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool
	service_account: [email protected]

	- name: "Set up Cloud SDK"
	uses: "google-github-actions/setup-gcloud@v2"
	with:
	# https://github.com/google-github-actions/auth#authenticating-via-workload-identity-federation
	# You must use the Cloud SDK version 390.0.0 or later to authenticate the bq and gsutil tools.
	version: ">= 390.0.0"

	- name: Upload gocd deployment assets
	run: \|
	set -euxo pipefail
	VERSION="$(docker run --rm "${GHCR_DOCKER_IMAGE}:${REVISION}" --version \| cut -d" " -f2)"
	echo "${{ matrix.image_name }}@${VERSION}+${REVISION}" > release-name

	docker run --rm --entrypoint cat "${GHCR_DOCKER_IMAGE}:${REVISION}" /opt/relay-debug.zip > relay-debug.zip
	docker run --rm --entrypoint cat "${GHCR_DOCKER_IMAGE}:${REVISION}" /opt/relay.src.zip > relay.src.zip
	docker run --rm --entrypoint tar "${GHCR_DOCKER_IMAGE}:${REVISION}" -cf - /lib/x86_64-linux-gnu > libs.tar

	# debugging for mysterious "Couldn't write tracker file" issue:
	(env \| grep runner) \|\| true
	ls -ld \
	/home \
	/home/runner \
	/home/runner/.gsutil \
	/home/runner/.gsutil/tracker-files \
	/home/runner/.gsutil/tracker-files/upload_TRACKER_*.rc.zip__JSON.url \
	\|\| true
	gsutil -m cp -L gsutil.log ./libs.tar ./relay-debug.zip ./relay.src.zip ./release-name \
	"gs://dicd-team-devinfra-cd--relay/deployment-assets/${REVISION}/${{ matrix.image_name }}/" \|\| status=$? && status=$?
	cat gsutil.log
	exit "$status"

	test_integration:
	name: Integration Tests
	runs-on: ubuntu-latest
	timeout-minutes: 30

	# Skip redundant checks for library releases
	if: "!startsWith(github.ref, 'refs/heads/release-library/')"

	services:
	redis: # https://docs.github.com/en/actions/guides/creating-redis-service-containers
	image: redis
	ports:
	- 6379:6379
	redis_secondary:
	image: redis
	ports:
	- 6380:6379

	# Kafka + Zookeeper version synced with
	# https://github.com/getsentry/sentry/blob/363509c242aff197409207ce4990fb061f3534a3/.github/actions/setup-sentry/action.yml#L174

	zookeeper:
	image: confluentinc/cp-zookeeper:4.1.0
	env:
	ZOOKEEPER_CLIENT_PORT: 2181

	kafka:
	image: confluentinc/cp-kafka:5.1.2
	env:
	KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
	KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
	KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
	KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1
	ports:
	- 9092:9092

	steps:
	- name: Install libcurl-dev
	run: \|
	sudo apt-get update
	sudo apt-get install -y libcurl4-openssl-dev

	- uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Install Rust Toolchain
	run: rustup toolchain install stable --profile minimal --no-self-update

	- uses: swatinem/rust-cache@v2
	with:
	key: ${{ github.job }}

	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"

	- run: make test-integration
	env:
	PYTEST_N: 6
	RELAY_VERSION_CHAIN: "23.12.0,latest"

	sentry-relay-integration-tests:
	name: Sentry-Relay Integration Tests
	runs-on: ubuntu-latest
	timeout-minutes: 30
	needs: build-docker

	# Skip redundant checks for library releases
	if: "!startsWith(github.ref, 'refs/heads/release-library/')"

	steps:
	# Checkout Sentry and run integration tests against latest Relay
	- name: Checkout Sentry
	uses: actions/checkout@v4
	with:
	repository: getsentry/sentry
	path: sentry

	- name: Setup steps
	id: setup
	run: \|
	# GITHUB_SHA in pull requests points to the merge commit
	RELAY_TEST_IMAGE=ghcr.io/getsentry/relay:${{ github.event.pull_request.head.sha \|\| github.sha }}
	echo "We expected GCB to push this image $RELAY_TEST_IMAGE"
	echo "relay-test-image=$RELAY_TEST_IMAGE" >> "$GITHUB_OUTPUT"
	# We cannot execute actions that are not placed under .github of the main repo
	mkdir -p .github/actions
	cp -r sentry/.github/actions/setup-sentry .github/actions/

	- name: Setup Sentry
	uses: ./.github/actions/setup-sentry
	with:
	workdir: sentry
	snuba: true
	kafka: true
	symbolicator: true

	- name: Download Docker Image
	if: "github.event.pull_request.head.repo.fork \|\| github.actor == 'dependabot[bot]'"
	uses: actions/download-artifact@v4
	with:
	name: relay-docker-image

	- name: Import Docker Image
	if: "github.event.pull_request.head.repo.fork \|\| github.actor == 'dependabot[bot]'"
	run: docker load -i relay-docker-image

	- name: Run Sentry integration tests
	working-directory: sentry
	env:
	RELAY_TEST_IMAGE: ${{ steps.setup.outputs.relay-test-image }}
	run: \|
	echo "Testing against ${RELAY_TEST_IMAGE}"
	make test-relay-integration

	self-hosted-end-to-end:
	runs-on: ubuntu-latest
	timeout-minutes: 25
	needs: build-docker

	# - Skip redundant checks for library releases
	# - Skip for dependabot or if it's a fork as the image cannot be uploaded to ghcr since this test attempts to pull
	# the image from ghcr
	if: "!startsWith(github.ref, 'refs/heads/release-library/') && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'"

	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Run Sentry self-hosted e2e CI
	uses: getsentry/action-self-hosted-e2e-tests@main
	with:
	project_name: relay
	image_url: ghcr.io/getsentry/relay:${{ github.event.pull_request.head.sha \|\| github.sha }}
	docker_repo: getsentry/relay
	docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(spooler): Hash only own_key for partition routing #16538

Workflow file

feat(spooler): Hash only own_key for partition routing #16538

Jobs

Run details

Workflow file for this run