feat(s2n-quic-core): add aggregate metrics support #4980

Workflow file for this run

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main

	name: qns

	env:
	CARGO_INCREMENTAL: 0
	CARGO_NET_RETRY: 10
	RUSTUP_MAX_RETRIES: 10
	RUST_BACKTRACE: 1
	# This kept breaking builds so we're pinning for now. We should do our best to keep
	# up with the changes, though.
	INTEROP_RUNNER_REF: 4be6491794a08899f295dc5cdf9eeba8e9fa5431
	# This should be updated when updating wesleyrosenblum/quic-network-simulator
	NETWORK_SIMULATOR_REF: sha256:20abe0bed8c0e39e1d8750507b24295f7c978bdd7e05fa6f3a5afed4b76dc191
	IPERF_ENDPOINT_REF: sha256:cb50cc8019d45d9cad5faecbe46a3c21dd5e871949819a5175423755a9045106
	WIRESHARK_VERSION: 3.7.1
	CDN: https://dnglbrstg7yg.cloudfront.net
	LOG_URL: logs/latest/SERVER_CLIENT/TEST/index.html

	# By default depandabot only receives read permissions. Explicitly give it write
	# permissions which is needed by the ouzi-dev/commit-status-updater task.
	#
	# Updating status is relatively safe (doesnt modify source code) and caution
	# should we taken before adding more permissions.
	permissions:
	statuses: write
	id-token: write # This is required for requesting the JWT/OIDC
	contents: read # This is required for actions/checkout

	jobs:
	env:
	runs-on: ubuntu-22.04
	outputs:
	matrix: ${{ steps.implementations.outputs.matrix }}
	steps:
	- uses: actions/checkout@v4
	with:
	path: s2n-quic

	- uses: actions/checkout@v4
	with:
	repository: marten-seemann/quic-interop-runner
	ref: ${{ env.INTEROP_RUNNER_REF }}
	path: quic-interop-runner

	- name: Patch quic-interop-runner
	working-directory: quic-interop-runner
	run: \|
	git apply --3way ../s2n-quic/.github/interop/runner.patch

	- name: Define implementations
	id: implementations
	working-directory: quic-interop-runner
	run: \|
	CLIENTS=$(cat implementations.json \
	\| jq -c '[. \| to_entries[] \| select(.value.role == "both" or .value.role == "client") \| {"client": .key, "server": "s2n-quic"}] \| sort'
	)
	echo "Clients: $CLIENTS"
	SERVERS=$(cat implementations.json \
	\| jq -c '[. \| to_entries[] \| select(.value.role == "both" or .value.role == "server") \| {"client": "s2n-quic", "server": .key}] \| sort'
	)
	echo "Servers: $SERVERS"
	MATRIX=$(echo "[$CLIENTS, $SERVERS]" \| jq -c '{"include": . \| flatten \| sort \| unique}')
	echo "Matrix: $MATRIX"
	echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

	s2n-quic-qns:
	runs-on: ubuntu-22.04
	strategy:
	matrix:
	mode: ["debug", "release"]
	# enable debug information
	env:
	RUSTFLAGS: "-g --cfg s2n_internal_dev --cfg s2n_quic_dump_on_panic --cfg s2n_quic_unstable"
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true

	- name: Install rust toolchain
	id: toolchain
	run: \|
	rustup toolchain install stable --profile minimal
	rustup override set stable

	- uses: camshaft/rust-cache@v1
	with:
	key: ${{ matrix.mode }}-${{ env.RUSTFLAGS }}

	- name: Run cargo build
	run: cargo build --bin s2n-quic-qns ${{ matrix.mode == 'release' && '--release' \|\| '' }}

	- name: Prepare artifact
	run: \|
	mkdir -p s2n-quic-qns
	cp target/${{ matrix.mode }}/s2n-quic-qns s2n-quic-qns/s2n-quic-qns-${{ matrix.mode }}

	- uses: actions/upload-artifact@v4
	with:
	name: s2n-quic-qns-${{ matrix.mode }}
	path: s2n-quic-qns/

	interop:
	runs-on: ubuntu-22.04
	needs: [env, s2n-quic-qns]
	strategy:
	matrix: ${{ fromJson(needs.env.outputs.matrix) }}
	steps:
	- uses: actions/checkout@v4
	with:
	path: s2n-quic

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-debug
	path: s2n-quic-qns/

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-release
	path: s2n-quic-qns/

	- name: Setup dockerfile
	working-directory: s2n-quic-qns
	run: \|
	cp ../s2n-quic/quic/s2n-quic-qns/etc/Dockerfile .
	cp ../s2n-quic/quic/s2n-quic-qns/etc/run_endpoint.sh .

	- name: Run docker build
	working-directory: s2n-quic-qns
	env:
	DOCKER_BUILDKIT: 1
	run: \|
	docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns --build-arg tls=s2n-tls
	docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns-rustls --build-arg tls=rustls

	- uses: actions/checkout@v4
	with:
	repository: marten-seemann/quic-interop-runner
	ref: ${{ env.INTEROP_RUNNER_REF }}
	path: quic-interop-runner

	- name: Patch quic-interop-runner
	working-directory: quic-interop-runner
	run: \|
	git apply --3way ../s2n-quic/.github/interop/runner.patch

	- name: Run docker pull
	working-directory: quic-interop-runner
	run: \|
	docker pull "wesleyrosenblum/quic-network-simulator@$NETWORK_SIMULATOR_REF"
	docker pull "martenseemann/quic-interop-iperf-endpoint@$IPERF_ENDPOINT_REF"

	- uses: actions/setup-python@v5
	with:
	python-version: 3.7

	- name: Install tshark
	run: \|
	wget --no-verbose https://dnglbrstg7yg.cloudfront.net/tshark/v$WIRESHARK_VERSION/tshark
	chmod +x tshark
	sudo mv tshark /usr/bin
	/usr/bin/tshark -v

	- name: Install dependencies
	working-directory: quic-interop-runner
	run: \|
	python3 -m pip install --upgrade pip
	pip3 install wheel
	pip3 install --upgrade -r requirements.txt

	- name: Run quic-interop-runner
	working-directory: quic-interop-runner
	run: \|
	# enable IPv6 support
	sudo modprobe ip6table_filter
	python3 run.py --client ${{ matrix.client }} --server ${{ matrix.server }} --json results/result.json --debug --log-dir results/logs
	mkdir -p results/logs

	- name: Prepare artifacts
	working-directory: quic-interop-runner
	run: \|
	ls -al results
	# clean up invalid path characters
	find results -name ':' \| while read from; do
	echo "Invalid filename: $from"
	to=$(echo $from \| sed 's/:/_/g')
	mv $from $to
	done
	# remove files we don't do anything with to reduce the artifact size
	find results -name '*.qlog' -exec rm {} \;
	# remove cross traffic and goodput packet captures as they are large and not useful
	find results -maxdepth 7 -type d -path "*/crosstraffic//sim" \| xargs rm -rf \;
	find results -maxdepth 7 -type d -path "*/goodput//sim" \| xargs rm -rf \;

	# Add index files for easy browsing
	find results -maxdepth 3 -type d -path "/logs//*" \| while read from; do
	tree -H "." \
	-h \
	-L 3 \
	-I 'index.html' \
	-T "${{ matrix.client }} client / ${{ matrix.server }} server - $(basename $from)" \
	--noreport \
	--charset utf-8 \
	-o $from/index.html \
	$from
	done

	- uses: aws-actions/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCRole
	role-session-name: S2nQuicGHAS3Session
	aws-region: us-west-2

	- name: Upload to S3
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	id: s3
	working-directory: quic-interop-runner
	run: \|
	TARGET="${{ github.sha }}/interop/logs/latest"
	aws s3 sync results/logs "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks

	- uses: actions/upload-artifact@v4
	with:
	name: interop-${{ matrix.client }}-client-${{ matrix.server }}-server
	path: quic-interop-runner/results/result.json

	- name: Assert no crashes
	working-directory: quic-interop-runner
	run: \|
	! grep -Rq 'The s2n-quic-qns application shut down unexpectedly' results

	interop-report:
	runs-on: ubuntu-22.04
	needs: [interop]
	steps:
	- uses: actions/checkout@v4

	- uses: actions/download-artifact@v4
	with:
	path: results/

	- name: Download latest results
	id: download
	run: \|
	rm -f result.json
	INTEROP_BASE_URL="https://interop.seemann.io/logs/"
	wget ${INTEROP_BASE_URL}latest/result.json \|\| echo '{}' > result.json
	mv result.json latest.json
	INTEROP_LOG_URL=${INTEROP_BASE_URL}$(jq --raw-output '.log_dir' latest.json)/SERVER_CLIENT/TEST/
	echo "INTEROP_LOG_URL=$INTEROP_LOG_URL" >> $GITHUB_OUTPUT

	- name: Get latest successful interop commit SHA on main branch
	id: mainsha
	if: github.event.pull_request
	run: \|
	curl \
	--url "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/qns.yml/runs?branch=main&status=success&per_page=1" \
	--header "Accept: application/vnd.github.v3+json" > latest_workflow_run.json
	MAIN_SHA=$(jq --raw-output '.workflow_runs[0] \| .head_sha' latest_workflow_run.json)
	rm -f latest_workflow_run.json
	echo "MAIN_SHA=$MAIN_SHA" >> $GITHUB_OUTPUT

	- name: Download latest main interop result
	if: github.event.pull_request
	run: \|
	rm -f prev_result.json
	wget $CDN/${{ steps.mainsha.outputs.MAIN_SHA }}/interop/logs/latest/result.json \|\| echo '{}' > result.json
	mv result.json prev_result.json

	- name: Generate report for pull request
	if: github.event.pull_request
	run: \|
	mkdir -p web/logs/latest
	MAIN_SHA=${{ steps.mainsha.outputs.MAIN_SHA }}
	python3 .github/interop/merge.py \
	--prev_version prev_result.json \
	--new_version_suffix "pr${{github.event.pull_request.number}}" \
	--new_version_url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/pull/${{github.event.pull_request.number}}" \
	--new_version_log_url "$LOG_URL" \
	--prev_version_log_url "$CDN/$MAIN_SHA/interop/$LOG_URL" \
	--prev_version_url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/tree/$MAIN_SHA" \
	--interop_log_url "${{ steps.download.outputs.INTEROP_LOG_URL }}" \
	latest.json \
	results/**/result.json > \
	web/logs/latest/result.json

	- name: Generate report for push to main
	if: github.event_name == 'push'
	run: \|
	mkdir -p web/logs/latest
	python3 .github/interop/merge.py \
	--new_version_log_url "$LOG_URL" \
	--new_version_url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/tree/$GITHUB_SHA" \
	--interop_log_url "${{ steps.download.outputs.INTEROP_LOG_URL }}" \
	latest.json \
	results/**/result.json > \
	web/logs/latest/result.json

	- uses: aws-actions/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCRole
	role-session-name: S2nQuicGHAS3Session
	aws-region: us-west-2

	- name: Upload to S3
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	id: s3
	run: \|
	cp .github/interop/*.html web/
	cp .github/interop/*.js web/
	TARGET="${{ github.sha }}/interop"
	aws s3 sync web "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
	URL="$CDN/$TARGET/index.html"
	echo "URL=$URL" >> $GITHUB_OUTPUT

	- uses: ouzi-dev/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	name: "interop / report"
	status: "success"
	url: "${{ steps.s3.outputs.URL }}"

	- name: Check for regressions
	run: \|
	python3 .github/interop/check.py \
	--required .github/interop/required.json \
	web/logs/latest/result.json

	bench:
	runs-on: ubuntu-22.04
	needs: [env, s2n-quic-qns]
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true

	- name: Install rust toolchain
	id: toolchain
	run: \|
	rustup toolchain install stable
	rustup override set stable

	- uses: camshaft/rust-cache@v1

	- name: Install tshark
	run: \|
	wget --no-verbose https://dnglbrstg7yg.cloudfront.net/tshark/v$WIRESHARK_VERSION/tshark
	chmod +x tshark
	sudo mv tshark /usr/bin
	/usr/bin/tshark -v

	- name: Install gnuplot
	run: \|
	sudo apt-get -o Acquire::Retries=3 update
	sudo apt-get -o Acquire::Retries=3 install -y gnuplot

	- uses: aws-actions/[email protected]
	if: github.repository == github.event.pull_request.head.repo.full_name
	with:
	role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCEcrRole
	role-session-name: S2nQuicGHAECRSession
	aws-region: us-east-1 # Required for ECR

	# authenticate pull to avoid hitting pull quota
	- name: Login to Amazon ECR Public
	if: github.repository == github.event.pull_request.head.repo.full_name
	id: login-ecr-public
	uses: aws-actions/amazon-ecr-login@v2
	with:
	registry-type: public

	- name: Pull s2n-quic-qns:main
	if: github.event.pull_request
	run: docker pull public.ecr.aws/s2n/s2n-quic-qns:main

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-debug
	path: s2n-quic-qns-build/

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-release
	path: s2n-quic-qns-build/

	- name: Setup dockerfile
	working-directory: s2n-quic-qns-build
	run: \|
	cp ../quic/s2n-quic-qns/etc/Dockerfile .
	cp ../quic/s2n-quic-qns/etc/run_endpoint.sh .

	- name: Run docker build
	working-directory: s2n-quic-qns-build
	env:
	DOCKER_BUILDKIT: 1
	run: \|
	docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns

	- name: Run script for pull request
	if: github.event.pull_request
	run: sudo env "PATH=$PATH" "BUILD_S2N_QUIC=false" "COMPARE_TO_MAIN=true" ./scripts/benchmark/run-all

	- name: Run script for push to main
	if: github.event_name == 'push'
	run: sudo env "PATH=$PATH" "BUILD_S2N_QUIC=false" ./scripts/benchmark/run-all

	- uses: aws-actions/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCRole
	role-session-name: S2nQuicGHAS3Session
	aws-region: us-west-2

	- name: Upload results
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	id: s3
	run: \|
	TARGET="${{ github.sha }}/bench"
	aws s3 sync target/benchmark/results "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
	URL="$CDN/$TARGET/index.html"
	echo "URL=$URL" >> $GITHUB_OUTPUT

	- uses: ouzi-dev/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	name: "bench / report"
	status: "success"
	url: "${{ steps.s3.outputs.URL }}"

	- name: Assert no crashes
	run: \|
	! grep -Rq 'The s2n-quic-qns application shut down unexpectedly' target/benchmark/results

	h3spec:
	runs-on: ubuntu-22.04
	needs: [s2n-quic-qns]
	strategy:
	matrix:
	tls: ["s2n-tls", "rustls"]

	steps:
	- uses: actions/checkout@v4

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-debug

	- name: Run test
	run: \|
	chmod +x s2n-quic-qns-debug
	./s2n-quic-qns-debug interop server --port 4433 --tls ${{ matrix.tls }} &
	# wait for the server to boot
	sleep 3
	./scripts/h3spec

	perf:
	runs-on: ubuntu-22.04
	needs: [s2n-quic-qns]
	strategy:
	matrix:
	include:
	- client: "s2n-quic"
	server: "s2n-quic"
	- client: "s2n-quic-null"
	server: "s2n-quic-null"

	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true

	- name: Install rust toolchain
	id: toolchain
	run: \|
	rustup toolchain install stable --profile minimal
	rustup override set stable

	- name: Install perf
	run: \|
	sudo apt-get -y update
	sudo apt-get install -y linux-tools-$(uname -r) linux-tools-generic

	- name: Install inferno
	uses: camshaft/install@v1
	with:
	crate: inferno
	bins: inferno-collapse-perf,inferno-flamegraph

	- name: Install ultraman
	uses: camshaft/install@v1
	with:
	crate: ultraman

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-release
	path: target/release/

	- name: Setup artifacts
	run: \|
	mv target/release/s2n-quic-qns-release target/release/s2n-quic-qns
	chmod +x target/release/s2n-quic-qns

	- name: Run script
	env:
	# ultraman wants a SHELL var to spawn tasks
	SHELL: /bin/bash
	run: \|
	set -e

	# set larger socket buffers
	sudo sysctl -w net.core.wmem_default=2000000
	sudo sysctl -w net.core.rmem_default=2000000

	mkdir -p target/perf/results
	sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 10000 0 ${{ matrix.server }} ${{ matrix.client }}
	sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 7500 2500 ${{ matrix.server }} ${{ matrix.client }}
	sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 5000 5000 ${{ matrix.server }} ${{ matrix.client }}
	sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 2500 7500 ${{ matrix.server }} ${{ matrix.client }}
	sudo env "PATH=$PATH" "SHELL=$SHELL" ./scripts/perf/test 0 10000 ${{ matrix.server }} ${{ matrix.client }}
	sudo chown -R $(whoami) target/perf/results

	- name: Prepare artifacts
	run: \|
	cd ./target/perf/results
	zip perf.zip */.script
	rm -rf */.script

	- uses: actions/upload-artifact@v4
	with:
	name: perf-results-${{ matrix.server }}-${{ matrix.client }}
	path: target/perf/results

	perf-report:
	runs-on: ubuntu-22.04
	needs: [perf]
	steps:
	- uses: actions/checkout@v4

	# add any additional perf tests here
	- uses: actions/download-artifact@v4
	with:
	name: perf-results-s2n-quic-s2n-quic
	path: perf-results/
	- uses: actions/download-artifact@v4
	with:
	name: perf-results-s2n-quic-null-s2n-quic-null
	path: perf-results/

	- name: Generate report
	run: \|
	cd perf-results
	tree -H "." -T "Performance Results" --noreport --charset utf-8 > index.html

	- uses: aws-actions/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCRole
	role-session-name: S2nQuicGHAS3Session
	aws-region: us-west-2

	- name: Upload results
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	id: s3
	run: \|
	TARGET="${{ github.sha }}/perf"
	aws s3 sync perf-results "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks
	URL="$CDN/$TARGET/index.html"
	echo "URL=$URL" >> $GITHUB_OUTPUT

	- uses: ouzi-dev/[email protected]
	if: github.event_name == 'push' \|\| github.repository == github.event.pull_request.head.repo.full_name
	with:
	name: "perf / report"
	status: "success"
	url: "${{ steps.s3.outputs.URL }}"

	attack:
	runs-on: ubuntu-22.04
	needs: [s2n-quic-qns]
	strategy:
	matrix:
	attack: ["udp"]

	steps:
	- uses: actions/checkout@v4

	- name: Install rust toolchain
	id: toolchain
	run: \|
	rustup toolchain install stable --profile minimal
	rustup override set stable

	- uses: actions/download-artifact@v4
	with:
	name: s2n-quic-qns-debug

	- name: Run cargo build
	working-directory: tools/${{ matrix.attack }}-attack
	run: cargo +stable build --release

	- name: Start client
	working-directory: tools/${{ matrix.attack }}-attack
	run: \|
	./target/release/${{ matrix.attack }}-attack localhost:4433 &

	- name: Start server
	shell: bash
	run: \|
	chmod +x ./s2n-quic-qns-debug
	# disable exiting on errors to capture the timeout status
	set +e

	timeout 5m ./s2n-quic-qns-debug interop server --port 4433
	EXIT_CODE="$?"

	set -e
	# `timeout` exits with `124` if the time limit was reached
	[[ "$EXIT_CODE" == "124" ]] \|\| exit $EXIT_CODE

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(s2n-quic-core): add aggregate metrics support #4980

Workflow file

feat(s2n-quic-core): add aggregate metrics support #4980

Jobs

Run details

Workflow file for this run