Skip to content

Release

Release #131

Workflow file for this run

name: Release
on:
push:
tags:
- "v*"
schedule:
# Release at 00:00 UTC+8
- cron: "0 16 * * *"
workflow_dispatch:
inputs:
tags:
description: The tags to be released
required: false
type: string
permissions:
id-token: write
pull-requests: write
checks: write
statuses: write
contents: write
env:
BUILD_PROFILE: release
jobs:
create_release:
name: create release
runs-on: ubuntu-latest
outputs:
version: ${{ steps.generated-tag.outputs.tag }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get latest tag
id: get-latest-tag
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "tag=`gh release list -L 1 | cut -f 1`" >> $GITHUB_OUTPUT
- name: Bump version
id: generated-tag
uses: actions/github-script@v6
with:
script: |
if (context.ref.startsWith("refs/tags/")) {
let tag = context.ref.replace("refs/tags/", "");
core.setOutput('tag', tag);
console.log(`This event pushed a tag ${tag}, return directly.`)
return
}
if ("${{ github.event.inputs.tags }}") {
let tag = "${{ github.event.inputs.tags }}";
core.setOutput('tag', tag);
console.log(`This event triggered by workflow_dispatch with a tag ${tag}, return directly.`)
return
}
let tag = "${{ steps.get-latest-tag.outputs.tag }}";
let result = /v(\d+)\.(\d+)\.(\d+)/g.exec(tag);
if (result === null) {
throw `The previous tag ${{ steps.get-latest-tag.outputs.tag }} is invalid, ignoring`;
}
let major = result[1];
let minor = result[2];
let patch = (parseInt(result[3]) + 1).toString();
let next_tag = `v${major}.${minor}.${patch}-nightly`;
console.log(`This event is triggered, return generated ${next_tag}.`)
core.setOutput('tag', next_tag)
- name: Create github release if not exist
# Only create release when the tag is not exist
if: 'steps.generated-tag.outputs.tag != steps.get-latest-tag.outputs.tag'
# Allow this action failure
# continue-on-error: true
# Reference: https://cli.github.com/manual/gh_release_create
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "Create a release for ${{ steps.generated-tag.outputs.tag }}"
gh release create ${{ steps.generated-tag.outputs.tag }} --generate-notes -p
macos:
runs-on: macos-11
needs: create_release
env:
RUNNER_PROVIDER: github
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- aarch64
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get target
id: target
run: echo 'target=${{ matrix.arch }}-apple-darwin' >> $GITHUB_OUTPUT
- name: Install coreutils for macOS sha256sum
run: brew install coreutils
- name: Build Release
uses: ./.github/actions/build_macos
with:
target: ${{ steps.target.outputs.target }}
artifacts: sqllogictests,metactl,meta,query
upload: false
- name: Basic Sqllogic Test
if: matrix.arch == 'x86_64'
shell: bash
env:
TEST_HANDLERS: http
run: |
mkdir -p target/release
cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,metactl,query,sqllogictests} ./target/release/
bash ./scripts/ci/ci-run-sqllogic-tests.sh base
- name: Pack binaries
run: |
brew install gnu-tar
sudo /usr/sbin/purge
target=${{ steps.target.outputs.target }}
version=${{ needs.create_release.outputs.version }}
mkdir -p release/${target}/{bin,configs,scripts}
cp ./target/${target}/release/databend-* release/${target}/bin/
rm -f release/${target}/bin/*.d
cp ./scripts/distribution/configs/databend-* release/${target}/configs/
cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
gtar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs scripts readme.txt
- name: generate sha256sums
run: |
target=${{ steps.target.outputs.target }}
version=${{ needs.create_release.outputs.version }}
sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
- name: post sha256
uses: actions/upload-artifact@v3
with:
name: sha256sums
path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
retention-days: 1
- name: Publish Binaries
uses: ./.github/actions/publish_binary
env:
GH_TOKEN: ${{ github.token }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
with:
version: ${{ needs.create_release.outputs.version }}
target: ${{ steps.target.outputs.target }}
linux:
runs-on: [self-hosted, X64, Linux, 16c32g, gcp]
needs: create_release
env:
RUNNER_PROVIDER: gcp
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- aarch64
libc:
- gnu
- musl
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get the version
id: get_version
run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
- name: Get target
id: target
run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}' >> $GITHUB_OUTPUT
- name: Build Release
uses: ./.github/actions/build_linux
with:
sha: ${{ github.sha }}
target: ${{ steps.target.outputs.target }}
artifacts: sqllogictests,metactl,meta,query
upload: false
- name: Basic Sqllogic Test
if: matrix.arch == 'x86_64'
shell: bash
env:
TEST_HANDLERS: http
run: |
mkdir -p target/release
cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,query,sqllogictests} ./target/release/
bash ./scripts/ci/ci-run-sqllogic-tests.sh base
- name: Pack binaries
run: |
target=${{ steps.target.outputs.target }}
version=${{ needs.create_release.outputs.version }}
mkdir -p release/${target}/{bin,configs,systemd,scripts}
cp ./target/${target}/release/databend-* release/${target}/bin/
rm -f release/${target}/bin/*.d
cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/
cp ./scripts/distribution/configs/databend-* release/${target}/configs/
cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/
tar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs systemd scripts readme.txt
- name: generate sha256sums
run: |
target=${{ steps.target.outputs.target }}
version=${{ needs.create_release.outputs.version }}
sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
- name: post sha256
uses: actions/upload-artifact@v3
with:
name: sha256sums
path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
retention-days: 1
- name: Publish Binaries
uses: ./.github/actions/publish_binary
env:
GH_TOKEN: ${{ github.token }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
with:
version: ${{ needs.create_release.outputs.version }}
target: ${{ steps.target.outputs.target }}
hdfs:
runs-on: [self-hosted, X64, Linux, 16c32g, gcp]
needs: create_release
env:
RUNNER_PROVIDER: gcp
strategy:
fail-fast: false
matrix:
arch:
- x86_64
libc:
- gnu
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get the version
id: get_version
run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
- name: Get target
id: target
run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}' >> $GITHUB_OUTPUT
- name: Setup Build Tool
uses: ./.github/actions/setup_build_tool
with:
target: ${{ steps.target.outputs.target }}
- name: Build Release
uses: ./.github/actions/build_linux
with:
sha: ${{ github.sha }}
target: ${{ steps.target.outputs.target }}
artifacts: meta,query
category: hdfs
features: storage-hdfs
upload: false
- name: Pack binaries
run: |
target=${{ steps.target.outputs.target }}
version=${{ needs.create_release.outputs.version }}
mkdir -p release/${target}/{bin,configs,systemd,scripts}
cp ./target/${target}/release/databend-* release/${target}/bin/
rm -f release/${target}/bin/*.d
cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/
cp ./scripts/distribution/configs/databend-* release/${target}/configs/
cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/
tar -C ./release/${target} -czvf databend-hdfs-${version}-${target}.tar.gz bin configs systemd scripts readme.txt
- name: generate sha256sums
run: |
target=${{ steps.target.outputs.target }}
version=${{ needs.create_release.outputs.version }}
sha256sum databend-hdfs-${version}-${target}.tar.gz >> sha256-hdfs-${version}-${target}.txt
- name: post sha256
uses: actions/upload-artifact@v3
with:
name: sha256sums
path: sha256-hdfs-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
retention-days: 1
- name: Publish Binaries
uses: ./.github/actions/publish_binary
env:
GH_TOKEN: ${{ github.token }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
with:
version: ${{ needs.create_release.outputs.version }}
target: ${{ steps.target.outputs.target }}
category: hdfs
testsuites:
runs-on: ubuntu-latest
needs: create_release
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get the version
id: get_version
run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
- name: Upload to github release
env:
GH_TOKEN: ${{ github.token }}
run: |
version=${{ needs.create_release.outputs.version }}
tar -C ./tests/sqllogictests -czvf testsuites-${version}.tar.gz suites
gh release upload ${version} testsuites-${version}.tar.gz --clobber
docker_combined:
name: docker combined
runs-on: ubuntu-latest
needs: [create_release, linux]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get the version
id: get_version
run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
- name: Download binaries for usage
id: download_binaries
env:
GH_TOKEN: ${{ github.token }}
run: |
version="${{ needs.create_release.outputs.version }}"
declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu")
mkdir -p ./distro/
for platform in ${!platform_targets[@]}; do
target=${platform_targets[$platform]}
gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
mkdir -p ./target/${target}/release
tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
mkdir -p ./distro/linux/${platform}
cp ./target/${target}/release/databend-* ./distro/linux/${platform}
done
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- uses: ./.github/actions/setup_docker
id: login
with:
repo: databend
ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
id: docker_build
uses: docker/build-push-action@v3
with:
push: true
tags: |
${{ steps.login.outputs.dockerhub_repo }}:latest
${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}
${{ steps.login.outputs.ecr_repo }}:latest
${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}
platforms: linux/amd64,linux/arm64
context: .
file: ./docker/Dockerfile
- name: Update repo description
continue-on-error: true
uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: ${{ steps.login.outputs.dockerhub_repo }}
short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com."
readme-filepath: ./docker/README.md
docker_separate:
name: docker separate
runs-on: ubuntu-latest
needs: [create_release, linux]
strategy:
fail-fast: false
matrix:
service:
- meta
- query
distro:
- debian
- distroless
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get the version
id: get_version
run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
- name: Download binaries for usage
id: download_binaries
env:
GH_TOKEN: ${{ github.token }}
run: |
version="${{ needs.create_release.outputs.version }}"
declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu")
mkdir -p ./distro/
for platform in ${!platform_targets[@]}; do
target=${platform_targets[$platform]}
gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
mkdir -p ./target/${target}/release
tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
mkdir -p ./distro/linux/${platform}
cp ./target/${target}/release/databend-* ./distro/linux/${platform}
done
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- uses: ./.github/actions/setup_docker
id: login
with:
repo: databend-${{ matrix.service }}
ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
- name: get image tags
id: get_image_tags
shell: bash
run: |
_tags="${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}"
_tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}"
if [[ "${{ matrix.distro }}" == "debian" ]]; then
_tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}"
_tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}"
_tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:latest"
_tags="${_tags},${{ steps.login.outputs.ecr_repo }}:latest"
fi
echo "IMAGE_TAGS=${_tags}" >> $GITHUB_OUTPUT
- name: push service image
uses: docker/build-push-action@v3
with:
push: true
tags: ${{ steps.get_image_tags.outputs.IMAGE_TAGS }}
platforms: linux/amd64,linux/arm64
context: .
file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile
distribution:
runs-on: ubuntu-latest
needs: [create_release, linux]
strategy:
matrix:
arch:
- x86_64
- aarch64
packager:
- deb
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install nfpm@latest
run: |
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
tar xf nfpm.tar.gz
sudo mv nfpm /usr/local/bin
sudo chmod a+x /usr/local/bin/nfpm
rm nfpm.tar.gz
- name: Get target
id: target
run: |
echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
- name: Download binaries for usage
id: download_binaries
env:
GH_TOKEN: ${{ github.token }}
run: |
target=${{ steps.target.outputs.target }}
version="${{ needs.create_release.outputs.version }}"
mkdir -p ./distro/
gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz
- name: Build Packages
id: build_packages
run: |
export name="databend"
export version="${{ needs.create_release.outputs.version }}"
export path="distro"
case "${{ matrix.arch }}" in
x86_64)
export arch="amd64"
;;
aarch64)
export arch="arm64"
;;
esac
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
- name: Update release to github
shell: bash
env:
GH_TOKEN: ${{ github.token }}
# Reference: https://cli.github.com/manual/gh_release_upload
run: |
version="${{ needs.create_release.outputs.version }}"
# name looks like: `databend_0.8.144~nightly_amd64.deb`
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
deb:
runs-on: ubuntu-latest
needs: [create_release, distribution]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/publish_deb
env:
GH_TOKEN: ${{ github.token }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
with:
version: ${{ needs.create_release.outputs.version }}
gpg_signing_key: ${{ secrets.GPG_KEY_DEB }}
sha256sums:
needs: [create_release, linux, macos, distribution]
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
- name: download sha256sums
uses: actions/download-artifact@v3
with:
name: sha256sums
- shell: bash
run: |
for file in *.txt
do
cat ${file} >> sha256sums.txt
done
- name: Upload checksums
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
version="${{ needs.create_release.outputs.version }}"
gh release upload ${version} sha256sums.txt --clobber
benchmark:
needs: [create_release, linux]
uses: ./.github/workflows/reuse.benchmark.yml
secrets: inherit
with:
sha: ${{ github.sha }}
run_id: ${{ github.run_id }}
source: release
source_id: ${{ needs.create_release.outputs.version }}
version: ${{ needs.create_release.outputs.version }}
runner_provider: aws
bindings_python:
needs: create_release
uses: ./.github/workflows/bindings.python.yml
secrets: inherit
with:
tag: ${{ needs.create_release.outputs.version }}
notify:
runs-on: ubuntu-latest
if: always()
needs:
- create_release
- linux
- docker_combined
- docker_separate
- distribution
- deb
steps:
- uses: actions/checkout@v4
- run: |
status="${{ (contains(needs.*.result, 'failure') && 'failure') || (contains(needs.*.result, 'cancelled') && 'cancelled') || 'success' }}"
jq -n -f .github/release-report.jq \
--arg title "[Release] ${{ needs.create_release.outputs.version }}" \
--arg content "Build result: ${status}" \
--arg link "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
> /tmp/release-report.json
curl -X POST "${{ secrets.RELEASE_REPORT_WEBHOOK }}" \
-H 'Content-Type: application/json' \
-H 'cache-control: no-cache' \
-d @/tmp/release-report.json