Release #427
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Release | |
on: | |
schedule: | |
- cron: "0 22 * * *" | |
workflow_dispatch: | |
inputs: | |
tag: | |
description: The tags to be released | |
required: false | |
type: string | |
stable: | |
description: Make a stable release | |
required: false | |
type: boolean | |
permissions: | |
id-token: write | |
pull-requests: write | |
checks: write | |
statuses: write | |
contents: write | |
env: | |
BUILD_PROFILE: release | |
RUNNER_PROVIDER: gcp | |
jobs: | |
create_release: | |
runs-on: ubuntu-latest | |
outputs: | |
version: ${{ steps.bump.outputs.tag }} | |
sha: ${{ steps.bump.outputs.sha }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- name: Bump version | |
id: bump | |
uses: actions/github-script@v7 | |
env: | |
STABLE: "${{ inputs.stable }}" | |
TAG: "${{ inputs.tag }}" | |
with: | |
script: | | |
const script = require('./.github/scripts/bump_version.js') | |
await script({ github, context, core }) | |
- name: Create release | |
env: | |
# we need workflow:write permission to create release if there were any workflow changes | |
# which is not possible for github actions token | |
GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }} | |
run: | | |
echo "Creating release ${{ steps.bump.outputs.tag }} from ${{ steps.bump.outputs.sha }}" | |
if [[ "${{ inputs.stable }}" == "true" ]]; then | |
echo "Stable release" | |
previous=$(gh release list --limit 1 --exclude-pre-releases | cut -f 1) | |
gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag $previous --latest --draft | |
else | |
echo "Nightly release" | |
previous=$(gh release list --limit 1 | cut -f 1) | |
gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag $previous --prerelease --draft | |
fi | |
changelog: | |
runs-on: ubuntu-latest | |
if: inputs.stable | |
needs: create_release | |
steps: | |
- name: Checkout Docs | |
uses: actions/checkout@v4 | |
with: | |
repository: datafuselabs/databend-docs | |
ref: main | |
- name: Get date | |
shell: bash | |
run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV | |
- name: Generate Release Note | |
env: | |
GH_TOKEN: ${{ github.token }} | |
run: | | |
mkdir -p docs/release-stable | |
df="docs/release-stable/${{ env.DATE }}_${{ needs.create_release.outputs.version }}.md" | |
echo "---" > $df | |
gh release view --repo datafuselabs/databend ${{ needs.create_release.outputs.version }} >> $df | |
sed -i -E 's/^--$/---/g' $df | |
sed -i -E '/^asset:/d' $df | |
sed -i -E 's_https://github.com/datafuselabs/databend/pull/([0-9]+)_[#\1](https://github.com/datafuselabs/databend/pull/\1)_g' $df | |
git add docs/release-stable | |
git status | |
- uses: peter-evans/create-pull-request@v4 | |
with: | |
token: ${{ secrets.DATABEND_BOT_TOKEN }} | |
title: "chore(docs): Update Release Notes - ${{ env.DATE }}" | |
base: main | |
commit-message: "chore(docs): Update Release Notes - ${{ env.DATE }}" | |
branch-suffix: random | |
delete-branch: true | |
build_default: | |
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] | |
needs: create_release | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- { target: x86_64-unknown-linux-gnu, runner: X64 } | |
- { target: aarch64-unknown-linux-gnu, runner: ARM64 } | |
- { target: x86_64-unknown-linux-musl, runner: X64 } | |
- { target: aarch64-unknown-linux-musl, runner: X64 } | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
fetch-depth: 0 | |
- name: Build Release | |
uses: ./.github/actions/build_linux | |
env: | |
DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} | |
with: | |
sha: ${{ github.sha }} | |
target: ${{ matrix.target }} | |
artifacts: sqllogictests,sqlsmith,metactl,meta,query | |
- name: Basic Sqllogic Test | |
if: matrix.target != 'aarch64-unknown-linux-musl' | |
shell: bash | |
env: | |
TEST_HANDLERS: http | |
run: | | |
mkdir -p ./target/${{ env.BUILD_PROFILE }} | |
cp ./target/${{ matrix.target }}/${{ env.BUILD_PROFILE }}/databend-* ./target/${{ env.BUILD_PROFILE }}/ | |
bash ./scripts/ci/ci-run-sqllogic-tests.sh base | |
build_udf: | |
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] | |
needs: create_release | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- { target: x86_64-unknown-linux-gnu, runner: X64 } | |
- { target: aarch64-unknown-linux-gnu, runner: ARM64 } | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
fetch-depth: 0 | |
- name: Build Release | |
uses: ./.github/actions/build_linux | |
env: | |
DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} | |
with: | |
sha: ${{ github.sha }} | |
target: ${{ matrix.target }} | |
artifacts: sqllogictests,sqlsmith,metactl,meta,query | |
features: python-udf | |
category: udf | |
build_hdfs: | |
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, gcp] | |
needs: create_release | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- { target: x86_64-unknown-linux-gnu, runner: X64 } | |
- { target: aarch64-unknown-linux-gnu, runner: ARM64 } | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
fetch-depth: 0 | |
- name: Build Release | |
uses: ./.github/actions/build_linux | |
env: | |
DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} | |
with: | |
sha: ${{ github.sha }} | |
target: ${{ matrix.target }} | |
artifacts: sqllogictests,sqlsmith,metactl,meta,query | |
features: storage-hdfs | |
category: hdfs | |
publish: | |
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c8g, gcp] | |
needs: [create_release, build_default, build_hdfs] | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- category: default | |
target: x86_64-unknown-linux-gnu | |
runner: X64 | |
- category: default | |
target: aarch64-unknown-linux-gnu | |
runner: ARM64 | |
- category: default | |
target: x86_64-unknown-linux-musl | |
runner: X64 | |
- category: default | |
target: aarch64-unknown-linux-musl | |
runner: X64 | |
- category: hdfs | |
target: x86_64-unknown-linux-gnu | |
runner: X64 | |
- category: hdfs | |
target: aarch64-unknown-linux-gnu | |
runner: ARM64 | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Download artifact | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: ${{ matrix.target }} | |
category: ${{ matrix.category }} | |
path: distro/bin | |
artifacts: metactl,meta,query | |
- name: Pack Binaries | |
run: | | |
target=${{ matrix.target }} | |
version=${{ needs.create_release.outputs.version }} | |
case ${{ matrix.category }} in | |
default) | |
pkg_name="databend-${version}-${target}" | |
;; | |
*) | |
pkg_name="databend-${{ matrix.category }}-${version}-${target}" | |
;; | |
esac | |
mkdir -p distro/{bin,configs,systemd,scripts} | |
cp ./scripts/distribution/systemd/databend-* distro/systemd/ | |
cp ./scripts/distribution/configs/databend-* distro/configs/ | |
cp ./scripts/distribution/release-readme.txt distro/readme.txt | |
cp -r ./scripts/distribution/local-scripts/* distro/scripts/ | |
cp -r ./scripts/distribution/package-scripts/* distro/scripts/ | |
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt | |
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt | |
- name: post sha256 | |
uses: actions/upload-artifact@v4 | |
with: | |
name: sha256sums-${{ matrix.category }}-${{ matrix.target }} | |
path: sha256-*.txt | |
retention-days: 1 | |
- name: Publish Binaries | |
uses: ./.github/actions/publish_binary | |
env: | |
GH_TOKEN: ${{ github.token }} | |
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
AWS_DEFAULT_REGION: auto | |
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} | |
with: | |
version: ${{ needs.create_release.outputs.version }} | |
target: ${{ matrix.target }} | |
category: ${{ matrix.category }} | |
publish_testsuite: | |
runs-on: [self-hosted, X64, Linux, 4c8g, gcp] | |
needs: [create_release, build_default] | |
strategy: | |
fail-fast: false | |
matrix: | |
target: | |
- x86_64-unknown-linux-gnu | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Download artifact | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: ${{ matrix.target }} | |
category: default | |
artifacts: sqllogictests,sqlsmith | |
path: distro/bin | |
- name: Pack Testsuite | |
run: | | |
target=${{ matrix.target }} | |
version=${{ needs.create_release.outputs.version }} | |
pkg_name="databend-testsuite-${version}-${target}" | |
cp -r ./tests/sqllogictests/suites ./distro/ | |
tar -C ./distro -czvf ${pkg_name}.tar.gz bin suites | |
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt | |
- name: post sha256 | |
uses: actions/upload-artifact@v4 | |
with: | |
name: sha256sums-testsuite-${{ matrix.target }} | |
path: sha256-*.txt | |
retention-days: 1 | |
- name: Publish Testsuite | |
uses: ./.github/actions/publish_binary | |
env: | |
GH_TOKEN: ${{ github.token }} | |
with: | |
version: ${{ needs.create_release.outputs.version }} | |
target: ${{ matrix.target }} | |
category: testsuite | |
docker_all_in_one: | |
runs-on: [self-hosted, X64, Linux, 4c8g, gcp] | |
needs: [create_release, build_default] | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Download artifacts for x86_64 | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: x86_64-unknown-linux-gnu | |
category: default | |
artifacts: metactl,meta,query | |
path: distro/linux/amd64 | |
- name: Download artifacts for aarch64 | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: aarch64-unknown-linux-gnu | |
category: default | |
artifacts: metactl,meta,query | |
path: distro/linux/arm64 | |
- name: Set up QEMU | |
uses: docker/setup-qemu-action@v3 | |
- uses: ./.github/actions/setup_docker | |
id: login | |
with: | |
repo: databend | |
ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
- name: Get Image Tags | |
id: tags | |
uses: actions/github-script@v7 | |
env: | |
REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }} | |
REPO_ECR: ${{ steps.login.outputs.ecr_repo }} | |
VERSION: ${{ needs.create_release.outputs.version }} | |
STABLE: ${{ inputs.stable }} | |
with: | |
script: | | |
const version = process.env.VERSION; | |
const repos = [process.env.REPO_DOCKERHUB, process.env.REPO_ECR]; | |
const stable = process.env.STABLE; | |
let tags = []; | |
for (const repo of repos) { | |
tags.push(`${repo}:${version}`); | |
if (stable === 'true') { | |
tags.push(`${repo}:latest`); | |
} else { | |
tags.push(`${repo}:nightly`); | |
} | |
} | |
core.setOutput('tags', tags.join(',')); | |
- name: Build and push | |
id: docker_build | |
uses: docker/build-push-action@v5 | |
with: | |
push: true | |
tags: ${{ steps.tags.outputs.tags }} | |
platforms: linux/amd64,linux/arm64 | |
context: . | |
file: ./docker/Dockerfile | |
- name: Update repo description | |
continue-on-error: true | |
uses: peter-evans/dockerhub-description@v4 | |
with: | |
username: ${{ secrets.DOCKERHUB_USERNAME }} | |
password: ${{ secrets.DOCKERHUB_TOKEN }} | |
repository: ${{ steps.login.outputs.dockerhub_repo }} | |
short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com." | |
readme-filepath: ./docker/README.md | |
docker_service: | |
runs-on: [self-hosted, X64, Linux, 4c8g, gcp] | |
needs: [create_release, build_udf] | |
strategy: | |
fail-fast: false | |
matrix: | |
service: | |
- meta | |
- query | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Download artifacts for x86_64 | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: x86_64-unknown-linux-gnu | |
category: udf | |
artifacts: ${{ matrix.service }},metactl | |
path: distro/linux/amd64 | |
- name: Download artifacts for aarch64 | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: aarch64-unknown-linux-gnu | |
category: udf | |
artifacts: ${{ matrix.service }},metactl | |
path: distro/linux/arm64 | |
- name: Set up QEMU | |
uses: docker/setup-qemu-action@v3 | |
- uses: ./.github/actions/setup_docker | |
id: login | |
with: | |
repo: databend-${{ matrix.service }} | |
ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
- name: Get Image Tags | |
id: tags | |
uses: actions/github-script@v7 | |
env: | |
REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }} | |
REPO_ECR: ${{ steps.login.outputs.ecr_repo }} | |
VERSION: ${{ needs.create_release.outputs.version }} | |
STABLE: ${{ inputs.stable }} | |
with: | |
script: | | |
const version = process.env.VERSION; | |
const repos = [process.env.REPO_DOCKERHUB, process.env.REPO_ECR]; | |
const stable = process.env.STABLE; | |
let tags = []; | |
for (const repo of repos) { | |
tags.push(`${repo}:${version}`); | |
if (stable === 'true') { | |
tags.push(`${repo}:latest`); | |
} else { | |
tags.push(`${repo}:nightly`); | |
} | |
} | |
core.setOutput('tags', tags.join(',')); | |
- name: push service image | |
uses: docker/build-push-action@v5 | |
with: | |
push: true | |
tags: ${{ steps.tags.outputs.tags }} | |
platforms: linux/amd64,linux/arm64 | |
context: . | |
file: ./docker/service/${{ matrix.service }}.Dockerfile | |
distribution: | |
runs-on: [self-hosted, X64, Linux, 4c8g, gcp] | |
needs: [create_release, build_default] | |
strategy: | |
matrix: | |
arch: | |
- x86_64 | |
- aarch64 | |
packager: | |
- deb | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Install nfpm@latest | |
run: | | |
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz | |
tar xf nfpm.tar.gz | |
sudo mv nfpm /usr/local/bin | |
sudo chmod a+x /usr/local/bin/nfpm | |
rm nfpm.tar.gz | |
- name: Get target | |
id: target | |
run: | | |
echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT | |
- name: Download artifacts | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: ${{ steps.target.outputs.target }} | |
category: default | |
artifacts: metactl,meta,query | |
path: distro/bin | |
- name: Build Packages | |
id: build_packages | |
run: | | |
export name="databend" | |
export version="${{ needs.create_release.outputs.version }}" | |
export path="distro" | |
case "${{ matrix.arch }}" in | |
x86_64) | |
export arch="amd64" | |
;; | |
aarch64) | |
export arch="arm64" | |
;; | |
esac | |
mkdir -p distro/{bin,configs,systemd,scripts} | |
cp ./scripts/distribution/systemd/databend-* distro/systemd/ | |
cp ./scripts/distribution/configs/databend-* distro/configs/ | |
cp ./scripts/distribution/release-readme.txt distro/readme.txt | |
cp -r ./scripts/distribution/local-scripts/* distro/scripts/ | |
cp -r ./scripts/distribution/package-scripts/* distro/scripts/ | |
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) | |
- name: Update release to github | |
shell: bash | |
env: | |
GH_TOKEN: ${{ github.token }} | |
# Reference: https://cli.github.com/manual/gh_release_upload | |
run: | | |
version="${{ needs.create_release.outputs.version }}" | |
# name looks like: `databend_0.8.144~nightly_amd64.deb` | |
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber | |
# bindings_python: | |
# if: inputs.stable | |
# needs: create_release | |
# uses: ./.github/workflows/bindings.python.yml | |
# secrets: inherit | |
# with: | |
# tag: ${{ needs.create_release.outputs.version }} | |
notify: | |
runs-on: ubuntu-latest | |
if: always() | |
needs: | |
- create_release | |
- publish | |
- docker_all_in_one | |
- docker_service | |
- distribution | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Notify Release | |
uses: actions/github-script@v7 | |
env: | |
JOBS_STATUS: ${{ join(needs.*.result, ',') }} | |
REPORT_WEBHOOK: ${{ secrets.RELEASE_REPORT_WEBHOOK }} | |
VERSION: ${{ needs.create_release.outputs.version }} | |
with: | |
script: | | |
const script = require('./.github/scripts/notify_release.js') | |
await script({context, core}) | |
- name: Publish release | |
env: | |
GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }} | |
run: | | |
gh release edit ${{ needs.create_release.outputs.version }} --draft=false | |
deb: | |
runs-on: ubuntu-latest | |
if: inputs.stable | |
needs: [create_release, distribution] | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- uses: ./.github/actions/publish_deb | |
env: | |
GH_TOKEN: ${{ github.token }} | |
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
AWS_DEFAULT_REGION: auto | |
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} | |
with: | |
version: ${{ needs.create_release.outputs.version }} | |
gpg_signing_key: ${{ secrets.GPG_KEY_DEB }} | |
sha256sums: | |
needs: [create_release, publish, distribution] | |
runs-on: ubuntu-latest | |
steps: | |
- name: checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: download sha256sums | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: sha256sums-* | |
merge-multiple: true | |
- shell: bash | |
run: | | |
for file in *.txt | |
do | |
cat ${file} >> sha256sums.txt | |
done | |
- name: Upload checksums | |
shell: bash | |
env: | |
GH_TOKEN: ${{ github.token }} | |
run: | | |
version="${{ needs.create_release.outputs.version }}" | |
gh release upload ${version} sha256sums.txt --clobber | |
sqlsmith: | |
needs: [create_release, build_default] | |
runs-on: [self-hosted, X64, Linux, 4c8g, gcp] | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
ref: ${{ needs.create_release.outputs.sha }} | |
- name: Download artifacts | |
uses: ./.github/actions/artifact_download | |
with: | |
sha: ${{ github.sha }} | |
target: x86_64-unknown-linux-gnu | |
category: default | |
artifacts: meta,query,sqlsmith | |
- name: Run sqlsmith | |
timeout-minutes: 60 | |
shell: bash | |
run: | | |
bash ./scripts/ci/ci-run-sqlsmith-tests.sh | |
- name: Upload failure | |
if: failure() | |
uses: ./.github/actions/artifact_failure | |
with: | |
name: test-sqlsmith | |
benchmark: | |
needs: [create_release, docker_service, notify] | |
uses: ./.github/workflows/reuse.benchmark.yml | |
secrets: inherit | |
with: | |
sha: ${{ github.sha }} | |
run_id: ${{ github.run_id }} | |
source: release | |
source_id: ${{ needs.create_release.outputs.version }} | |
version: ${{ needs.create_release.outputs.version }} | |
runner_provider: gcp | |
target: all |