Dekaf materialization endpoint support #4978
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Controls when the action will run. Triggers the workflow on push | |
| # or pull request events, but only for the primary branch. | |
| on: | |
| push: | |
| branches: [master] | |
| paths-ignore: | |
| - 'site/**' | |
| pull_request: | |
| branches: [master] | |
| paths-ignore: | |
| - 'site/**' | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_CACHE_VERSION: 0.8.2 | |
| RUSTFLAGS: "-C link-arg=-fuse-ld=lld" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| gnu-binaries: | |
| runs-on: ubuntu-2404-large | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| lfs: true | |
| - uses: actions/setup-go@v2 | |
| with: | |
| go-version: "1.21" | |
| - uses: dtolnay/rust-toolchain@stable | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Google Cloud Auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| - name: Set up Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| - uses: supabase/setup-cli@v1 | |
| with: | |
| version: latest | |
| - run: supabase start | |
| - name: SQL tests | |
| run: ./supabase/run_sql_tests.sh | |
| - run: make extra-ci-runner-setup | |
| - run: make print-versions | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: "2404" | |
| - name: Cache/Restore Go dependencies. | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} | |
| - run: make install-tools | |
| - run: go mod download | |
| - run: make rust-gnu-test | |
| - run: make linux-gnu-binaries | |
| - name: Ensure that we didn't dirty the tree. | |
| run: git clean --force -d && git diff --exit-code || exit 1 | |
| - name: Upload Binaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux-gnu-binaries | |
| path: | | |
| .build/package/bin/* | |
| target/x86_64-unknown-linux-gnu/release/libbindings.a | |
| crates/bindings/flow_bindings.h | |
| target/x86_64-unknown-linux-gnu/release/librocksdb-exp/ | |
| musl-binaries: | |
| runs-on: ubuntu-2404-large | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: x86_64-unknown-linux-musl | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: make extra-ci-runner-setup | |
| - run: make print-versions | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: "2404" | |
| - run: go mod download | |
| - run: make rust-musl-test | |
| - run: make linux-musl-binaries | |
| - name: Upload Binaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux-musl-binaries | |
| path: | | |
| .build/package/bin/* | |
| assembly: | |
| runs-on: ubuntu-2404-large | |
| needs: [gnu-binaries, musl-binaries] | |
| services: | |
| postgres: | |
| image: postgres:latest | |
| env: | |
| POSTGRES_DB: flow | |
| POSTGRES_USER: flow | |
| POSTGRES_PASSWORD: flow | |
| ports: | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-go@v2 | |
| with: | |
| go-version: "1.23" | |
| - name: Cache/Restore Go dependencies. | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} | |
| - run: make extra-ci-runner-setup | |
| - run: make print-versions | |
| - name: Google Cloud Auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| - name: Set up Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| - name: Login to GitHub container registry | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | \ | |
| docker login --username ${{ github.actor }} --password-stdin ghcr.io | |
| - name: Download gnu-linux-binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linux-gnu-binaries | |
| path: linux-gnu-binaries | |
| - name: Move linux-gnu-binaries to PKGBIN | |
| run: | | |
| mkdir -p .build/package/bin && \ | |
| mv linux-gnu-binaries/.build/package/bin/* .build/package/bin && \ | |
| mkdir -p target/x86_64-unknown-linux-gnu/release && \ | |
| mv -f linux-gnu-binaries/target/x86_64-unknown-linux-gnu/release/* target/x86_64-unknown-linux-gnu/release/ && \ | |
| mv -f linux-gnu-binaries/crates/bindings/flow_bindings.h crates/bindings/ | |
| - name: Download linux-musl-binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linux-musl-binaries | |
| path: linux-musl-binaries | |
| - name: Move linux-musl-binaries to PKGBIN | |
| run: | | |
| mkdir -p .build/package/bin && \ | |
| mv linux-musl-binaries/* .build/package/bin | |
| # Downloaded artifacts lose their prior permission settings | |
| - name: Adjust binaries permissions | |
| run: | | |
| chmod +x .build/package/bin/* | |
| - name: Add PKGDIR/bin to PATH | |
| run: echo "$GITHUB_WORKSPACE/.build/package/bin" >> $GITHUB_PATH | |
| - name: make catalog-test | |
| run: | | |
| make catalog-test SKIP_BUILD=true | |
| - name: make go-test-ci | |
| run: | | |
| make go-test-ci SKIP_BUILD=true | |
| - name: make end-to-end-test | |
| run: | | |
| make end-to-end-test SKIP_BUILD=true | |
| - name: make package (tar only) | |
| run: | | |
| make /home/runner/work/flow/flow/.build/package/flow-x86-linux.tar.gz | |
| - name: Ensure that we didn't dirty the tree. | |
| run: git clean --force -d && git diff --exit-code || exit 1 | |
| - run: make docker-image | |
| - run: make docker-push | |
| - if: ${{ github.ref == 'refs/heads/master' }} | |
| run: make docker-push-dev | |
| - if: ${{ github.ref == 'refs/heads/master' }} | |
| name: Upload release assets | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| body: | | |
| This is the latest development build, which gets updated on every successful build of the master branch. | |
| target_commitish: ${{ github.sha }} | |
| draft: false | |
| prerelease: true | |
| tag_name: dev | |
| generate_release_notes: false | |
| files: .build/package/flow-x86-linux.tar.gz |