From d9a1d4261f78dfd3ead235d6b850c64e0ae9bec6 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 11 Jan 2024 13:16:31 -0800 Subject: [PATCH] [CI] Refactor CI builders (#8826) [CI] refactor CI builders * Update .github/actions/setup-macos-builder/action.yaml Co-authored-by: Andrew Lamb * Update .github/actions/setup-windows-builder/action.yaml Co-authored-by: Andrew Lamb --------- Co-authored-by: Andrew Lamb --- .github/actions/setup-builder/action.yaml | 15 +--- .../actions/setup-macos-builder/action.yaml | 47 +++++++++++++ .../actions/setup-rust-runtime/action.yaml | 41 +++++++++++ .../actions/setup-windows-builder/action.yaml | 46 ++++++++++++ .github/workflows/rust.yml | 70 ++----------------- 5 files changed, 143 insertions(+), 76 deletions(-) create mode 100644 .github/actions/setup-macos-builder/action.yaml create mode 100644 .github/actions/setup-rust-runtime/action.yaml create mode 100644 .github/actions/setup-windows-builder/action.yaml diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 5a93f6f27b43..5578517ec359 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -38,19 +38,8 @@ runs: rustup toolchain install ${{ inputs.rust-version }} rustup default ${{ inputs.rust-version }} rustup component add rustfmt - - name: Disable debuginfo generation - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - shell: bash - run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV - - name: Disable incremental compilation - # Disable incremental compilation to save diskspace (the CI doesn't recompile modified files) - # https://github.com/apache/arrow-datafusion/issues/6676 - shell: bash - run: echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV - - name: Enable backtraces - shell: bash - run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + - name: Configure rust runtime env + uses: ./.github/actions/setup-rust-runtime - name: Fixup git permissions # https://github.com/actions/checkout/issues/766 shell: bash diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml new file mode 100644 index 000000000000..02419f617942 --- /dev/null +++ b/.github/actions/setup-macos-builder/action.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Rust Builder for MacOS +description: 'Prepare Rust Build Environment for MacOS' +inputs: + rust-version: + description: 'version of rust to install (e.g. stable)' + required: true + default: 'stable' +runs: + using: "composite" + steps: + - name: Install protobuf compiler + shell: bash + run: | + mkdir -p $HOME/d/protoc + cd $HOME/d/protoc + export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + unzip $PROTO_ZIP + echo "$HOME/d/protoc/bin" >> $GITHUB_PATH + export PATH=$PATH:$HOME/d/protoc/bin + protoc --version + - name: Setup Rust toolchain + shell: bash + run: | + rustup update stable + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Configure rust runtime env + uses: ./.github/actions/setup-rust-runtime diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml new file mode 100644 index 000000000000..90e09a957cd4 --- /dev/null +++ b/.github/actions/setup-rust-runtime/action.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Setup Rust Runtime +description: 'Setup Rust Runtime Environment' +runs: + using: "composite" + steps: + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: Configure runtime env + shell: bash + # do not produce debug symbols to keep memory usage down + # hardcoding other profile params to avoid profile override values + # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings + # + # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build + # See for more details: https://github.com/rust-lang/rust/issues/119560 + # + # set RUST_MIN_STACK to avoid rust stack overflows on tpc-ds tests + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV + echo "RUST_FLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV + diff --git a/.github/actions/setup-windows-builder/action.yaml b/.github/actions/setup-windows-builder/action.yaml new file mode 100644 index 000000000000..9ab5c4a8b1bb --- /dev/null +++ b/.github/actions/setup-windows-builder/action.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Rust Builder for Windows +description: 'Prepare Rust Build Environment for Windows' +inputs: + rust-version: + description: 'version of rust to install (e.g. stable)' + required: true + default: 'stable' +runs: + using: "composite" + steps: + - name: Install protobuf compiler + shell: bash + run: | + mkdir -p $HOME/d/protoc + cd $HOME/d/protoc + export PROTO_ZIP="protoc-21.4-win64.zip" + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP + unzip $PROTO_ZIP + export PATH=$PATH:$HOME/d/protoc/bin + protoc.exe --version + - name: Setup Rust toolchain + shell: bash + run: | + rustup update stable + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Configure rust runtime env + uses: ./.github/actions/setup-rust-runtime diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ae6c1ee56129..62992e7acf68 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -96,17 +96,9 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: - rust-version: stable + rust-version: stable - name: Run tests (excluding doctests) run: cargo test --lib --tests --bins --features avro,json,backtrace - env: - # do not produce debug symbols to keep memory usage down - # hardcoding other profile params to avoid profile override values - # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings - RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256" - RUST_BACKTRACE: "1" - # avoid rust stack overflows on tpc-ds tests - RUST_MIN_STACK: "3000000" - name: Verify Working Directory Clean run: git diff --exit-code @@ -284,24 +276,8 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Install protobuf compiler - shell: bash - run: | - mkdir -p $HOME/d/protoc - cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-win64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - unzip $PROTO_ZIP - export PATH=$PATH:$HOME/d/protoc/bin - protoc.exe --version - # TODO: this won't cache anything, which is expensive. Setup this action - # with a OS-dependent path. - name: Setup Rust toolchain - run: | - rustup update stable - rustup toolchain install stable - rustup default stable - rustup component add rustfmt + uses: ./.github/actions/setup-windows-builder - name: Run tests (excluding doctests) shell: bash run: | @@ -309,54 +285,22 @@ jobs: cargo test --lib --tests --bins --features avro,json,backtrace cd datafusion-cli cargo test --lib --tests --bins --all-features - env: - # Minimize producing debug symbols to keep memory usage down - # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build - # See for more details: https://github.com/rust-lang/rust/issues/119560 - RUSTFLAGS: "-C debuginfo=line-tables-only" - RUST_BACKTRACE: "1" - # avoid rust stack overflows on tpc-ds tests - RUST_MIN_STACK: "3000000" + macos: - name: cargo test (mac) + name: cargo test (macos) runs-on: macos-latest steps: - uses: actions/checkout@v4 with: - submodules: true - - name: Install protobuf compiler - shell: bash - run: | - mkdir -p $HOME/d/protoc - cd $HOME/d/protoc - export PROTO_ZIP="protoc-21.4-osx-x86_64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP - unzip $PROTO_ZIP - echo "$HOME/d/protoc/bin" >> $GITHUB_PATH - export PATH=$PATH:$HOME/d/protoc/bin - protoc --version - # TODO: this won't cache anything, which is expensive. Setup this action - # with a OS-dependent path. + submodules: true - name: Setup Rust toolchain - run: | - rustup update stable - rustup toolchain install stable - rustup default stable - rustup component add rustfmt + uses: ./.github/actions/setup-macos-builder - name: Run tests (excluding doctests) shell: bash run: | cargo test --lib --tests --bins --features avro,json,backtrace cd datafusion-cli - cargo test --lib --tests --bins --all-features - env: - # do not produce debug symbols to keep memory usage down - # hardcoding other profile params to avoid profile override values - # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings - RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256" - RUST_BACKTRACE: "1" - # avoid rust stack overflows on tpc-ds tests - RUST_MIN_STACK: "3000000" + cargo test --lib --tests --bins --all-features test-datafusion-pyarrow: name: cargo test pyarrow (amd64)