rust.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Rust

on:
  # always trigger
  push:
  pull_request:

jobs:
  # build the library, a compilation step used by multiple steps below
  linux-build-lib:
    name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          # these represent dependencies downloaded by cargo
          # and thus do not depend on the OS, arch nor rust version.
          path: /github/home/.cargo
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          # these represent compiled steps of both dependencies and arrow
          # and thus are specific for a particular OS, arch and rust version.
          path: /github/home/target
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}-
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Build workspace in debug mode
        run: |
          cargo build
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target/debug"
      - name: Build workspace in release mode
        run: |
          cargo check --release
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target/release"
      - name: Check DataFusion builds without default features
        run: |
          cargo check --no-default-features -p datafusion
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"
      - name: Check Workspace builds with all features
        run: |
          cargo check --workspace --benches --features avro,jit,scheduler
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  # test the crate
  linux-test:
    name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Run tests
        run: |
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cargo test --features avro,jit,scheduler
          # test datafusion-sql examples
          cargo run --example sql
          # test datafusion examples
          cd datafusion-examples
          cargo run --example csv_sql
          cargo run --example parquet_sql
          cargo run --example avro_sql --features=datafusion/avro
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  integration-test:
    name: "Integration Test"
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:14
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: db_test
        ports:
          - 5432/tcp
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Setup toolchain
        run: |
          rustup toolchain install stable
          rustup default stable
      - uses: actions/setup-python@v3
        with:
          python-version: "3.10"
      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install -r integration-tests/requirements.txt
      - name: Allow access of psql
        run: |
          # make sure psql can access the server
          echo "$POSTGRES_HOST:$POSTGRES_PORT:$POSTGRES_DB:$POSTGRES_USER:$POSTGRES_PASSWORD" | tee ~/.pgpass
          chmod 0600 ~/.pgpass
          psql -d "$POSTGRES_DB" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -f integration-tests/create_test_table_postgres.sql
          psql -d "$POSTGRES_DB" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -c "\copy test FROM '$(pwd)/testing/data/csv/aggregate_test_100.csv' WITH (FORMAT csv, HEADER true);"
        env:
          POSTGRES_HOST: localhost
          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
          POSTGRES_DB: db_test
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
      - name: Build datafusion-cli
        run: (cd datafusion-cli && cargo build)
      - name: Test Psql Parity
        run: python -m pytest -v integration-tests/test_psql_parity.py
        env:
          POSTGRES_HOST: localhost
          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
          POSTGRES_DB: db_test
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres

  windows-and-macos:
    name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, macos-latest]
        rust: [stable]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      # TODO: this won't cache anything, which is expensive. Setup this action
      # with a OS-dependent path.
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run tests
        shell: bash
        run: |
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cargo test
        env:
          # do not produce debug symbols to keep memory usage down
          RUSTFLAGS: "-C debuginfo=0"

  test-datafusion-pyarrow:
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
      - uses: actions/setup-python@v3
        with:
          python-version: "3.8"
      - name: Install PyArrow
        run: |
          echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
          python -m pip install pyarrow
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Run tests
        run: |
          cd datafusion
          cargo test --features=pyarrow
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  lint:
    name: Lint
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v2
      - name: Setup toolchain
        run: |
          rustup toolchain install stable
          rustup default stable
          rustup component add rustfmt
      - name: Run
        run: ci/scripts/rust_fmt.sh
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  coverage:
    name: Coverage
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [ amd64 ]
        rust: [ stable ]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt clippy
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          path: /home/runner/.cargo
          # this key is not equal because the user is different than on a container (runner vs github)
          key: cargo-coverage-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          path: /home/runner/target
          # this key is not equal because coverage uses different compilation flags.
          key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache3-${{ matrix.rust }}-
      - name: Run coverage
        run: |
          export CARGO_HOME="/home/runner/.cargo"
          export CARGO_TARGET_DIR="/home/runner/target"

          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data

          rustup toolchain install stable
          rustup default stable
          cargo install --version 0.20.1 cargo-tarpaulin
          cargo tarpaulin --all --out Xml
      - name: Report coverage
        continue-on-error: true
        run: bash <(curl -s https://codecov.io/bash)

  clippy:
    name: Clippy
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Install Clippy
        run: |
          rustup component add clippy
      - name: Run clippy
        run: ci/scripts/rust_clippy.sh
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  miri-checks:
    name: MIRI
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2022-01-17]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt clippy miri
      - name: Run Miri Checks
        env:
          RUST_BACKTRACE: full
          RUST_LOG: "trace"
          MIRIFLAGS: "-Zmiri-disable-isolation"
        run: |
          cargo miri setup
          cargo clean
          # Ignore MIRI errors until we can get a clean run
          cargo miri test || true

  # Check answers are correct when hash values collide
  hash-collisions:
    name: Test Hash Collisions on AMD64 Rust ${{ matrix.rust }}
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Run tests
        run: |
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cd datafusion
          # Force all hash values to collide
          cargo test --all --features=force_hash_collisions
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  cargo-toml-formatting-checks:
    name: Check Cargo.toml formatting
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v3
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v3
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Install cargo-tomlfmt
        run: |
          which cargo-tomlfmt || cargo install cargo-tomlfmt
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"
      - name: Check Cargo.toml formatting
        run: |
          # if you encounter error, try rerun the command below, finally run 'git diff' to
          # check which Cargo.toml introduces formatting violation
          #
          # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read
          ci/scripts/rust_toml_fmt.sh
          git diff --exit-code
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"