integration-parquet.yml

name: Integration Parquet

on: [push, pull_request]

jobs:
  docker:
    name: Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install stable
          rustup default stable
          rustup component add rustfmt clippy
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /home/runner/.cargo
          key: cargo-parquet-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /home/runner/target
          key: ${{ runner.os }}-amd64-target-parquet-cache
      - uses: actions/setup-python@v2
        with:
          python-version: "3.10"
      - name: Build
        run: |
          export CARGO_HOME="/home/runner/.cargo"
          export CARGO_TARGET_DIR="/home/runner/target"

          cd arrow-parquet-integration-testing

          cargo build
      - name: Run
        run: |
          export CARGO_HOME="/home/runner/.cargo"
          export CARGO_TARGET_DIR="/home/runner/target"

          cd arrow-parquet-integration-testing

          python -m venv venv
          source venv/bin/activate
          pip install --upgrade pip
          pip install pyarrow==8 pyspark==3
          python main.py
          # test against spark
          python main_spark.py