Merge branch 'apache:master' into master

.asf.yaml

@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Documentation can be found here:
+# https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=127405038
+
 notifications:
   commits:      [email protected]
   issues:       [email protected]
@@ -28,4 +31,11 @@ github:
     merge: false
     rebase: false
   features:
-    issues: true
+    issues: true
+  protected_branches:
+    master:
+      required_status_checks:
+        # require branches to be up-to-date before merging
+        strict: true
+        # don't require any jobs to pass
+        contexts: []

.github/actions/setup-builder/action.yaml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Cache Cargo
+      uses: actions/cache@v3
+      with:
+        # these represent dependencies downloaded by cargo
+        # and thus do not depend on the OS, arch nor rust version.
+        #
+        # source https://github.com/actions/cache/blob/main/examples.md#rust---cargo
+        path: |
+          /usr/local/cargo/bin/
+          /usr/local/cargo/registry/index/
+          /usr/local/cargo/registry/cache/
+          /usr/local/cargo/git/db/
+        key: cargo-cache3-${{ hashFiles('**/Cargo.toml') }}
+        restore-keys: cargo-cache3-
+    - name: Generate lockfile
+      shell: bash
+      run: cargo fetch
+    - name: Cache Rust dependencies
+      uses: actions/cache@v3
+      with:
+        # these represent compiled steps of both dependencies and arrow
+        # and thus are specific for a particular OS, arch and rust version.
+        path: /github/home/target
+        key: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-${{ hashFiles('**/Cargo.lock') }}
+        restore-keys: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-
+    - name: Install Build Dependencies
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install -y protobuf-compiler
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        echo "Installing ${{ inputs.rust-version }}"
+        rustup toolchain install ${{ inputs.rust-version }}
+        rustup default ${{ inputs.rust-version }}
+        echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: master
+    labels: [auto-dependencies]

.github/workflows/integration.yml

@@ -39,7 +39,7 @@ jobs:
           path: rust
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.8
       - name: Setup Archery
@@ -64,17 +64,17 @@ jobs:
           rustup default ${{ matrix.rust }}
           rustup component add rustfmt clippy
       - name: Cache Cargo
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /home/runner/.cargo
           key: cargo-maturin-cache-
       - name: Cache Rust dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /home/runner/target
           # this key is not equal because maturin uses different compilation flags.
           key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}-
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
           python-version: '3.7'
       - name: Upgrade pip and setuptools

.github/workflows/miri.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Script
+#
+# Must be run with nightly rust for example
+# rustup default nightly
+
+
+# stacked borrows checking uses too much memory to run successfully in github actions
+# re-enable if the CI is migrated to something more powerful (https://github.com/apache/arrow-rs/issues/1833)
+# see also https://github.com/rust-lang/miri/issues/1367
+export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows"
+cargo miri setup
+cargo clean
+
+echo "Starting Arrow MIRI run..."
+cargo miri test -p arrow -- --skip csv --skip ipc --skip json

.github/workflows/miri.yaml

@@ -15,46 +15,29 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Rust
+name: MIRI
 
 on:
   # always trigger
   push:
   pull_request:
 
 jobs:
-
   miri-checks:
     name: MIRI
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        arch: [amd64]
-        rust: [nightly-2021-07-04]
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: true
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }}
       - name: Setup Rust toolchain
         run: |
-          rustup toolchain install ${{ matrix.rust }}
-          rustup default ${{ matrix.rust }}
-          rustup component add rustfmt clippy miri
+          rustup toolchain install nightly --component miri
+          rustup override set nightly
+          cargo miri setup
       - name: Run Miri Checks
         env:
           RUST_BACKTRACE: full
-          RUST_LOG: 'trace'
+          RUST_LOG: "trace"
         run: |
-          export MIRIFLAGS="-Zmiri-disable-isolation"
-          cargo miri setup
-          cargo clean
-          # Currently only the arrow crate is tested with miri
-          # IO related tests and some unsupported tests are skipped
-          cargo miri test -p arrow -- --skip csv --skip ipc --skip json
+          bash .github/workflows/miri.sh