From 62db476a2cd165a2b7aefff220405ec45f82e114 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 14:39:53 -0800
Subject: [PATCH 01/15] [CI] Test Mozilla sccache to make compile faster

---
 .github/actions/setup-builder/action.yaml | 13 -------------
 .github/workflows/rust.yml                | 15 ++++++++++++++-
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
index 5a93f6f27b43..3d15c1294aab 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -38,19 +38,6 @@ runs:
         rustup toolchain install ${{ inputs.rust-version }}
         rustup default ${{ inputs.rust-version }}
         rustup component add rustfmt
-    - name: Disable debuginfo generation
-      # Disable full debug symbol generation to speed up CI build and keep memory down
-      # "1" means line tables only, which is useful for panic tracebacks.
-      shell: bash
-      run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
-    - name: Disable incremental compilation
-      # Disable incremental compilation to save diskspace (the CI doesn't recompile modified files)
-      # https://github.com/apache/arrow-datafusion/issues/6676
-      shell: bash
-      run: echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
-    - name: Enable backtraces
-      shell: bash
-      run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766
       shell: bash
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index ae6c1ee56129..2971d2767bf5 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -97,6 +97,8 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.3          
       - name: Run tests (excluding doctests)
         run: cargo test --lib --tests --bins --features avro,json,backtrace
         env:
@@ -107,6 +109,8 @@ jobs:
           RUST_BACKTRACE: "1"
           # avoid rust stack overflows on tpc-ds tests
           RUST_MIN_STACK: "3000000"
+          SCCACHE_GHA_ENABLED: "true"
+          RUSTC_WRAPPER: "sccache"
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -294,6 +298,7 @@ jobs:
           unzip $PROTO_ZIP
           export PATH=$PATH:$HOME/d/protoc/bin
           protoc.exe --version
+
       # TODO: this won't cache anything, which is expensive. Setup this action
       # with a OS-dependent path.
       - name: Setup Rust toolchain
@@ -302,6 +307,8 @@ jobs:
           rustup toolchain install stable
           rustup default stable
           rustup component add rustfmt
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.3          
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
@@ -317,13 +324,15 @@ jobs:
           RUST_BACKTRACE: "1"
           # avoid rust stack overflows on tpc-ds tests
           RUST_MIN_STACK: "3000000"
+          SCCACHE_GHA_ENABLED: "true"
+          RUSTC_WRAPPER: "sccache"
   macos:
     name: cargo test (mac)
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
         with:
-          submodules: true
+          submodules: true      
       - name: Install protobuf compiler
         shell: bash
         run: |
@@ -343,6 +352,8 @@ jobs:
           rustup toolchain install stable
           rustup default stable
           rustup component add rustfmt
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.3          
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
@@ -357,6 +368,8 @@ jobs:
           RUST_BACKTRACE: "1"
           # avoid rust stack overflows on tpc-ds tests
           RUST_MIN_STACK: "3000000"
+          SCCACHE_GHA_ENABLED: "true"
+          RUSTC_WRAPPER: "sccache"
 
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)

From a50243066e9abd6701dd972fd422a1ea95ed8314 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 15:11:08 -0800
Subject: [PATCH 02/15] linux and macos make datafusion cli a sep job

---
 .github/workflows/rust.yml | 58 +++++++++++++++++++++++++++++++++++---
 1 file changed, 54 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 2971d2767bf5..26869b262211 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -314,8 +314,6 @@ jobs:
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
           cargo test --lib --tests --bins --features avro,json,backtrace
-          cd datafusion-cli
-          cargo test --lib --tests --bins --all-features
         env:
           # Minimize producing debug symbols to keep memory usage down
           # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
@@ -326,6 +324,34 @@ jobs:
           RUST_MIN_STACK: "3000000"
           SCCACHE_GHA_ENABLED: "true"
           RUSTC_WRAPPER: "sccache"
+  
+  windows-test-datafusion-cli:
+    name: cargo test datafusion-cli (windows)
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Run tests (excluding doctests)
+        run: |
+          export PATH=$PATH:$HOME/d/protoc/bin
+          cd datafusion-cli
+          cargo test --lib --tests --bins --all-features
+        env:
+          # Minimize producing debug symbols to keep memory usage down
+          # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
+          # See for more details: https://github.com/rust-lang/rust/issues/119560
+          RUSTFLAGS: "-C debuginfo=line-tables-only"
+          RUST_BACKTRACE: "1"
+          SCCACHE_GHA_ENABLED: "true"
+          RUSTC_WRAPPER: "sccache"    
+      - name: Verify Working Directory Clean
+        run: git diff --exit-code
+
   macos:
     name: cargo test (mac)
     runs-on: macos-latest
@@ -358,8 +384,6 @@ jobs:
         shell: bash
         run: |
           cargo test --lib --tests --bins --features avro,json,backtrace
-          cd datafusion-cli
-          cargo test --lib --tests --bins --all-features
         env:
           # do not produce debug symbols to keep memory usage down
           # hardcoding other profile params to avoid profile override values
@@ -371,6 +395,32 @@ jobs:
           SCCACHE_GHA_ENABLED: "true"
           RUSTC_WRAPPER: "sccache"
 
+  macos-test-datafusion-cli:
+    name: cargo test datafusion-cli (macos)
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Run tests (excluding doctests)
+        run: |
+          cd datafusion-cli
+          cargo test --lib --tests --bins --all-features
+        env:
+          # Minimize producing debug symbols to keep memory usage down
+          # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
+          # See for more details: https://github.com/rust-lang/rust/issues/119560
+          RUSTFLAGS: "-C debuginfo=line-tables-only"
+          RUST_BACKTRACE: "1"
+          SCCACHE_GHA_ENABLED: "true"
+          RUSTC_WRAPPER: "sccache"    
+      - name: Verify Working Directory Clean
+        run: git diff --exit-code        
+
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)
     needs: [ linux-build-lib ]

From e4ffe9fed031ee16e979acabe170e91675bc3a0f Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 15:32:45 -0800
Subject: [PATCH 03/15] split conf

---
 .../actions/setup-rust-builder/action.yaml    | 48 +++++++++++++++++
 .github/workflows/rust.yml                    | 51 ++-----------------
 2 files changed, 53 insertions(+), 46 deletions(-)
 create mode 100644 .github/actions/setup-rust-builder/action.yaml

diff --git a/.github/actions/setup-rust-builder/action.yaml b/.github/actions/setup-rust-builder/action.yaml
new file mode 100644
index 000000000000..4d2d3b77734c
--- /dev/null
+++ b/.github/actions/setup-rust-builder/action.yaml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-win64.zip"
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc.exe --version
+
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        rustup update stable
+        rustup toolchain install stable
+        rustup default stable
+        rustup component add rustfmt
+
+    - name: Run sccache-cache
+      uses: mozilla-actions/sccache-action@v0.0.3     
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 26869b262211..54379c02e2c0 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -288,27 +288,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-win64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc.exe --version
-
-      # TODO: this won't cache anything, which is expensive. Setup this action
-      # with a OS-dependent path.
       - name: Setup Rust toolchain
-        run: |
-          rustup update stable
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add rustfmt
-      - name: Run sccache-cache
-        uses: mozilla-actions/sccache-action@v0.0.3          
+        uses: ./.github/actions/setup-windows-builder
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
@@ -333,12 +314,9 @@ jobs:
         with:
           submodules: true
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
+        uses: ./.github/actions/setup-rust-builder
       - name: Run tests (excluding doctests)
         run: |
-          export PATH=$PATH:$HOME/d/protoc/bin
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
         env:
@@ -353,31 +331,14 @@ jobs:
         run: git diff --exit-code
 
   macos:
-    name: cargo test (mac)
+    name: cargo test (macos)
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true      
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc --version
-      # TODO: this won't cache anything, which is expensive. Setup this action
-      # with a OS-dependent path.
       - name: Setup Rust toolchain
-        run: |
-          rustup update stable
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add rustfmt
+        uses: ./.github/actions/setup-rust-builder
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.3          
       - name: Run tests (excluding doctests)
@@ -403,9 +364,7 @@ jobs:
         with:
           submodules: true
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
+        uses: ./.github/actions/setup-rust-builder
       - name: Run tests (excluding doctests)
         run: |
           cd datafusion-cli

From 750691739348d91607a57c643509d903de9971d0 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 16:13:25 -0800
Subject: [PATCH 04/15] fix conf

---
 .../actions/setup-macos-builder/action.yaml   | 48 +++++++++++++++++++
 .../action.yaml                               |  0
 .github/workflows/rust.yml                    |  6 +--
 3 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100644 .github/actions/setup-macos-builder/action.yaml
 rename .github/actions/{setup-rust-builder => setup-windows-builder}/action.yaml (100%)

diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml
new file mode 100644
index 000000000000..6c480bd3ad4d
--- /dev/null
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder
+description: 'Prepare Rust Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"        
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc --version
+
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        rustup update stable
+        rustup toolchain install stable
+        rustup default stable
+        rustup component add rustfmt
+
+    - name: Run sccache-cache
+      uses: mozilla-actions/sccache-action@v0.0.3     
diff --git a/.github/actions/setup-rust-builder/action.yaml b/.github/actions/setup-windows-builder/action.yaml
similarity index 100%
rename from .github/actions/setup-rust-builder/action.yaml
rename to .github/actions/setup-windows-builder/action.yaml
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 54379c02e2c0..175cc15a0566 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -314,7 +314,7 @@ jobs:
         with:
           submodules: true
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-rust-builder
+        uses: ./.github/actions/setup-windows-builder
       - name: Run tests (excluding doctests)
         run: |
           cd datafusion-cli
@@ -338,7 +338,7 @@ jobs:
         with:
           submodules: true      
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-rust-builder
+        uses: ./.github/actions/setup-macos-builder
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.3          
       - name: Run tests (excluding doctests)
@@ -364,7 +364,7 @@ jobs:
         with:
           submodules: true
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-rust-builder
+        uses: ./.github/actions/setup-macos-builder
       - name: Run tests (excluding doctests)
         run: |
           cd datafusion-cli

From 20ca32a67698698e41db6edfb086b0203ed59425 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 16:27:37 -0800
Subject: [PATCH 05/15] fix macos

---
 .github/actions/setup-macos-builder/action.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml
index 6c480bd3ad4d..06f3debc159c 100644
--- a/.github/actions/setup-macos-builder/action.yaml
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -33,6 +33,7 @@ runs:
         export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"        
         curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
         unzip $PROTO_ZIP
+        echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
         export PATH=$PATH:$HOME/d/protoc/bin
         protoc --version
 

From 1620e12867ac8ff1b8e640da77d3a34840ff06d4 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 17:05:47 -0800
Subject: [PATCH 06/15] reformat runtime

---
 .github/actions/setup-builder/action.yaml     |  4 ++
 .../actions/setup-macos-builder/action.yaml   |  6 +--
 .../actions/setup-rust-runtime/action.yaml    | 39 +++++++++++++++
 .../actions/setup-windows-builder/action.yaml |  6 +--
 .github/workflows/rust.yml                    | 50 +------------------
 5 files changed, 50 insertions(+), 55 deletions(-)
 create mode 100644 .github/actions/setup-rust-runtime/action.yaml

diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
index 3d15c1294aab..824b643658ac 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -38,6 +38,10 @@ runs:
         rustup toolchain install ${{ inputs.rust-version }}
         rustup default ${{ inputs.rust-version }}
         rustup component add rustfmt
+    - name: Run sccache-cache
+      uses: mozilla-actions/sccache-action@v0.0.3
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766
       shell: bash
diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml
index 06f3debc159c..549e3a193c86 100644
--- a/.github/actions/setup-macos-builder/action.yaml
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -36,7 +36,6 @@ runs:
         echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
         export PATH=$PATH:$HOME/d/protoc/bin
         protoc --version
-
     - name: Setup Rust toolchain
       shell: bash
       run: |
@@ -44,6 +43,7 @@ runs:
         rustup toolchain install stable
         rustup default stable
         rustup component add rustfmt
-
     - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.3     
+      uses: mozilla-actions/sccache-action@v0.0.3   
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        
diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml
new file mode 100644
index 000000000000..9f1cf9e78173
--- /dev/null
+++ b/.github/actions/setup-rust-runtime/action.yaml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Rust Runtime
+description: 'Setup Rust Runtime Environment'
+runs:
+  using: "composite"
+  steps:
+    - name: Configure runtime env
+      shell: bash
+      # do not produce debug symbols to keep memory usage down
+      # hardcoding other profile params to avoid profile override values
+      # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
+      # 
+      # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
+      # See for more details: https://github.com/rust-lang/rust/issues/119560
+      #
+      # set RUST_MIN_STACK to avoid rust stack overflows on tpc-ds tests
+      run: |
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV  
+        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
+        echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+        echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
+        echo "RUST_FLAGS=-C debuginfo=line-tables-only -C opt-level=0 -C incremental=false -C codegen-units=256" >> $GITHUB_ENV                                   
+     
diff --git a/.github/actions/setup-windows-builder/action.yaml b/.github/actions/setup-windows-builder/action.yaml
index 4d2d3b77734c..3c5acdccd43f 100644
--- a/.github/actions/setup-windows-builder/action.yaml
+++ b/.github/actions/setup-windows-builder/action.yaml
@@ -35,7 +35,6 @@ runs:
         unzip $PROTO_ZIP
         export PATH=$PATH:$HOME/d/protoc/bin
         protoc.exe --version
-
     - name: Setup Rust toolchain
       shell: bash
       run: |
@@ -43,6 +42,7 @@ runs:
         rustup toolchain install stable
         rustup default stable
         rustup component add rustfmt
-
     - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.3     
+      uses: mozilla-actions/sccache-action@v0.0.3   
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 175cc15a0566..c5dadacc2a8e 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -96,21 +96,9 @@ jobs:
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: stable
-      - name: Run sccache-cache
-        uses: mozilla-actions/sccache-action@v0.0.3          
+          rust-version: stable        
       - name: Run tests (excluding doctests)
         run: cargo test --lib --tests --bins --features avro,json,backtrace
-        env:
-          # do not produce debug symbols to keep memory usage down
-          # hardcoding other profile params to avoid profile override values
-          # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
-          RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256"
-          RUST_BACKTRACE: "1"
-          # avoid rust stack overflows on tpc-ds tests
-          RUST_MIN_STACK: "3000000"
-          SCCACHE_GHA_ENABLED: "true"
-          RUSTC_WRAPPER: "sccache"
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -295,16 +283,6 @@ jobs:
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
           cargo test --lib --tests --bins --features avro,json,backtrace
-        env:
-          # Minimize producing debug symbols to keep memory usage down
-          # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
-          # See for more details: https://github.com/rust-lang/rust/issues/119560
-          RUSTFLAGS: "-C debuginfo=line-tables-only"
-          RUST_BACKTRACE: "1"
-          # avoid rust stack overflows on tpc-ds tests
-          RUST_MIN_STACK: "3000000"
-          SCCACHE_GHA_ENABLED: "true"
-          RUSTC_WRAPPER: "sccache"
   
   windows-test-datafusion-cli:
     name: cargo test datafusion-cli (windows)
@@ -319,14 +297,6 @@ jobs:
         run: |
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
-        env:
-          # Minimize producing debug symbols to keep memory usage down
-          # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
-          # See for more details: https://github.com/rust-lang/rust/issues/119560
-          RUSTFLAGS: "-C debuginfo=line-tables-only"
-          RUST_BACKTRACE: "1"
-          SCCACHE_GHA_ENABLED: "true"
-          RUSTC_WRAPPER: "sccache"    
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -345,16 +315,6 @@ jobs:
         shell: bash
         run: |
           cargo test --lib --tests --bins --features avro,json,backtrace
-        env:
-          # do not produce debug symbols to keep memory usage down
-          # hardcoding other profile params to avoid profile override values
-          # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
-          RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256"
-          RUST_BACKTRACE: "1"
-          # avoid rust stack overflows on tpc-ds tests
-          RUST_MIN_STACK: "3000000"
-          SCCACHE_GHA_ENABLED: "true"
-          RUSTC_WRAPPER: "sccache"
 
   macos-test-datafusion-cli:
     name: cargo test datafusion-cli (macos)
@@ -369,14 +329,6 @@ jobs:
         run: |
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
-        env:
-          # Minimize producing debug symbols to keep memory usage down
-          # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
-          # See for more details: https://github.com/rust-lang/rust/issues/119560
-          RUSTFLAGS: "-C debuginfo=line-tables-only"
-          RUST_BACKTRACE: "1"
-          SCCACHE_GHA_ENABLED: "true"
-          RUSTC_WRAPPER: "sccache"    
       - name: Verify Working Directory Clean
         run: git diff --exit-code        
 

From 4e1975a26560202e7dd554497728ebef9e6de182 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 17:53:32 -0800
Subject: [PATCH 07/15] try win container

---
 .github/workflows/rust.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index c5dadacc2a8e..47f15ea694e6 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -272,6 +272,8 @@ jobs:
   windows:
     name: cargo test (win64)
     runs-on: windows-latest
+    container:
+      image: mcr.microsoft.com/windows
     steps:
       - uses: actions/checkout@v4
         with:
@@ -287,6 +289,8 @@ jobs:
   windows-test-datafusion-cli:
     name: cargo test datafusion-cli (windows)
     runs-on: windows-latest
+    container:
+      image: mcr.microsoft.com/windows
     steps:
       - uses: actions/checkout@v4
         with:

From 83963f47ffcae08ee4c5e0e413c0492a8fadee34 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 17:56:30 -0800
Subject: [PATCH 08/15] win

---
 .github/workflows/rust.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 47f15ea694e6..573f6c7b687d 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -273,7 +273,7 @@ jobs:
     name: cargo test (win64)
     runs-on: windows-latest
     container:
-      image: mcr.microsoft.com/windows
+      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:
@@ -290,7 +290,7 @@ jobs:
     name: cargo test datafusion-cli (windows)
     runs-on: windows-latest
     container:
-      image: mcr.microsoft.com/windows
+      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:

From d9d8e364b2c6c63cc4b3536f29862a6b0d5202a4 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 17:59:58 -0800
Subject: [PATCH 09/15] refactor

---
 .github/workflows/rust.yml | 38 ++------------------------------------
 1 file changed, 2 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 573f6c7b687d..62992e7acf68 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -272,8 +272,6 @@ jobs:
   windows:
     name: cargo test (win64)
     runs-on: windows-latest
-    container:
-      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:
@@ -285,24 +283,8 @@ jobs:
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
           cargo test --lib --tests --bins --features avro,json,backtrace
-  
-  windows-test-datafusion-cli:
-    name: cargo test datafusion-cli (windows)
-    runs-on: windows-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-windows-builder
-      - name: Run tests (excluding doctests)
-        run: |
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
-      - name: Verify Working Directory Clean
-        run: git diff --exit-code
 
   macos:
     name: cargo test (macos)
@@ -312,29 +294,13 @@ jobs:
         with:
           submodules: true      
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-macos-builder
-      - name: Run sccache-cache
-        uses: mozilla-actions/sccache-action@v0.0.3          
+        uses: ./.github/actions/setup-macos-builder       
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
           cargo test --lib --tests --bins --features avro,json,backtrace
-
-  macos-test-datafusion-cli:
-    name: cargo test datafusion-cli (macos)
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-macos-builder
-      - name: Run tests (excluding doctests)
-        run: |
           cd datafusion-cli
-          cargo test --lib --tests --bins --all-features
-      - name: Verify Working Directory Clean
-        run: git diff --exit-code        
+          cargo test --lib --tests --bins --all-features  
 
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)

From fd2edb27164e045eac0548891b17c548797d4f45 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 18:02:57 -0800
Subject: [PATCH 10/15] reformat

---
 .github/actions/setup-builder/action.yaml         | 2 --
 .github/actions/setup-macos-builder/action.yaml   | 2 --
 .github/actions/setup-rust-runtime/action.yaml    | 2 ++
 .github/actions/setup-windows-builder/action.yaml | 2 --
 4 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
index 824b643658ac..5578517ec359 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -38,8 +38,6 @@ runs:
         rustup toolchain install ${{ inputs.rust-version }}
         rustup default ${{ inputs.rust-version }}
         rustup component add rustfmt
-    - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.3
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime
     - name: Fixup git permissions
diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml
index 549e3a193c86..5f368aa0ee08 100644
--- a/.github/actions/setup-macos-builder/action.yaml
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -43,7 +43,5 @@ runs:
         rustup toolchain install stable
         rustup default stable
         rustup component add rustfmt
-    - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.3   
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime        
diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml
index 9f1cf9e78173..9a2ba2e5c961 100644
--- a/.github/actions/setup-rust-runtime/action.yaml
+++ b/.github/actions/setup-rust-runtime/action.yaml
@@ -20,6 +20,8 @@ description: 'Setup Rust Runtime Environment'
 runs:
   using: "composite"
   steps:
+    - name: Run sccache-cache
+      uses: mozilla-actions/sccache-action@v0.0.3   
     - name: Configure runtime env
       shell: bash
       # do not produce debug symbols to keep memory usage down
diff --git a/.github/actions/setup-windows-builder/action.yaml b/.github/actions/setup-windows-builder/action.yaml
index 3c5acdccd43f..00fce185644f 100644
--- a/.github/actions/setup-windows-builder/action.yaml
+++ b/.github/actions/setup-windows-builder/action.yaml
@@ -42,7 +42,5 @@ runs:
         rustup toolchain install stable
         rustup default stable
         rustup component add rustfmt
-    - name: Run sccache-cache
-      uses: mozilla-actions/sccache-action@v0.0.3   
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime        

From ba37bd7f29296b4cfc658bdb4367fb9406f1579e Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 18:53:28 -0800
Subject: [PATCH 11/15] disable backtrace for win

---
 .github/workflows/rust.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 62992e7acf68..7b42f6692437 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -282,7 +282,7 @@ jobs:
         shell: bash
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --lib --tests --bins --features avro,json,backtrace
+          cargo test --lib --tests --bins --features avro,json
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
 

From 361d4558fdd7663ed511f1000096251f70d21520 Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Wed, 10 Jan 2024 20:47:21 -0800
Subject: [PATCH 12/15] reformat

---
 .github/actions/setup-rust-runtime/action.yaml | 2 +-
 .github/workflows/rust.yml                     | 2 +-
 datafusion/common/src/error.rs                 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml
index 9a2ba2e5c961..90e09a957cd4 100644
--- a/.github/actions/setup-rust-runtime/action.yaml
+++ b/.github/actions/setup-rust-runtime/action.yaml
@@ -37,5 +37,5 @@ runs:
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
         echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
-        echo "RUST_FLAGS=-C debuginfo=line-tables-only -C opt-level=0 -C incremental=false -C codegen-units=256" >> $GITHUB_ENV                                   
+        echo "RUST_FLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV                                   
      
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 7b42f6692437..62992e7acf68 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -282,7 +282,7 @@ jobs:
         shell: bash
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --lib --tests --bins --features avro,json
+          cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
 
diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rs
index 331f5910d7e5..d1d1ec9f2e2f 100644
--- a/datafusion/common/src/error.rs
+++ b/datafusion/common/src/error.rs
@@ -435,7 +435,7 @@ impl DataFusionError {
     /// Example:
     /// cargo build --features 'backtrace'
     /// RUST_BACKTRACE=1 ./app
-    #[inline(always)]
+    #[inline(never)]
     pub fn get_back_trace() -> String {
         #[cfg(feature = "backtrace")]
         {

From 80a5809a54075da6cdb0324894e83d148e2ebadd Mon Sep 17 00:00:00 2001
From: comphead <comphead@ukr.net>
Date: Thu, 11 Jan 2024 08:36:21 -0800
Subject: [PATCH 13/15] revert

---
 datafusion/common/src/error.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rs
index d1d1ec9f2e2f..331f5910d7e5 100644
--- a/datafusion/common/src/error.rs
+++ b/datafusion/common/src/error.rs
@@ -435,7 +435,7 @@ impl DataFusionError {
     /// Example:
     /// cargo build --features 'backtrace'
     /// RUST_BACKTRACE=1 ./app
-    #[inline(never)]
+    #[inline(always)]
     pub fn get_back_trace() -> String {
         #[cfg(feature = "backtrace")]
         {

From 299aae7d8a7a17bea6f4b2ecafd0f20a6ec4ff1e Mon Sep 17 00:00:00 2001
From: comphead <comphead@users.noreply.github.com>
Date: Thu, 11 Jan 2024 12:07:00 -0800
Subject: [PATCH 14/15] Update .github/actions/setup-macos-builder/action.yaml

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
---
 .github/actions/setup-macos-builder/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml
index 5f368aa0ee08..02419f617942 100644
--- a/.github/actions/setup-macos-builder/action.yaml
+++ b/.github/actions/setup-macos-builder/action.yaml
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Prepare Rust Builder
-description: 'Prepare Rust Build Environment'
+name: Prepare Rust Builder for MacOS
+description: 'Prepare Rust Build Environment for MacOS'
 inputs:
   rust-version:
     description: 'version of rust to install (e.g. stable)'

From 94a5b5055b47aae2de2b5d142aef8c27c633231d Mon Sep 17 00:00:00 2001
From: comphead <comphead@users.noreply.github.com>
Date: Thu, 11 Jan 2024 12:07:07 -0800
Subject: [PATCH 15/15] Update
 .github/actions/setup-windows-builder/action.yaml

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
---
 .github/actions/setup-windows-builder/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/setup-windows-builder/action.yaml b/.github/actions/setup-windows-builder/action.yaml
index 00fce185644f..9ab5c4a8b1bb 100644
--- a/.github/actions/setup-windows-builder/action.yaml
+++ b/.github/actions/setup-windows-builder/action.yaml
@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Prepare Rust Builder
-description: 'Prepare Rust Build Environment'
+name: Prepare Rust Builder for Windows
+description: 'Prepare Rust Build Environment for Windows'
 inputs:
   rust-version:
     description: 'version of rust to install (e.g. stable)'