No commit message

.circleci/config.yml

@@ -0,0 +1,16 @@
+
+version: 2
+
+jobs:
+  build:
+    machine: true
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build:
+          filters:
+            branches:
+              only:
+                - /.*-circle-.*/

.github/workflows/crossbow.yml

@@ -0,0 +1,317 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# NOTE: must set "Crossbow" as name to have the badge links working in the
+# github comment reports!
+name: Crossbow
+on:
+  push:
+    branches:
+      - "*-github-*"
+
+
+jobs:
+
+  build-cpp-ubuntu:
+    name: Build C++ libraries Ubuntu
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          path: arrow
+          repository: lriggs/arrow
+          ref: b6593b36259c93208e7f3161a7634bdfb5a788dc
+          submodules: recursive
+
+      - name: Set up Python by actions/setup-python
+        if: runner.arch == 'X64'
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: 3.8
+      - name: Set up Python by apt
+        if: runner.os == 'Linux' && runner.arch != 'X64'
+        run: |
+          sudo apt update
+          sudo apt-get install -y python3-pip
+          pip install -U pip
+          echo "$HOME/.local/bin" >>"$GITHUB_PATH"
+      - name: Install Archery
+        shell: bash
+        run: pip install -e arrow/dev/archery[all]
+
+      - name: Free up disk space
+        run: |
+            df -h
+            sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+            df -h
+      - name: Build C++ libraries
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+          SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
+          SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
+          SCCACHE_S3_KEY_PREFIX: "sccache"
+
+        run: |
+          archery docker run \
+            -e ARROW_JAVA_BUILD=OFF \
+            -e ARROW_JAVA_TEST=OFF \
+            java-jni-manylinux-2014
+      - name: Compress into single artifact to keep directory structure
+        run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-shared-lib
+          path: arrow-shared-libs-linux.tar.gz
+
+  build-cpp-macos:
+    name: Build C++ libraries macOS ${{ matrix.platform.arch }}
+    runs-on: ${{ matrix.platform.runs_on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - { runs_on: ["macos-latest"], arch: "x86_64"}
+    env:
+      MACOSX_DEPLOYMENT_TARGET: "10.13"
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          path: arrow
+          repository: lriggs/arrow
+          ref: b6593b36259c93208e7f3161a7634bdfb5a788dc
+          submodules: recursive
+
+      - name: Set up Python
+        if: |
+          !contains(matrix.platform.runs_on, 'self-hosted')
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: 3.8
+      - name: Install Archery
+        shell: bash
+        run: pip install -e arrow/dev/archery[all]
+      - name: Install dependencies
+        run: |
+          # We want to use llvm@14 to avoid shared z3
+          # dependency. llvm@14 doesn't depend on z3 and llvm depends
+          # on z3. And Homebrew's z3 provides only shared library. It
+          # doesn't provides static z3 because z3's CMake doesn't accept
+          # building both shared and static libraries at once.
+          # See also: Z3_BUILD_LIBZ3_SHARED in
+          # https://github.com/Z3Prover/z3/blob/master/README-CMake.md
+          #
+          # If llvm is installed, Apache Arrow C++ uses llvm rather than
+          # llvm@14 because llvm is newer than llvm@14.
+          brew uninstall llvm || :
+
+          brew bundle --file=arrow/cpp/Brewfile
+          # We want to link aws-sdk-cpp statically but Homebrew's
+          # aws-sdk-cpp provides only shared library. If we have
+          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
+          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
+          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
+          brew uninstall aws-sdk-cpp
+          # We want to use bundled RE2 for static linking. If
+          # Homebrew's RE2 is installed, its header file may be used.
+          # We uninstall Homebrew's RE2 to ensure using bundled RE2.
+          brew uninstall grpc || : # gRPC depends on RE2
+          brew uninstall re2 || :
+
+          brew bundle --file=arrow/java/Brewfile
+      - name: Build C++ libraries
+        env:
+          ARROW_USE_CCACHE: "ON"
+        run: |
+          set -e
+          # make brew Java available to CMake
+          if [ "${{ matrix.platform.arch }}" = "aarch_64" ]; then
+            export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
+          fi
+          arrow/ci/scripts/java_jni_macos_build.sh \
+            $GITHUB_WORKSPACE/arrow \
+            $GITHUB_WORKSPACE/arrow/cpp-build \
+            $GITHUB_WORKSPACE/arrow/java-dist
+      - name: Compress into single artifact to keep directory structure
+        run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: macos-shared-lib-${{ matrix.platform.arch }}
+          path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz
+
+  build-cpp-windows:
+    name: Build C++ libraries Windows
+    runs-on: windows-2019
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          path: arrow
+          repository: lriggs/arrow
+          ref: b6593b36259c93208e7f3161a7634bdfb5a788dc
+          submodules: recursive
+
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Download Timezone Database
+        shell: bash
+        run: arrow/ci/scripts/download_tz_database.sh
+      - name: Install sccache
+        shell: bash
+        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
+      - name: Build C++ libraries
+        shell: cmd
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+          SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
+          SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
+          SCCACHE_S3_KEY_PREFIX: "sccache"
+
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          REM For ORC
+          set TZDIR=/c/msys64/usr/share/zoneinfo
+          bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist"
+      - name: Compress into single artifact to keep directory structure
+        shell: bash
+        run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: windows-shared-lib
+          path: arrow-shared-libs-windows.tar.gz
+
+  package-jars:
+    name: Build jar files
+    runs-on: macos-latest
+    needs:
+      - build-cpp-ubuntu
+      - build-cpp-macos
+      - build-cpp-windows
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          path: arrow
+          repository: lriggs/arrow
+          ref: b6593b36259c93208e7f3161a7634bdfb5a788dc
+          submodules: recursive
+
+      - name: Download Libraries
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Decompress artifacts
+        run: |
+          mv artifacts/*/*.tar.gz .
+          tar -xvzf arrow-shared-libs-linux.tar.gz
+          tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
+          tar -xvzf arrow-shared-libs-windows.tar.gz
+      - name: Test that shared libraries exist
+        run: |
+          set -x
+
+          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so
+          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so
+          test -f arrow/java-dist/x86_64/libarrow_orc_jni.so
+          test -f arrow/java-dist/x86_64/libgandiva_jni.so
+
+          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
+
+          test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll
+          test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll
+          test -f arrow/java-dist/x86_64/arrow_orc_jni.dll
+      - name: Build bundled jar
+        run: |
+          set -e
+          pushd arrow/java
+          mvn versions:set -DnewVersion=12.0.4-SNAPSHOT
+          popd
+          arrow/ci/scripts/java_full_build.sh \
+            $GITHUB_WORKSPACE/arrow \
+            $GITHUB_WORKSPACE/arrow/java-dist
+      - name: Set up Python by actions/setup-python
+        if: runner.arch == 'X64'
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Set up Python by apt
+        if: runner.os == 'Linux' && runner.arch != 'X64'
+        run: |
+          sudo apt update
+          sudo apt install -y \
+            libgit2-dev \
+            libpython3-dev \
+            python3-pip
+          sudo python3 -m pip install --upgrade pip
+      - name: Checkout Crossbow
+        uses: actions/checkout@v3
+        with:
+          path: crossbow
+          ref: nightly-123
+      - name: Setup Crossbow
+        shell: bash
+        run: |
+          python3 -m pip install -e arrow/dev/archery[crossbow]
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+      - name: Upload artifacts
+        shell: bash
+        run: |
+          archery crossbow \
+            --queue-path $(pwd)/crossbow \
+            --queue-remote https://github.com/lriggs/arrow-build \
+            upload-artifacts \
+            --sha nightly-123-github-java-jars \
+            --tag nightly-123-github-java-jars \
+            "arrow/java-dist/*.jar" \
+            "arrow/java-dist/*.json" \
+            "arrow/java-dist/*.pom" \
+            "arrow/java-dist/*.xml" \
+            "arrow/java-dist/*.zip" 
+        env:
+          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+      - name: Verify uploaded artifacts
+        shell: bash
+        run: |
+          archery crossbow \
+            --queue-path $(pwd)/crossbow \
+            --queue-remote https://github.com/lriggs/arrow-build \
+            status \
+            --task-filter 'java-jars' \
+            --no-fetch \
+            --validate \
+            nightly-123
+        env:
+          CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}