From 47dca2119401dddec1bcdc82f89eda423d5d7064 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 17 Nov 2023 08:09:04 +0000 Subject: [PATCH] --- .circleci/config.yml | 16 ++ .github/workflows/crossbow.yml | 317 ++++++++++++++++++++++++++++ .github/workflows/generate_jar.yaml | 61 ------ .travis.yml | 9 + 4 files changed, 342 insertions(+), 61 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .github/workflows/crossbow.yml delete mode 100644 .github/workflows/generate_jar.yaml create mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..6c155ea9 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,16 @@ + +version: 2 + +jobs: + build: + machine: true + +workflows: + version: 2 + build: + jobs: + - build: + filters: + branches: + only: + - /.*-circle-.*/ diff --git a/.github/workflows/crossbow.yml b/.github/workflows/crossbow.yml new file mode 100644 index 00000000..0061aada --- /dev/null +++ b/.github/workflows/crossbow.yml @@ -0,0 +1,317 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow +on: + push: + branches: + - "*-github-*" + + +jobs: + + build-cpp-ubuntu: + name: Build C++ libraries Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 1 + path: arrow + repository: lriggs/arrow + ref: b6593b36259c93208e7f3161a7634bdfb5a788dc + submodules: recursive + + - name: Set up Python by actions/setup-python + if: runner.arch == 'X64' + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: 3.8 + - name: Set up Python by apt + if: runner.os == 'Linux' && runner.arch != 'X64' + run: | + sudo apt update + sudo apt-get install -y python3-pip + pip install -U pip + echo "$HOME/.local/bin" >>"$GITHUB_PATH" + - name: Install Archery + shell: bash + run: pip install -e arrow/dev/archery[all] + + - name: Free up disk space + run: | + df -h + sudo rm -rf /opt/hostedtoolcache/CodeQL || : + df -h + - name: Build C++ libraries + env: + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}" + SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}" + SCCACHE_S3_KEY_PREFIX: "sccache" + + run: | + archery docker run \ + -e ARROW_JAVA_BUILD=OFF \ + -e ARROW_JAVA_TEST=OFF \ + java-jni-manylinux-2014 + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/ + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: ubuntu-shared-lib + path: arrow-shared-libs-linux.tar.gz + + build-cpp-macos: + name: Build C++ libraries macOS ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: ["macos-latest"], arch: "x86_64"} + env: + MACOSX_DEPLOYMENT_TARGET: "10.13" + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 1 + path: arrow + repository: lriggs/arrow + ref: b6593b36259c93208e7f3161a7634bdfb5a788dc + submodules: recursive + + - name: Set up Python + if: | + !contains(matrix.platform.runs_on, 'self-hosted') + uses: actions/setup-python@v4 + with: + cache: 'pip' + python-version: 3.8 + - name: Install Archery + shell: bash + run: pip install -e arrow/dev/archery[all] + - name: Install dependencies + run: | + # We want to use llvm@14 to avoid shared z3 + # dependency. llvm@14 doesn't depend on z3 and llvm depends + # on z3. And Homebrew's z3 provides only shared library. It + # doesn't provides static z3 because z3's CMake doesn't accept + # building both shared and static libraries at once. + # See also: Z3_BUILD_LIBZ3_SHARED in + # https://github.com/Z3Prover/z3/blob/master/README-CMake.md + # + # If llvm is installed, Apache Arrow C++ uses llvm rather than + # llvm@14 because llvm is newer than llvm@14. + brew uninstall llvm || : + + brew bundle --file=arrow/cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall re2 || : + + brew bundle --file=arrow/java/Brewfile + - name: Build C++ libraries + env: + ARROW_USE_CCACHE: "ON" + run: | + set -e + # make brew Java available to CMake + if [ "${{ matrix.platform.arch }}" = "aarch_64" ]; then + export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + fi + arrow/ci/scripts/java_jni_macos_build.sh \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/arrow/cpp-build \ + $GITHUB_WORKSPACE/arrow/java-dist + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/ + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: macos-shared-lib-${{ matrix.platform.arch }} + path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz + + build-cpp-windows: + name: Build C++ libraries Windows + runs-on: windows-2019 + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 1 + path: arrow + repository: lriggs/arrow + ref: b6593b36259c93208e7f3161a7634bdfb5a788dc + submodules: recursive + + - name: Set up Java + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + - name: Download Timezone Database + shell: bash + run: arrow/ci/scripts/download_tz_database.sh + - name: Install sccache + shell: bash + run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache + - name: Build C++ libraries + shell: cmd + env: + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}" + SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}" + SCCACHE_S3_KEY_PREFIX: "sccache" + + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + REM For ORC + set TZDIR=/c/msys64/usr/share/zoneinfo + bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/ + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: windows-shared-lib + path: arrow-shared-libs-windows.tar.gz + + package-jars: + name: Build jar files + runs-on: macos-latest + needs: + - build-cpp-ubuntu + - build-cpp-macos + - build-cpp-windows + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: arrow + repository: lriggs/arrow + ref: b6593b36259c93208e7f3161a7634bdfb5a788dc + submodules: recursive + + - name: Download Libraries + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: Decompress artifacts + run: | + mv artifacts/*/*.tar.gz . + tar -xvzf arrow-shared-libs-linux.tar.gz + tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz + tar -xvzf arrow-shared-libs-windows.tar.gz + - name: Test that shared libraries exist + run: | + set -x + + test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so + test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so + test -f arrow/java-dist/x86_64/libarrow_orc_jni.so + test -f arrow/java-dist/x86_64/libgandiva_jni.so + + test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib + test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib + test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib + test -f arrow/java-dist/x86_64/libgandiva_jni.dylib + + test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll + test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll + test -f arrow/java-dist/x86_64/arrow_orc_jni.dll + - name: Build bundled jar + run: | + set -e + pushd arrow/java + mvn versions:set -DnewVersion=12.0.4-SNAPSHOT + popd + arrow/ci/scripts/java_full_build.sh \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/arrow/java-dist + - name: Set up Python by actions/setup-python + if: runner.arch == 'X64' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Set up Python by apt + if: runner.os == 'Linux' && runner.arch != 'X64' + run: | + sudo apt update + sudo apt install -y \ + libgit2-dev \ + libpython3-dev \ + python3-pip + sudo python3 -m pip install --upgrade pip + - name: Checkout Crossbow + uses: actions/checkout@v3 + with: + path: crossbow + ref: nightly-123 + - name: Setup Crossbow + shell: bash + run: | + python3 -m pip install -e arrow/dev/archery[crossbow] + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Upload artifacts + shell: bash + run: | + archery crossbow \ + --queue-path $(pwd)/crossbow \ + --queue-remote https://github.com/lriggs/arrow-build \ + upload-artifacts \ + --sha nightly-123-github-java-jars \ + --tag nightly-123-github-java-jars \ + "arrow/java-dist/*.jar" \ + "arrow/java-dist/*.json" \ + "arrow/java-dist/*.pom" \ + "arrow/java-dist/*.xml" \ + "arrow/java-dist/*.zip" + env: + CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + - name: Verify uploaded artifacts + shell: bash + run: | + archery crossbow \ + --queue-path $(pwd)/crossbow \ + --queue-remote https://github.com/lriggs/arrow-build \ + status \ + --task-filter 'java-jars' \ + --no-fetch \ + --validate \ + nightly-123 + env: + CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/generate_jar.yaml b/.github/workflows/generate_jar.yaml deleted file mode 100644 index f8e2fc7d..00000000 --- a/.github/workflows/generate_jar.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: Trigger Crossbow build - -on: workflow_dispatch - -env: - CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }} - ARCHERY_DEFAULT_BRANCH: 'master' -jobs: - trigger-crossbow: - name: Trigger Crossbow - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: 'master' - fetch-depth: 0 - - name: Install Conda - run: | - set -e - echo "" - echo "Installing Miniconda." - MINICONDA_URL="https://repo.continuum.io/miniconda" - MINICONDA_FILE="Miniconda3-py39_22.11.1-1-Linux-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b - - name: Configuring Conda - run: | - set -e - echo "" - echo "Configuring conda." - source /home/runner/miniconda3/bin/activate root - conda config --remove channels defaults - conda config --add channels defaults - conda config --add channels conda-forge - conda config --set show_channel_urls true - - name: Install packages - run: | - set -e - source /home/runner/miniconda3/bin/activate root - conda install -y \ - click \ - github3.py \ - jinja2 \ - jira \ - pygit2=1.5.0 \ - ruamel.yaml \ - setuptools_scm \ - toolz - - name: Run Crossbow script - run: | - set -e - pushd .. - source /home/runner/miniconda3/bin/activate root - git clone -b array_types_24.3 https://github.com/lriggs/arrow.git - pip install -e arrow/dev/archery[crossbow] - echo GITHUB_WORKSPACE:$GITHUB_WORKSPACE - echo GITHUB_REPOSITORY: $GITHUB_REPOSITORY - cd arrow-build - echo pwd - pwd - archery crossbow --queue-path $GITHUB_WORKSPACE --queue-remote https://github.com/$GITHUB_REPOSITORY submit java-jars --job-prefix nightly diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..f4d0062a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ + +branches: + only: + - master + - /.*-travis-.*/ + +os: linux +dist: trusty +language: generic