From 942409a9a3c1938c111d7b742b3ec919a05e7932 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 26 Jul 2024 14:37:11 +0800 Subject: [PATCH] Initial --- .github/workflows/velox_be.yml.deprecated | 700 ------------------ .../workflows/velox_tpch_bench.yml.disabled | 57 -- .../workflows/velox_tpch_merge.yml.disabled | 60 -- .github/workflows/velox_velox_ut.yml.disabled | 67 -- 4 files changed, 884 deletions(-) delete mode 100644 .github/workflows/velox_be.yml.deprecated delete mode 100644 .github/workflows/velox_tpch_bench.yml.disabled delete mode 100644 .github/workflows/velox_tpch_merge.yml.disabled delete mode 100644 .github/workflows/velox_velox_ut.yml.disabled diff --git a/.github/workflows/velox_be.yml.deprecated b/.github/workflows/velox_be.yml.deprecated deleted file mode 100644 index 6ff5ec743c14..000000000000 --- a/.github/workflows/velox_be.yml.deprecated +++ /dev/null @@ -1,700 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Velox backend - -on: - pull_request: - paths: - - '.github/**' - - 'pom.xml' - - 'backends-velox/**' - - 'gluten-celeborn/**' - - 'gluten-core/**' - - 'gluten-data/**' - - 'gluten-delta/**' - - 'gluten-iceberg/**' - - 'gluten-ut/**' - - 'shims/**' - - 'tools/gluten-it/**' - - 'tools/gluten-te/**' - - 'ep/build-velox/**' - - 'cpp/*' - - 'cpp/CMake/**' - - 'cpp/velox/**' - - 'cpp/core/**' - - 'dev/**' - -env: - HTTP_PROXY_HOST: proxy-shz.intel.com - HTTP_PROXY_PORT: 911 - PATH_TO_GLUTEN_TE: ./tools/gluten-te - DOCKER_PULL_REGISTRY: 10.1.0.25:5000 - MAVEN_OPTS: -Dmaven.wagon.http.retryHandler.count=3 - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - ubuntu2004-test-spark32: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF --build_test_utils=ON' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --build_tests=ON --build_examples=ON --build_benchmarks=ON' - - name: Run CPP unit test - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/cpp/build && \ - ctest -V' - - name: Run HBM CPP unit test - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/cpp/build && \ - cmake -DBUILD_TESTS=ON -DENABLE_HBM=ON .. && \ - cmake --build . --target hbw_allocator_test -- -j && \ - ctest -V -R TestHbw' - - name: Build and run unit test for Spark 3.2.2 (other tests) - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest' - # Cpp micro benchmarks will use generated files from unit test in backends-velox module. - - name: Run micro benchmarks - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/cpp/build/velox/benchmarks && \ - ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1' - - name: Copy golden files from container to host - if: failure() - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/cp.sh /tmp/tpch-approved-plan/ /tmp/$GITHUB_RUN_ID/spark32/tpch-approved-plan - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: golden-files-spark32 - path: | - /tmp/${{ github.run_id }}/spark32/tpch-approved-plan/** - - name: Clean temp golden files - run: | - rm -rf /tmp/$GITHUB_RUN_ID/spark32 - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2004-test-spark32-slow: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON' - - name: Build and run unit test for Spark 3.2.2 (slow tests) - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -Piceberg -Pdelta -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2004-test-spark33-slow: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON' - - name: Build and Run unit test for Spark 3.3.1 (slow tests) - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \ - mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.3 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 Q38 flush - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 --queries=q38 \ - --disable-bhj \ - --extra-conf=spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.1 \ - --extra-conf=spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.2 \ - --extra-conf=spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100 \ - --extra-conf=spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2004-test-spark33: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --build_examples=ON' - - name: Build and Run unit test for Spark 3.3.1 (other tests) - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \ - mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest' - - name: Copy golden files from container to host - if: failure() - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/cp.sh /tmp/tpch-approved-plan/ /tmp/$GITHUB_RUN_ID/spark33/tpch-approved-plan - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: golden-files-spark33 - path: | - /tmp/${{ github.run_id }}/spark33/tpch-approved-plan/** - - name: Clean temp golden files - run: | - rm -rf /tmp/$GITHUB_RUN_ID/spark33 - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2004-test-spark34-slow: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON ' - - name: Build and Run unit test for Spark 3.4.2 (slow tests) - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \ - mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.4 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.4 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2004-test-spark34: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --build_examples=ON' - - name: Build and Run unit test for Spark 3.4.2 (other tests) - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \ - mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest' - - name: Copy golden files from container to host - if: failure() - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/cp.sh /tmp/tpch-approved-plan/ /tmp/$GITHUB_RUN_ID/spark34/tpch-approved-plan - - name: Upload golden files - if: failure() - uses: actions/upload-artifact@v4 - with: - name: golden-files-spark34 - path: | - /tmp/${{ github.run_id }}/spark34/tpch-approved-plan/** - - name: Clean temp golden files - run: | - rm -rf /tmp/$GITHUB_RUN_ID/spark34 - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2004-test-spark35: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 20.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON ' - - name: Build for Spark 3.5 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.5 -Pbackends-velox -Prss -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.5 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.5 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2204-test-spark33-spark34: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 22.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build for Spark 3.3.1 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.3 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.3 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=30g -s=10.0 --threads=32 --iterations=1' - - name: Build for Spark 3.4.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.4 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.4 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=30g -s=10.0 --threads=32 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - ubuntu2204-test: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: ubuntu - OS_IMAGE_TAG: 22.04 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build for Spark 3.2.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg -Pdelta -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx20G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=40g -s=10.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with Celeborn 0.4.0 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh \ - 'cd /opt && mkdir -p celeborn && \ - tar xzf apache-celeborn-0.4.0-incubating-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ - mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \ - echo -e "CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g" > ./conf/celeborn-env.sh && \ - echo -e "celeborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \ - && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \ - cd /opt/gluten/tools/gluten-it && mvn clean install -Pspark-3.2,rss,celeborn-0.4 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \ - bash /opt/celeborn/sbin/stop-worker.sh \ - && bash /opt/celeborn/sbin/stop-master.sh && rm -rf /opt/celeborn' - - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with Celeborn 0.3.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh \ - 'cd /opt && mkdir -p celeborn && \ - tar xzf apache-celeborn-0.3.2-incubating-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \ - mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \ - echo -e "CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g" > ./conf/celeborn-env.sh && \ - echo -e "celeborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64" > ./conf/celeborn-defaults.conf \ - && bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \ - cd /opt/gluten/tools/gluten-it && mvn clean install -Pspark-3.2,rss \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox-with-celeborn --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 && \ - bash /opt/celeborn/sbin/stop-worker.sh \ - && bash /opt/celeborn/sbin/stop-master.sh' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - centos8-test: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: centos - OS_IMAGE_TAG: 8 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - source /env.sh && \ - sudo yum -y install patch && \ - cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - source /env.sh && \ - cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build for Spark 3.2.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg -Pdelta -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 random kill tasks - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 --data-gen-strategy=skip --random-kill-tasks \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --threads=32 --iterations=1 --data-gen-strategy=skip --random-kill-tasks' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - centos7-test: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: centos - OS_IMAGE_TAG: 7 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Gluten velox third party - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - yum -y install epel-release centos-release-scl patch sudo && \ - cd /opt/gluten/ep/build-velox/src && \ - source /opt/rh/devtoolset-9/enable && \ - ./get_velox.sh --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON && \ - ./build_velox.sh --run_setup_script=ON --enable_ep_cache=OFF --enable_s3=ON --enable_gcs=ON --enable_abfs=ON --enable_hdfs=ON' - - name: Build Gluten CPP library - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten/cpp && \ - source /opt/rh/devtoolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON' - - name: Build for Spark 3.2.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg -Pdelta -DskipTests' - - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --threads=32 --iterations=1' - - name: TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory, memory isolation off - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen-strategy=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ - -d=OFFHEAP_SIZE:5g,spark.memory.offHeap.size=5g \ - -d=OFFHEAP_SIZE:3g,spark.memory.offHeap.size=3g \ - -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ - -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5' - - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory, memory isolation on - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2 \ - && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen-strategy=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:5g,spark.memory.offHeap.size=5g \ - -d=OFFHEAP_SIZE:3g,spark.memory.offHeap.size=3g \ - -d=OVER_ACQUIRE:0.3,spark.gluten.memory.overAcquiredMemoryRatio=0.3 \ - -d=OVER_ACQUIRE:0.5,spark.gluten.memory.overAcquiredMemoryRatio=0.5' || true - - name: TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen-strategy=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ - -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \ - -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0' - - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q97 low memory # The case currently causes crash with "free: invalid size". - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten/tools/gluten-it && \ - GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh parameterized \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q97 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen-strategy=skip -m=OffHeapExecutionMemory \ - -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \ - -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \ - -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \ - -d=OFFHEAP_SIZE:1g,spark.memory.offHeap.size=1g' || true - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh - - - static-build-centos7-test: - runs-on: velox-self-hosted - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID -e NUM_THREADS=30 -detach $DOCKER_PULL_REGISTRY/gluten-te/gluten-buildenv-centos:7 \ - bash -c 'cd /opt/gluten && sleep 14400' - - name: Build Gluten CPP library - run: | - docker exec -i static-build-test-$GITHUB_RUN_ID bash -c ' - source /env.sh && \ - sudo yum -y install patch && \ - cd /opt/gluten && \ - sudo -E ./dev/vcpkg/setup-build-depends.sh && \ - source ./dev/vcpkg/env.sh && \ - ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON \ - --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --build_type=Debug' - - name: Build for Spark 3.2.2 - run: | - docker exec static-build-test-$GITHUB_RUN_ID bash -c ' - cd /opt/gluten && \ - mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg -Pdelta -DskipTests && \ - cd /opt/gluten/tools/gluten-it && \ - mvn clean install -Pspark-3.2' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 20.04) - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 ubuntu:20.04 \ - bash -c 'apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install openjdk-8-jre -y \ - && cd /opt/gluten/tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 (ubuntu 22.04) - run: | - docker run --rm --init --privileged --ulimit nofile=65536:65536 --ulimit core=-1 --security-opt seccomp=unconfined \ - -v $PWD:/opt/gluten --name static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30 ubuntu:22.04 \ - bash -c 'apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install openjdk-8-jre -y \ - && cd /opt/gluten/tools/gluten-it \ - && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - && GLUTEN_IT_JVM_ARGS=-Xmx10G sbin/gluten-it.sh queries-compare \ - --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --threads=32 --iterations=1' - - name: Exit docker container - if: ${{ always() }} - run: | - docker stop static-build-test-$GITHUB_RUN_ID || true - - build-script-test: - runs-on: velox-self-hosted - env: - OS_IMAGE_NAME: centos - OS_IMAGE_TAG: 8 - steps: - - uses: actions/checkout@v4 - - name: Setup docker container - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/checkout.sh - - name: Build Script Test - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh ' - source /env.sh && \ - sudo yum -y install patch && \ - cd /opt/gluten/ && \ - ./dev/package.sh' - - name: Exit docker container - if: ${{ always() }} - run: | - $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/clean.sh diff --git a/.github/workflows/velox_tpch_bench.yml.disabled b/.github/workflows/velox_tpch_bench.yml.disabled deleted file mode 100644 index 6be12305c623..000000000000 --- a/.github/workflows/velox_tpch_bench.yml.disabled +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Velox backend Performance Tests On Comments - -on: - issue_comment: - types: [created, edited, deleted] - -env: - JENKINS_HOST: 10.0.0.68 - JENKINS_PORT: 8080 - JENKINS_USER: jenkins:11fd1b5a82bfd638bd9b3749c96b324ff2 - -jobs: - velox-trigger-benchmark-on-comment: - # check the comment if it contains the keywords - if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/Benchmark Velox') }} - runs-on: velox-self-hosted - container: ubuntu:22.04 - steps: - - uses: actions/checkout@v4 - - run: apt-get update && apt-get install ca-certificates -y && update-ca-certificates - - run: sed -i 's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g' /etc/apt/sources.list - - run: apt-get update - - run: apt-get install -y curl - - name: trigger_tpch_bench - run: | - branchname=origin/pr/${{ github.event.issue.number }}/head - curl --noproxy $JENKINS_HOST -L -X POST http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_Github_Action_Check/build --user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1", "value":"'$branchname'"}]}' - velox-trigger-tpcds-on-comment: - # check the comment if it contains the keywords - if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/Benchmark Velox TPCDS') }} - runs-on: velox-self-hosted - container: ubuntu:22.04 - steps: - - uses: actions/checkout@v4 - - run: apt-get update && apt-get install ca-certificates -y && update-ca-certificates - - run: sed -i 's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g' /etc/apt/sources.list - - run: apt-get update - - run: apt-get install -y curl - - name: trigger_tpch_bench - run: | - branchname=origin/pr/${{ github.event.issue.number }}/head - curl --noproxy $JENKINS_HOST -L -X POST http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_DS_Github_Action_Check/build --user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1", "value":"'$branchname'"}]}' diff --git a/.github/workflows/velox_tpch_merge.yml.disabled b/.github/workflows/velox_tpch_merge.yml.disabled deleted file mode 100644 index 5012e830111e..000000000000 --- a/.github/workflows/velox_tpch_merge.yml.disabled +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Velox backend Performance Tests on Merge - -on: - pull_request: - types: [ closed ] - branches: - - main - paths: - - '.github/**' - - 'pom.xml' - - 'backends-velox/**' - - 'gluten-celeborn/**' - - 'gluten-core/**' - - 'gluten-data/**' - - 'gluten-ut/**' - - 'shims/**' - - 'tools/gluten-it/**' - - 'tools/gluten-te/**' - - 'ep/build-velox/**' - - 'cpp/*' - - 'cpp/CMake/**' - - 'cpp/velox/**' - - 'cpp/core/**' - - 'dev/**' - -env: - JENKINS_HOST: 10.0.0.68 - JENKINS_PORT: 8080 - JENKINS_USER: jenkins:11fd1b5a82bfd638bd9b3749c96b324ff2 - -jobs: - velox-trigger-benchmark: - if: github.event.pull_request.merged == true - runs-on: velox-self-hosted - container: ubuntu:22.04 - steps: - - uses: actions/checkout@v4 - - run: apt-get update && apt-get install ca-certificates -y && update-ca-certificates - - run: sed -i 's/http\:\/\/archive.ubuntu.com/https\:\/\/mirrors.ustc.edu.cn/g' /etc/apt/sources.list - - run: apt-get update - - run: apt-get install -y curl - - name: trigger_tpch_bench - run: | - branchname=origin/pr/${{github.event.number}}/head - curl --noproxy $JENKINS_HOST -L -X POST http://$JENKINS_HOST:$JENKINS_PORT/job/Gluten_Perf_Github_Action_Check/build --user $JENKINS_USER --data-urlencode json='{"parameter": [{"name":"sha1", "value":"'$branchname'"}]}' diff --git a/.github/workflows/velox_velox_ut.yml.disabled b/.github/workflows/velox_velox_ut.yml.disabled deleted file mode 100644 index efdf512983d6..000000000000 --- a/.github/workflows/velox_velox_ut.yml.disabled +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Velox backend Velox Unit test - - -on: - pull_request: - paths: - - '.github/workflows/velox_velox_ut.yml' - - 'dev/**' - - 'ep/**' #get_velox change - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - run-velox-unit-test: - runs-on: ubuntu-20.04 - env: - CCACHE_DIR: "${{ github.workspace }}/.ccache" - container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx - steps: - - uses: actions/checkout@v2 - - name: Get Ccache - uses: actions/cache/restore@v3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-ubuntu-release-default - - name: Ensure Cache Dirs Exists - working-directory: ${{ github.workspace }} - run: | - mkdir -p '${{ env.CCACHE_DIR }}' - - name: Build Gluten velox third party - run: | - rm -rf /opt/miniconda-for-velox/ - cd ep/build-velox/src && \ - ./get_velox.sh - cd ../build/velox_ep/ - git reset --hard - make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" - - - name: CCache after - run: | - ccache -vs - - - uses: actions/cache/save@v3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-ubuntu-release-default - - - name: Run Tests - run: | - cd ${GITHUB_WORKSPACE}/ep/build-velox/build/velox_ep/_build/release && ctest -E "velox_cache_test|velox_exec_test" -j 4 --output-on-failure --no-tests=error