From f423c42ad342aa7d579dd4af51e808da899b0f23 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 16 Aug 2024 17:24:06 +0800 Subject: [PATCH 1/9] Initial --- .github/workflows/velox_docker.yml | 97 +++++++++++++++------------- dev/ci-velox-buildstatic-centos-7.sh | 6 +- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index c310d5298f12..2c5dfd698d4d 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -63,22 +63,31 @@ jobs: with: path: | ./cpp/build/releases/ + ./cpp/build/velox/udf/examples/ + ./cpp/build/velox/benchmarks/ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }} - name: Build Gluten native libraries if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | df -a + cd $GITHUB_WORKSPACE/ bash dev/ci-velox-buildstatic-centos-7.sh - - name: Upload Artifact Native - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2 with: - path: ./cpp/build/releases/ name: velox-native-lib-centos-7-${{github.sha}} - - name: Upload Artifact Arrow Jar - uses: actions/upload-artifact@v2 + path: ./cpp/build/releases/ + - uses: actions/upload-artifact@v2 + with: + name: udf-example-lib-centos-7-${{github.sha}} + path: ./cpp/build/velox/udf/examples/ + - uses: actions/upload-artifact@v2 + with: + name: benchmark-centos-7-${{github.sha}} + path: ./cpp/build/velox/benchmarks/ + - uses: actions/upload-artifact@v2 with: - path: /root/.m2/repository/org/apache/arrow/ name: velox-arrow-jar-centos-7-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ run-tpc-test-ubuntu: needs: build-native-lib-centos-7 @@ -624,32 +633,28 @@ jobs: path: /root/.m2/repository/org/apache/arrow/ run-spark-test-spark32: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: CCACHE_DIR: "${{ github.workspace }}/.ccache" steps: - uses: actions/checkout@v2 - - name: Download All Artifacts - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download UDF Example Lib - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v2 with: - name: udf-example-lib-centos-8-${{github.sha}} + name: udf-example-lib-centos-7-${{github.sha}} path: ./cpp/build/velox/udf/examples/ - - name: Download Benchmark - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v2 with: - name: benchmark-centos-8-${{github.sha}} + name: benchmark-centos-7-${{github.sha}} path: ./cpp/build/velox/benchmarks/ - - name: Download Arrow Jars - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -709,7 +714,7 @@ jobs: ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 run-spark-test-spark32-slow: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -719,7 +724,7 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download Arrow Jars uses: actions/download-artifact@v2 @@ -759,7 +764,7 @@ jobs: $MVN_CMD clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark33: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -769,17 +774,17 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download UDF Example Lib uses: actions/download-artifact@v2 with: - name: udf-example-lib-centos-8-${{github.sha}} + name: udf-example-lib-centos-7-${{github.sha}} path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -834,7 +839,7 @@ jobs: run-spark-test-spark33-slow: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -844,12 +849,12 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -884,7 +889,7 @@ jobs: $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark34: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -894,17 +899,17 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download UDF Example Lib uses: actions/download-artifact@v2 with: - name: udf-example-lib-centos-8-${{github.sha}} + name: udf-example-lib-centos-7-${{github.sha}} path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -959,7 +964,7 @@ jobs: run-spark-test-spark34-slow: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -969,12 +974,12 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -1009,7 +1014,7 @@ jobs: $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark35: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -1019,17 +1024,17 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download UDF Example Lib uses: actions/download-artifact@v2 with: - name: udf-example-lib-centos-8-${{github.sha}} + name: udf-example-lib-centos-7-${{github.sha}} path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -1083,7 +1088,7 @@ jobs: path: /tmp/tpch-approved-plan/** run-spark-test-spark35-scala213: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -1093,17 +1098,17 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download UDF Example Lib uses: actions/download-artifact@v2 with: - name: udf-example-lib-centos-8-${{github.sha}} + name: udf-example-lib-centos-7-${{github.sha}} path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -1151,7 +1156,7 @@ jobs: $MVN_CMD test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest run-spark-test-spark35-slow: - needs: build-native-lib-centos-8 + needs: build-native-lib-centos-7 runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 env: @@ -1161,12 +1166,12 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@v2 with: - name: velox-native-lib-centos-8-${{github.sha}} + name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index d83d443fc489..ed47940ad8bb 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -3,8 +3,6 @@ set -e source /opt/rh/devtoolset-9/enable -cd $GITHUB_WORKSPACE/ -source ./dev/vcpkg/env.sh export NUM_THREADS=4 -./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --build_arrow=OFF --enable_s3=ON \ - --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON +./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=ON --build_benchmarks=ON \ + --build_examples=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON From 6396f4aad38cfa290c1acb647cf7ae317b5e7f8c Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Mon, 19 Aug 2024 22:45:37 +0800 Subject: [PATCH 2/9] Disable benchmark/test --- .github/workflows/velox_docker.yml | 84 ++++++++++++++-------------- dev/ci-velox-buildstatic-centos-7.sh | 4 +- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 2c5dfd698d4d..06bdec3b6fd1 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -76,14 +76,14 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases/ - - uses: actions/upload-artifact@v2 - with: - name: udf-example-lib-centos-7-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ - - uses: actions/upload-artifact@v2 - with: - name: benchmark-centos-7-${{github.sha}} - path: ./cpp/build/velox/benchmarks/ +# - uses: actions/upload-artifact@v2 +# with: +# name: udf-example-lib-centos-7-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ +# - uses: actions/upload-artifact@v2 +# with: +# name: benchmark-centos-7-${{github.sha}} +# path: ./cpp/build/velox/benchmarks/ - uses: actions/upload-artifact@v2 with: name: velox-arrow-jar-centos-7-${{github.sha}} @@ -644,14 +644,14 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - - uses: actions/download-artifact@v2 - with: - name: udf-example-lib-centos-7-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ - - uses: actions/download-artifact@v2 - with: - name: benchmark-centos-7-${{github.sha}} - path: ./cpp/build/velox/benchmarks/ +# - uses: actions/download-artifact@v2 +# with: +# name: udf-example-lib-centos-7-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ +# - uses: actions/download-artifact@v2 +# with: +# name: benchmark-centos-7-${{github.sha}} +# path: ./cpp/build/velox/benchmarks/ - uses: actions/download-artifact@v2 with: name: arrow-jars-centos-7-${{github.sha}} @@ -706,12 +706,12 @@ jobs: with: name: golden-files-spark32 path: /tmp/tpch-approved-plan/** - - name: Gluten CPP Benchmark Test - run: | - # This test depends on example.json generated by the above mvn test. - cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ - sudo chmod +x ./generic_benchmark && \ - ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 +# - name: Gluten CPP Benchmark Test +# run: | +# # This test depends on example.json generated by the above mvn test. +# cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ +# sudo chmod +x ./generic_benchmark && \ +# ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 run-spark-test-spark32-slow: needs: build-native-lib-centos-7 @@ -776,11 +776,11 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download UDF Example Lib - uses: actions/download-artifact@v2 - with: - name: udf-example-lib-centos-7-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ +# - name: Download UDF Example Lib +# uses: actions/download-artifact@v2 +# with: +# name: udf-example-lib-centos-7-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -901,11 +901,11 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download UDF Example Lib - uses: actions/download-artifact@v2 - with: - name: udf-example-lib-centos-7-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ +# - name: Download UDF Example Lib +# uses: actions/download-artifact@v2 +# with: +# name: udf-example-lib-centos-7-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -1026,11 +1026,11 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download UDF Example Lib - uses: actions/download-artifact@v2 - with: - name: udf-example-lib-centos-7-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ +# - name: Download UDF Example Lib +# uses: actions/download-artifact@v2 +# with: +# name: udf-example-lib-centos-7-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -1100,11 +1100,11 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases - - name: Download UDF Example Lib - uses: actions/download-artifact@v2 - with: - name: udf-example-lib-centos-7-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ +# - name: Download UDF Example Lib +# uses: actions/download-artifact@v2 +# with: +# name: udf-example-lib-centos-7-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index ed47940ad8bb..3272de95d910 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -4,5 +4,5 @@ set -e source /opt/rh/devtoolset-9/enable export NUM_THREADS=4 -./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=ON --build_benchmarks=ON \ - --build_examples=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON +./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ + --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON From 23ceb9a28068aadf90b088d3078947645ba795e1 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 08:43:24 +0800 Subject: [PATCH 3/9] Change lib cache name --- .github/workflows/velox_docker.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 06bdec3b6fd1..820ca05b8cb1 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -86,7 +86,7 @@ jobs: # path: ./cpp/build/velox/benchmarks/ - uses: actions/upload-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ run-tpc-test-ubuntu: @@ -127,7 +127,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Setup tzdata run: | @@ -198,7 +198,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -283,7 +283,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | @@ -395,7 +395,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | @@ -488,7 +488,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -555,7 +555,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: velox-arrow-jar-centos-7-${{github.sha}} + name: arrow-jar-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Setup tzdata run: | From ca0181cbe736dd74a1059703785efb2af29bddc0 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 09:33:27 +0800 Subject: [PATCH 4/9] Update doc --- docs/developers/UsingGperftoolsInCH.md | 7 +++++++ docs/developers/UsingJemallocWithCH.md | 7 +++++++ docs/developers/VeloxUDF.md | 7 +++++++ docs/developers/velox_backend_CI.md | 22 ++++++++++++++++++++++ 4 files changed, 43 insertions(+) create mode 100644 docs/developers/velox_backend_CI.md diff --git a/docs/developers/UsingGperftoolsInCH.md b/docs/developers/UsingGperftoolsInCH.md index f0d5c720b30b..5c0c001c9974 100644 --- a/docs/developers/UsingGperftoolsInCH.md +++ b/docs/developers/UsingGperftoolsInCH.md @@ -1,3 +1,10 @@ +--- +layout: page +title: Developer Overview +nav_order: 11 +has_children: true +permalink: /developer-overview/ +--- We need using gpertools to find the memory or CPU issue. That's what this document is about. ## Install gperftools diff --git a/docs/developers/UsingJemallocWithCH.md b/docs/developers/UsingJemallocWithCH.md index 626f7522d7c8..133e536ce4a4 100644 --- a/docs/developers/UsingJemallocWithCH.md +++ b/docs/developers/UsingJemallocWithCH.md @@ -1,3 +1,10 @@ +--- +layout: page +title: Developer Overview +nav_order: 12 +has_children: true +permalink: /developer-overview/ +--- We need using jemalloc to find the memory issue. That's what this document is about. ## Change code of jemalloc diff --git a/docs/developers/VeloxUDF.md b/docs/developers/VeloxUDF.md index 25b896929a43..9bdce17a423b 100644 --- a/docs/developers/VeloxUDF.md +++ b/docs/developers/VeloxUDF.md @@ -1,3 +1,10 @@ +--- +layout: page +title: Developer Overview +nav_order: 13 +has_children: true +permalink: /developer-overview/ +--- # Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF) ## Introduction diff --git a/docs/developers/velox_backend_CI.md b/docs/developers/velox_backend_CI.md new file mode 100644 index 000000000000..b713b421a19a --- /dev/null +++ b/docs/developers/velox_backend_CI.md @@ -0,0 +1,22 @@ +--- +layout: page +title: CPP Code Style +nav_order: 14 +parent: Developer Overview +--- +# Velox Backend CI + +GHA workflows are defined under `.github/workflows/`. + +## Docker Build +We have a weekly job to build a docker based on Dockerfile.gha for CI verification, defined in docker_image.yml. + +## Vcpkg Caching +Gluten main branch is pulled down during docker build. And vcpkg will cache binary data of all dependencies defined under dev/vcpkg. +These binary data is cached into `/var/cache/vcpkg` and CI job can re-use them in new build. By setting `VCPKG_BINARY_SOURCES=clear` in env., reusing cache can be disabled. + +## Arrow Libs Pre-installation +Arrow libs are also pre-installed in docker, assuming they are not actively changed and not necessarily to be re-built every time. + +## Updating Docker +Two Github secrets `DOCKERHUB_USER` & `DOCKERHUB_TOKEN` can be used to push docker to docker hub: https://hub.docker.com/r/apache/gluten/tags. From c60f7e888ef07344ab8c3cb47218d80c91d39580 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 09:41:40 +0800 Subject: [PATCH 5/9] Fix --- .github/workflows/velox_docker.yml | 20 ++++++++++---------- dev/vcpkg/ports/simdjson/vcpkg.json | 3 +-- docs/developers/UsingGperftoolsInCH.md | 4 ++-- docs/developers/UsingJemallocWithCH.md | 4 ++-- docs/developers/VeloxUDF.md | 4 ++-- docs/developers/velox_backend_CI.md | 14 ++++++++------ 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 820ca05b8cb1..a83ec4f7da3a 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -63,8 +63,8 @@ jobs: with: path: | ./cpp/build/releases/ - ./cpp/build/velox/udf/examples/ - ./cpp/build/velox/benchmarks/ +# ./cpp/build/velox/udf/examples/ +# ./cpp/build/velox/benchmarks/ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }} - name: Build Gluten native libraries if: ${{ steps.cache.outputs.cache-hit != 'true' }} @@ -86,7 +86,7 @@ jobs: # path: ./cpp/build/velox/benchmarks/ - uses: actions/upload-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ run-tpc-test-ubuntu: @@ -127,7 +127,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Setup tzdata run: | @@ -198,7 +198,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -283,7 +283,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | @@ -395,7 +395,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /home/runner/.m2/repository/org/apache/arrow/ - name: Setup java and maven run: | @@ -488,7 +488,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | @@ -555,7 +555,7 @@ jobs: - name: Download All Arrow Jar Artifacts uses: actions/download-artifact@v2 with: - name: arrow-jar-centos-7-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Setup tzdata run: | @@ -729,7 +729,7 @@ jobs: - name: Download Arrow Jars uses: actions/download-artifact@v2 with: - name: arrow-jars-centos-8-${{github.sha}} + name: arrow-jars-centos-7-${{github.sha}} path: /root/.m2/repository/org/apache/arrow/ - name: Update mirror list run: | diff --git a/dev/vcpkg/ports/simdjson/vcpkg.json b/dev/vcpkg/ports/simdjson/vcpkg.json index 6e46382e42f8..2b74be554cfc 100644 --- a/dev/vcpkg/ports/simdjson/vcpkg.json +++ b/dev/vcpkg/ports/simdjson/vcpkg.json @@ -17,8 +17,7 @@ "default-features": [ "deprecated", "exceptions", - "threads", - "utf8-validation" + "threads" ], "features": { "deprecated": { diff --git a/docs/developers/UsingGperftoolsInCH.md b/docs/developers/UsingGperftoolsInCH.md index 5c0c001c9974..5a4bbea3fbbc 100644 --- a/docs/developers/UsingGperftoolsInCH.md +++ b/docs/developers/UsingGperftoolsInCH.md @@ -1,9 +1,9 @@ --- layout: page -title: Developer Overview +title: Debug for CH Backend with gpertools Tool nav_order: 11 has_children: true -permalink: /developer-overview/ +parent: /developer-overview/ --- We need using gpertools to find the memory or CPU issue. That's what this document is about. diff --git a/docs/developers/UsingJemallocWithCH.md b/docs/developers/UsingJemallocWithCH.md index 133e536ce4a4..365a35dd39fe 100644 --- a/docs/developers/UsingJemallocWithCH.md +++ b/docs/developers/UsingJemallocWithCH.md @@ -1,9 +1,9 @@ --- layout: page -title: Developer Overview +title: Use Jemalloc for CH Backend nav_order: 12 has_children: true -permalink: /developer-overview/ +parent: /developer-overview/ --- We need using jemalloc to find the memory issue. That's what this document is about. diff --git a/docs/developers/VeloxUDF.md b/docs/developers/VeloxUDF.md index 9bdce17a423b..6872f2d0c841 100644 --- a/docs/developers/VeloxUDF.md +++ b/docs/developers/VeloxUDF.md @@ -1,9 +1,9 @@ --- layout: page -title: Developer Overview +title: Velox UDF and UDAF nav_order: 13 has_children: true -permalink: /developer-overview/ +parent: /developer-overview/ --- # Velox User-Defined Functions (UDF) and User-Defined Aggregate Functions (UDAF) diff --git a/docs/developers/velox_backend_CI.md b/docs/developers/velox_backend_CI.md index b713b421a19a..e4dceffdf72e 100644 --- a/docs/developers/velox_backend_CI.md +++ b/docs/developers/velox_backend_CI.md @@ -1,6 +1,6 @@ --- layout: page -title: CPP Code Style +title: Velox Backend CI nav_order: 14 parent: Developer Overview --- @@ -9,14 +9,16 @@ parent: Developer Overview GHA workflows are defined under `.github/workflows/`. ## Docker Build -We have a weekly job to build a docker based on Dockerfile.gha for CI verification, defined in docker_image.yml. +We have a weekly job to build a docker based on `Dockerfile.gha` for CI verification, defined in `docker_image.yml`. ## Vcpkg Caching Gluten main branch is pulled down during docker build. And vcpkg will cache binary data of all dependencies defined under dev/vcpkg. -These binary data is cached into `/var/cache/vcpkg` and CI job can re-use them in new build. By setting `VCPKG_BINARY_SOURCES=clear` in env., reusing cache can be disabled. +These binary data is cached into `/var/cache/vcpkg` and CI job can re-use them in new build. By setting `VCPKG_BINARY_SOURCES=clear` in env., +reusing vcpkg cache can be disabled. ## Arrow Libs Pre-installation -Arrow libs are also pre-installed in docker, assuming they are not actively changed and not necessarily to be re-built every time. +Arrow libs are pre-installed in docker, assuming they are not actively changed, then not necessarily to be re-built every time. -## Updating Docker -Two Github secrets `DOCKERHUB_USER` & `DOCKERHUB_TOKEN` can be used to push docker to docker hub: https://hub.docker.com/r/apache/gluten/tags. +## Updating Docker Image +Two GitHub secrets `DOCKERHUB_USER` & `DOCKERHUB_TOKEN` can be used to push docker image to docker hub: https://hub.docker.com/r/apache/gluten/tags. +Note GitHub secrets are not retrievable in PR from forked repo. \ No newline at end of file From 7b65a63856228cc2f913e3f107c42f1a19f355e2 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 17:04:36 +0800 Subject: [PATCH 6/9] Move tests location --- .github/workflows/velox_docker.yml | 91 +++++++++++++++++++--------- dev/ci-velox-buildshared-centos-8.sh | 2 - 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index a83ec4f7da3a..d88866bd9df0 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -592,7 +592,7 @@ jobs: GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox-with-celeborn --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1 - build-native-lib-centos-8: + Run CPP unit/benchmark test & UDF test: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 steps: @@ -615,22 +615,37 @@ jobs: run: | df -a bash dev/ci-velox-buildshared-centos-8.sh - - uses: actions/upload-artifact@v2 - with: - name: velox-native-lib-centos-8-${{github.sha}} - path: ./cpp/build/releases/ - - uses: actions/upload-artifact@v2 - with: - name: udf-example-lib-centos-8-${{github.sha}} - path: ./cpp/build/velox/udf/examples/ - - uses: actions/upload-artifact@v2 - with: - name: benchmark-centos-8-${{github.sha}} - path: ./cpp/build/velox/benchmarks/ - - uses: actions/upload-artifact@v2 - with: - name: arrow-jars-centos-8-${{github.sha}} - path: /root/.m2/repository/org/apache/arrow/ + - name: Run CPP unit test + run: | + cd ./cpp/build && ctest -V + - name: Run CPP benchmark test + run: | + mvn test -Pspark-3.2 -Pbackends-velox -Pceleborn -pl backends-velox -am \ + -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none \ + -DfailIfNoTests=false -Dexec.skip + # This test depends on example.json generated by the above mvn test. + cd cpp/build/velox/benchmarks && sudo chmod +x ./generic_benchmark + ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 + - name: Run UDF test + # Depends on --build_example=ON. + $MVN_CMD test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None \ + -DtagsToInclude=org.apache.gluten.tags.UDFTest +# - uses: actions/upload-artifact@v2 +# with: +# name: velox-native-lib-centos-8-${{github.sha}} +# path: ./cpp/build/releases/ +# - uses: actions/upload-artifact@v2 +# with: +# name: udf-example-lib-centos-8-${{github.sha}} +# path: ./cpp/build/velox/udf/examples/ +# - uses: actions/upload-artifact@v2 +# with: +# name: benchmark-centos-8-${{github.sha}} +# path: ./cpp/build/velox/benchmarks/ +# - uses: actions/upload-artifact@v2 +# with: +# name: arrow-jars-centos-8-${{github.sha}} +# path: /root/.m2/repository/org/apache/arrow/ run-spark-test-spark32: needs: build-native-lib-centos-7 @@ -698,8 +713,10 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - $MVN_CMD clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - $MVN_CMD test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg \ + -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags +# $MVN_CMD test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -828,8 +845,10 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - $MVN_CMD test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags +# $MVN_CMD test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -886,7 +905,9 @@ jobs: - name: Build and Run unit test for Spark 3.3.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut + -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark34: needs: build-native-lib-centos-7 @@ -953,8 +974,10 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - $MVN_CMD test -Pspark-3.4 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags +# $MVN_CMD test -Pspark-3.4 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -1011,7 +1034,9 @@ jobs: - name: Build and Run unit test for Spark 3.4.2 (slow tests) run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest run-spark-test-spark35: needs: build-native-lib-centos-7 @@ -1078,8 +1103,10 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags +# $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -1152,8 +1179,10 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 - $MVN_CMD clean install -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \ - $MVN_CMD test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest + $MVN_CMD clean install -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Pceleborn -Piceberg \ + -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags +# $MVN_CMD test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest run-spark-test-spark35-slow: needs: build-native-lib-centos-7 @@ -1209,4 +1238,6 @@ jobs: - name: Build and Run unit test for Spark 3.5.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index 362900bd009a..e77deaf899fd 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -14,5 +14,3 @@ export PATH="${PATH}:/usr/lib/maven/bin" source /opt/rh/gcc-toolset-9/enable ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON --build_protobuf=ON - -cd ./cpp/build && ctest -V From 81b30761f8a9a94b46d612951f4e9132e133717d Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 17:07:42 +0800 Subject: [PATCH 7/9] Rename doc --- docs/developers/{velox_backend_CI.md => velox-backend-CI.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/developers/{velox_backend_CI.md => velox-backend-CI.md} (100%) diff --git a/docs/developers/velox_backend_CI.md b/docs/developers/velox-backend-CI.md similarity index 100% rename from docs/developers/velox_backend_CI.md rename to docs/developers/velox-backend-CI.md From 522b083538b3161a9d39f36be9c4bbaf85a018bb Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 17:26:26 +0800 Subject: [PATCH 8/9] Fix build --- .github/workflows/velox_docker.yml | 12 ++++++------ dev/ci-velox-buildshared-centos-8.sh | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index d88866bd9df0..265394c6b983 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -592,7 +592,7 @@ jobs: GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox-with-celeborn --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=8 --iterations=1 - Run CPP unit/benchmark test & UDF test: + run-CPP-test-UDF-test: runs-on: ubuntu-20.04 container: ghcr.io/facebookincubator/velox-dev:centos8 steps: @@ -620,15 +620,15 @@ jobs: cd ./cpp/build && ctest -V - name: Run CPP benchmark test run: | - mvn test -Pspark-3.2 -Pbackends-velox -Pceleborn -pl backends-velox -am \ - -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none \ - -DfailIfNoTests=false -Dexec.skip + $MVN_CMD test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \ + -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip # This test depends on example.json generated by the above mvn test. cd cpp/build/velox/benchmarks && sudo chmod +x ./generic_benchmark ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 - name: Run UDF test + run: | # Depends on --build_example=ON. - $MVN_CMD test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None \ + $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None \ -DtagsToInclude=org.apache.gluten.tags.UDFTest # - uses: actions/upload-artifact@v2 # with: @@ -905,7 +905,7 @@ jobs: - name: Build and Run unit test for Spark 3.3.1 (slow tests) run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut + $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index e77deaf899fd..f337185de379 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -10,6 +10,7 @@ yum install sudo patch java-1.8.0-openjdk-devel wget -y wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven export PATH="${PATH}:/usr/lib/maven/bin" +echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV source /opt/rh/gcc-toolset-9/enable ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \ From 42ffa7c980c3b730d0b4657e4836ca40ff764de5 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 20 Aug 2024 22:22:15 +0800 Subject: [PATCH 9/9] Remove commented code --- .github/workflows/velox_docker.yml | 65 ------------------------------ 1 file changed, 65 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 265394c6b983..644fe62b932e 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -63,8 +63,6 @@ jobs: with: path: | ./cpp/build/releases/ -# ./cpp/build/velox/udf/examples/ -# ./cpp/build/velox/benchmarks/ key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }} - name: Build Gluten native libraries if: ${{ steps.cache.outputs.cache-hit != 'true' }} @@ -76,14 +74,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases/ -# - uses: actions/upload-artifact@v2 -# with: -# name: udf-example-lib-centos-7-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ -# - uses: actions/upload-artifact@v2 -# with: -# name: benchmark-centos-7-${{github.sha}} -# path: ./cpp/build/velox/benchmarks/ - uses: actions/upload-artifact@v2 with: name: arrow-jars-centos-7-${{github.sha}} @@ -630,22 +620,6 @@ jobs: # Depends on --build_example=ON. $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None \ -DtagsToInclude=org.apache.gluten.tags.UDFTest -# - uses: actions/upload-artifact@v2 -# with: -# name: velox-native-lib-centos-8-${{github.sha}} -# path: ./cpp/build/releases/ -# - uses: actions/upload-artifact@v2 -# with: -# name: udf-example-lib-centos-8-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ -# - uses: actions/upload-artifact@v2 -# with: -# name: benchmark-centos-8-${{github.sha}} -# path: ./cpp/build/velox/benchmarks/ -# - uses: actions/upload-artifact@v2 -# with: -# name: arrow-jars-centos-8-${{github.sha}} -# path: /root/.m2/repository/org/apache/arrow/ run-spark-test-spark32: needs: build-native-lib-centos-7 @@ -659,14 +633,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases -# - uses: actions/download-artifact@v2 -# with: -# name: udf-example-lib-centos-7-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ -# - uses: actions/download-artifact@v2 -# with: -# name: benchmark-centos-7-${{github.sha}} -# path: ./cpp/build/velox/benchmarks/ - uses: actions/download-artifact@v2 with: name: arrow-jars-centos-7-${{github.sha}} @@ -716,19 +682,12 @@ jobs: $MVN_CMD clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg \ -Pdelta -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags -# $MVN_CMD test -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 with: name: golden-files-spark32 path: /tmp/tpch-approved-plan/** -# - name: Gluten CPP Benchmark Test -# run: | -# # This test depends on example.json generated by the above mvn test. -# cd $GITHUB_WORKSPACE/cpp/build/velox/benchmarks && \ -# sudo chmod +x ./generic_benchmark && \ -# ./generic_benchmark --run-example --with-shuffle --threads 1 --iterations 1 run-spark-test-spark32-slow: needs: build-native-lib-centos-7 @@ -793,11 +752,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases -# - name: Download UDF Example Lib -# uses: actions/download-artifact@v2 -# with: -# name: udf-example-lib-centos-7-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -848,7 +802,6 @@ jobs: $MVN_CMD clean install -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags -# $MVN_CMD test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -922,11 +875,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases -# - name: Download UDF Example Lib -# uses: actions/download-artifact@v2 -# with: -# name: udf-example-lib-centos-7-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -977,7 +925,6 @@ jobs: $MVN_CMD clean install -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags -# $MVN_CMD test -Pspark-3.4 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -1051,11 +998,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases -# - name: Download UDF Example Lib -# uses: actions/download-artifact@v2 -# with: -# name: udf-example-lib-centos-7-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -1106,7 +1048,6 @@ jobs: $MVN_CMD clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags -# $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload golden files if: failure() uses: actions/upload-artifact@v4 @@ -1127,11 +1068,6 @@ jobs: with: name: velox-native-lib-centos-7-${{github.sha}} path: ./cpp/build/releases -# - name: Download UDF Example Lib -# uses: actions/download-artifact@v2 -# with: -# name: udf-example-lib-centos-7-${{github.sha}} -# path: ./cpp/build/velox/udf/examples/ - name: Download Arrow Jars uses: actions/download-artifact@v2 with: @@ -1182,7 +1118,6 @@ jobs: $MVN_CMD clean install -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Pceleborn -Piceberg \ -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags -# $MVN_CMD test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Piceberg -Pdelta -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest run-spark-test-spark35-slow: needs: build-native-lib-centos-7