From de3aeb136a2f03649d7fdf514d0e2d9c5c992b71 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 20 Dec 2024 17:23:01 +0800 Subject: [PATCH] [GLUTEN-8266][VL][CI] Pre-install spark sources in docker image (#8290) --- .../workflows/util/install_spark_resources.sh | 10 ++++----- .github/workflows/velox_backend.yml | 22 +++++++++---------- dev/docker/Dockerfile.centos8-dynamic-build | 7 ++++++ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh index dd2afec821d4..1b00fe3ff293 100755 --- a/.github/workflows/util/install_spark_resources.sh +++ b/.github/workflows/util/install_spark_resources.sh @@ -19,7 +19,7 @@ set -e -INSTALL_DIR=$GITHUB_WORKSPACE +INSTALL_DIR=/opt/ case "$1" in 3.2) # Spark-3.2 @@ -79,12 +79,12 @@ case "$1" in wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \ tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \ rm -rf spark-3.5.2-bin-hadoop3.tgz && \ - mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \ - mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \ + mkdir -p ${INSTALL_DIR}/shims/spark35-scala2.13/spark_home/assembly/target/scala-2.13 && \ + mv jars ${INSTALL_DIR}/shims/spark35-scala2.13/spark_home/assembly/target/scala-2.13 && \ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \ tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/ && \ - mkdir -p shims/spark35/spark_home/ && \ - mv sql shims/spark35/spark_home/ + mkdir -p shims/spark35-scala2.13/spark_home/ && \ + mv sql shims/spark35-scala2.13/spark_home/ ;; *) echo "Spark version is expected to be specified." diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 2bba907bb0db..3ec7e36ca08b 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -621,7 +621,7 @@ jobs: cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 $MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg \ - -Pdelta -Phudi -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" \ + -Pdelta -Phudi -DargLine="-Dspark.test.home=/opt/shims/spark32/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report if: always() @@ -659,7 +659,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + -DargLine="-Dspark.test.home=/opt/shims/spark32/spark_home/" -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report if: always() uses: actions/upload-artifact@v4 @@ -696,7 +696,7 @@ jobs: cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 $MVN_CMD clean test -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" \ + -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report if: always() @@ -735,7 +735,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean test -Pspark-3.3 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" \ + -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report if: always() @@ -773,7 +773,7 @@ jobs: cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 $MVN_CMD clean test -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" \ + -DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report if: always() @@ -812,7 +812,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean test -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -Phudi \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" \ + -DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report if: always() @@ -850,7 +850,7 @@ jobs: cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report if: always() @@ -894,7 +894,7 @@ jobs: cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Pceleborn -Piceberg \ - -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ + -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala2.13/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report if: always() @@ -926,7 +926,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report if: always() @@ -964,7 +964,7 @@ jobs: cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report uses: actions/upload-artifact@v4 @@ -995,7 +995,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ - -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report uses: actions/upload-artifact@v4 diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index 655a44c5ea74..e0229697f61d 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -15,6 +15,13 @@ RUN wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.8 mv apache-maven-3.8.8 /usr/lib/maven ENV PATH=${PATH}:/usr/lib/maven/bin +RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz -P /opt/ +RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.4.2/apache-celeborn-0.4.2-bin.tgz -P /opt/ +RUN wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.5.2/apache-celeborn-0.5.2-bin.tgz -P /opt/ + RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten +RUN cd /opt/gluten/.github/workflows/util/ && ./install_spark_resources.sh 3.2 && ./install_spark_resources.sh 3.3 \ + && ./install_spark_resources.sh 3.4 && ./install_spark_resources.sh 3.5 && ./install_spark_resources.sh 3.5-scala2.13 + RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten