From bacc546cbe9af7452c9701cdaa086f0ec390f3a6 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 20 Dec 2024 11:56:00 +0800 Subject: [PATCH] Initial --- .github/workflows/util/install_spark_resources.sh | 10 +++++----- dev/docker/Dockerfile.centos8-dynamic-build | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh index dd2afec821d4..1b00fe3ff293 100755 --- a/.github/workflows/util/install_spark_resources.sh +++ b/.github/workflows/util/install_spark_resources.sh @@ -19,7 +19,7 @@ set -e -INSTALL_DIR=$GITHUB_WORKSPACE +INSTALL_DIR=/opt/ case "$1" in 3.2) # Spark-3.2 @@ -79,12 +79,12 @@ case "$1" in wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \ tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \ rm -rf spark-3.5.2-bin-hadoop3.tgz && \ - mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \ - mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \ + mkdir -p ${INSTALL_DIR}/shims/spark35-scala2.13/spark_home/assembly/target/scala-2.13 && \ + mv jars ${INSTALL_DIR}/shims/spark35-scala2.13/spark_home/assembly/target/scala-2.13 && \ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \ tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/ && \ - mkdir -p shims/spark35/spark_home/ && \ - mv sql shims/spark35/spark_home/ + mkdir -p shims/spark35-scala2.13/spark_home/ && \ + mv sql shims/spark35-scala2.13/spark_home/ ;; *) echo "Spark version is expected to be specified." diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index 655a44c5ea74..9a35559dccfc 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -17,4 +17,7 @@ ENV PATH=${PATH}:/usr/lib/maven/bin RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten +RUN cd /opt/gluten/.github/workflows/util/ && ./install_spark_resources.sh 3.2 && ./install_spark_resources.sh 3.3 \ + && ./install_spark_resources.sh 3.4 && ./install_spark_resources.sh 3.5 && ./install_spark_resources.sh 3.5-scala2.13 + RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten