Initial

apache · Dec 20, 2024 · bacc546 · bacc546
1 parent 0530292
commit bacc546
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 5 deletions.
diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh
@@ -19,7 +19,7 @@
 
 set -e
 
-INSTALL_DIR=$GITHUB_WORKSPACE
+INSTALL_DIR=/opt/
 case "$1" in
 3.2)
     # Spark-3.2
@@ -79,12 +79,12 @@ case "$1" in
     wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \
     tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \
     rm -rf spark-3.5.2-bin-hadoop3.tgz && \
-    mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
-    mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
+    mkdir -p ${INSTALL_DIR}/shims/spark35-scala2.13/spark_home/assembly/target/scala-2.13 && \
+    mv jars ${INSTALL_DIR}/shims/spark35-scala2.13/spark_home/assembly/target/scala-2.13 && \
     wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \
     tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/  && \
-    mkdir -p shims/spark35/spark_home/ && \
-    mv sql shims/spark35/spark_home/
+    mkdir -p shims/spark35-scala2.13/spark_home/ && \
+    mv sql shims/spark35-scala2.13/spark_home/
     ;;
 *)
     echo "Spark version is expected to be specified."

diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -17,4 +17,7 @@ ENV PATH=${PATH}:/usr/lib/maven/bin
 
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
 
+RUN cd /opt/gluten/.github/workflows/util/ && ./install_spark_resources.sh 3.2 && ./install_spark_resources.sh 3.3 \
+    && ./install_spark_resources.sh 3.4 && ./install_spark_resources.sh 3.5 && ./install_spark_resources.sh 3.5-scala2.13
+
 RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten