diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh
index dd2afec821d4..d5f413ee2116 100755
--- a/.github/workflows/util/install_spark_resources.sh
+++ b/.github/workflows/util/install_spark_resources.sh
@@ -63,26 +63,26 @@ case "$1" in
3.5)
# Spark-3.5
cd ${INSTALL_DIR} && \
- wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \
- tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \
- rm -rf spark-3.5.2-bin-hadoop3.tgz && \
+ wget -nv https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz && \
+ tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz spark-3.5.3-bin-hadoop3/jars/ && \
+ rm -rf spark-3.5.3-bin-hadoop3.tgz && \
mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
- wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \
- tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/ && \
+ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz && \
+ tar --strip-components=1 -xf v3.5.3.tar.gz spark-3.5.3/sql/core/src/test/resources/ && \
mkdir -p shims/spark35/spark_home/ && \
mv sql shims/spark35/spark_home/
;;
3.5-scala2.13)
# Spark-3.5, scala 2.13
cd ${INSTALL_DIR} && \
- wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \
- tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \
- rm -rf spark-3.5.2-bin-hadoop3.tgz && \
+ wget -nv https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz && \
+ tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz spark-3.5.3-bin-hadoop3/jars/ && \
+ rm -rf spark-3.5.3-bin-hadoop3.tgz && \
mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
- wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \
- tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/ && \
+ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz && \
+ tar --strip-components=1 -xf v3.5.3.tar.gz spark-3.5.3/sql/core/src/test/resources/ && \
mkdir -p shims/spark35/spark_home/ && \
mv sql shims/spark35/spark_home/
;;
diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml
index 8637c92ffe7c..dd31b8e77bb2 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -860,13 +860,13 @@ jobs:
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.2 cython && \
+ pip3 install pyspark==3.5.3 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.2 (other tests)
+ - name: Build and Run unit test for Spark 3.5.3 (other tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -906,9 +906,9 @@ jobs:
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.2 cython && \
+ pip3 install pyspark==3.5.3 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.2 with scala-2.13 (other tests)
+ - name: Build and Run unit test for Spark 3.5.3 with scala-2.13 (other tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -949,10 +949,10 @@ jobs:
- name: Prepare spark.test.home for Spark 3.5.2 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
- - name: Build and Run unit test for Spark 3.5.2 (slow tests)
+ - name: Build and Run unit test for Spark 3.5.3 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
diff --git a/pom.xml b/pom.xml
index a3ce1c6f70eb..ea8a9f980854 100644
--- a/pom.xml
+++ b/pom.xml
@@ -341,12 +341,12 @@
3.5
spark35
spark-sql-columnar-shims-spark35
- 3.5.2
+ 3.5.3
1.5.0
delta-spark
- 3.2.0
- 32
- 0.15.0
+ 3.2.1
+ 32
+ 0.15.0
2.15.1
3.3.4
4.9.3
diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
index eab32ab9d0b9..ad99f7be2547 100644
--- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
+++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.sql.shims.{SparkShimDescriptor, SparkShims}
import org.apache.gluten.sql.shims.spark35.SparkShimProvider.DESCRIPTOR
object SparkShimProvider {
- val DESCRIPTOR = SparkShimDescriptor(3, 5, 2)
+ val DESCRIPTOR = SparkShimDescriptor(3, 5, 3)
}
class SparkShimProvider extends org.apache.gluten.sql.shims.SparkShimProvider {
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index bad4d6087f11..70b8d20c5ba4 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -164,7 +164,7 @@
spark-3.5
- 3.5.2
+ 3.5.3
2.12.18