[CORE] Refine maven Spark dependency (#3625)

- Remove the profile in shim and ut module, so when we add a new Spark version, we do not need change these two modules. Instead, add spark.shim.module and spark.test.module. - Remove spark32, spark33, spark34 properties, we only need spark.version and change it in each spark version profile - Remove the spark-hive-thriftserver dependency
apache · Nov 8, 2023 · 427971d · 427971d
1 parent b60fe75
commit 427971d
Show file tree

Hide file tree

Showing 13 changed files with 40 additions and 141 deletions.
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
@@ -53,7 +53,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+      <artifactId>spark-hive_${scala.binary.version}</artifactId>
       <scope>provided</scope>
     </dependency>
     <dependency>

diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
@@ -55,15 +55,9 @@
       <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
-      <scope>provided</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-hive_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>

diff --git a/gluten-celeborn/clickhouse/pom.xml b/gluten-celeborn/clickhouse/pom.xml
@@ -75,9 +75,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <type>test-jar</type>
+      <artifactId>spark-hive_${scala.binary.version}</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

diff --git a/gluten-celeborn/package/pom.xml b/gluten-celeborn/package/pom.xml
@@ -11,7 +11,7 @@
 
   <artifactId>gluten-celeborn-package</artifactId>
   <packaging>jar</packaging>
-  <name>Gluten Celeborn Common</name>
+  <name>Gluten Celeborn Package</name>
 
   <profiles>
     <profile>

diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml
@@ -52,7 +52,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+      <artifactId>spark-hive_${scala.binary.version}</artifactId>
       <scope>provided</scope>
     </dependency>
     <dependency>

diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml
@@ -54,7 +54,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+      <artifactId>spark-hive_${scala.binary.version}</artifactId>
       <scope>provided</scope>
     </dependency>
     <dependency>
@@ -174,35 +174,5 @@
         </plugin>
       </plugins>
     </pluginManagement>
-
   </build>
-
-  <profiles>
-    <profile>
-      <id>spark-3.2</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-      </properties>
-      <modules>
-        <module>spark32</module>
-        <module>common</module>
-      </modules>
-    </profile>
-    <profile>
-      <id>spark-3.3</id>
-      <modules>
-        <module>spark33</module>
-        <module>common</module>
-      </modules>
-    </profile>
-    <profile>
-      <id>spark-3.4</id>
-      <modules>
-        <module>spark34</module>
-        <module>common</module>
-      </modules>
-    </profile>
-  </profiles>
 </project>
diff --git a/package/pom.xml b/package/pom.xml
@@ -50,7 +50,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+      <artifactId>spark-hive_${scala.binary.version}</artifactId>
       <scope>provided</scope>
     </dependency>
   </dependencies>
@@ -198,6 +198,9 @@
                     <ignoreClass>org.apache.spark.unused.UnusedStubClass</ignoreClass>
                     <!-- jdo -->
                     <ignoreClass>javax.jdo.*</ignoreClass>
+                    <!-- javax -->
+                    <ignoreClass>javax.transaction.*</ignoreClass>
+                    <ignoreClass>javax.xml.*</ignoreClass>
                     <!-- log4j -->
                     <ignoreClass>org.apache.commons.logging.*</ignoreClass>
                     <!-- hive -->

diff --git a/pom.xml b/pom.xml
@@ -30,36 +30,31 @@
   </licenses>
 
   <modules>
-    <module>shims</module>
     <module>gluten-core</module>
+    <module>gluten-ui</module>
     <module>package</module>
+    <module>shims</module>
+    <module>shims/common</module>
+    <module>shims/${spark.shim.module}</module>
     <module>substrait/substrait-spark</module>
-    <module>gluten-ui</module>
   </modules>
 
   <properties>
     <caffeine.version.java8>2.9.3</caffeine.version.java8>
-    <spark32.version>3.2.2</spark32.version>
-    <spark32.scala>2.12.15</spark32.scala>
-    <spark32bundle.version>3.2</spark32bundle.version>
-    <spark33.version>3.3.1</spark33.version>
-    <spark34.version>3.4.1</spark34.version>
-    <spark33.scala>2.12.15</spark33.scala>
-    <spark33bundle.version>3.3</spark33bundle.version>
-    <spark34bundle.version>3.4</spark34bundle.version>
-    <delta20.version>2.0.1</delta20.version>
-    <delta22.version>2.2.0</delta22.version>
-    <delta.version>${delta20.version}</delta.version>
+    <delta.version>2.0.1</delta.version>
     <delta.binary.version>20</delta.binary.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <scala.version>${spark32.scala}</scala.version>
-    <spark.version>${spark32.version}</spark.version>
+    <scala.version>2.12.15</scala.version>
     <spark.major.version>3</spark.major.version>
+    <sparkbundle.version>3.2</sparkbundle.version>
+    <spark.shim.module>spark32</spark.shim.module>
+    <spark.test.module>spark32</spark.test.module>
+    <spark.version>3.2.2</spark.version>
+    <sparkshim.artifactId>spark-sql-columnar-shims-${spark.shim.module}</sparkshim.artifactId>
     <celeborn.version>0.3.0-incubating</celeborn.version>
-    <sparkbundle.version>${spark32bundle.version}</sparkbundle.version>
     <arrow.version>12.0.0</arrow.version>
     <arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
-    <hadoop.version>${hadoop.version}</hadoop.version>
+    <hadoop.version>2.7.4</hadoop.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <project.prefix>spark-sql-columnar</project.prefix>
@@ -70,8 +65,7 @@
     <!-- For unit tests -->
     <clickhouse.lib.path>/usr/local/clickhouse/lib/libch.so</clickhouse.lib.path>
     <tpcds.data.path>/data/tpcds-data-sf1</tpcds.data.path>
-    <fasterxml.spark33.version>2.13.3</fasterxml.spark33.version>
-    <fasterxml.version>${fasterxml.spark33.version}</fasterxml.version>
+    <fasterxml.version>2.13.3</fasterxml.version>
     <junit.version>4.13.1</junit.version>
 
     <substrait.version>0.5.0</substrait.version>
@@ -106,9 +100,6 @@
     <maven.jar.plugin>3.2.2</maven.jar.plugin>
     <scalastyle.version>1.0.0</scalastyle.version>
     <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
-    <spark32.shim.version>spark-sql-columnar-shims-spark32</spark32.shim.version>
-    <spark33.shim.version>spark-sql-columnar-shims-spark33</spark33.shim.version>
-    <spark34.shim.version>spark-sql-columnar-shims-spark34</spark34.shim.version>
   </properties>
 
   <profiles>
@@ -118,46 +109,38 @@
         <activeByDefault>true</activeByDefault>
       </activation>
       <properties>
-        <scala.version>${spark32.scala}</scala.version>
-        <spark.major.version>3</spark.major.version>
-        <spark.version>${spark32.version}</spark.version>
-        <delta.version>${delta20.version}</delta.version>
+        <sparkbundle.version>3.2</sparkbundle.version>
+        <spark.shim.module>spark32</spark.shim.module>
+        <spark.test.module>spark32</spark.test.module>
+        <spark.version>3.2.2</spark.version>
+        <delta.version>2.0.1</delta.version>
         <delta.binary.version>20</delta.binary.version>
-        <sparkbundle.version>${spark32bundle.version}</sparkbundle.version>
-        <sparkshim.artifactId>${spark32.shim.version}</sparkshim.artifactId>
       </properties>
     </profile>
     <profile>
       <id>spark-3.3</id>
       <properties>
-        <scala.version>${spark33.scala}</scala.version>
-        <spark.major.version>3</spark.major.version>
-        <spark.version>${spark33.version}</spark.version>
-        <delta.version>${delta22.version}</delta.version>
+        <sparkbundle.version>3.3</sparkbundle.version>
+        <spark.shim.module>spark33</spark.shim.module>
+        <spark.test.module>spark33</spark.test.module>
+        <spark.version>3.3.1</spark.version>
+        <delta.version>2.2.0</delta.version>
         <delta.binary.version>22</delta.binary.version>
-        <sparkbundle.version>${spark33bundle.version}</sparkbundle.version>
-        <sparkshim.artifactId>${spark33.shim.version}</sparkshim.artifactId>
       </properties>
     </profile>
     <profile>
       <id>spark-3.4</id>
       <properties>
-        <scala.version>${spark33.scala}</scala.version>
-        <spark.major.version>3</spark.major.version>
-        <spark.version>${spark34.version}</spark.version>
-        <delta.version>${delta22.version}</delta.version>
+        <sparkbundle.version>3.4</sparkbundle.version>
+        <spark.shim.module>spark34</spark.shim.module>
+        <spark.test.module>spark34</spark.test.module>
+        <spark.version>3.4.1</spark.version>
+        <delta.version>2.2.0</delta.version>
         <delta.binary.version>22</delta.binary.version>
-        <sparkbundle.version>${spark34bundle.version}</sparkbundle.version>
-        <sparkshim.artifactId>${spark34.shim.version}</sparkshim.artifactId>
       </properties>
     </profile>
     <profile>
       <id>hadoop-2.7.4</id>
-      <activation>
-        <property>
-          <name>!hadoop.version</name>
-        </property>
-      </activation>
       <properties>
         <hadoop.version>2.7.4</hadoop.version>
       </properties>
@@ -237,6 +220,8 @@
       <id>spark-ut</id>
       <modules>
         <module>gluten-ut</module>
+        <module>gluten-ut/common</module>
+        <module>gluten-ut/${spark.test.module}</module>
       </modules>
     </profile>
   </profiles>
@@ -286,7 +271,7 @@
       </dependency>
       <dependency>
         <groupId>org.apache.spark</groupId>
-        <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+        <artifactId>spark-hive_${scala.binary.version}</artifactId>
         <version>${spark.version}</version>
         <scope>provided</scope>
         <exclusions>

diff --git a/shims/common/pom.xml b/shims/common/pom.xml
@@ -32,13 +32,11 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-hive_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
       <scope>provided</scope>
     </dependency>
   </dependencies>

diff --git a/shims/pom.xml b/shims/pom.xml
@@ -46,7 +46,6 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
-      <version>${hadoop.version}</version>
     </dependency>
   </dependencies>
 
@@ -64,32 +63,5 @@
         </plugin>
       </plugins>
     </pluginManagement>
-
   </build>
-  <profiles>
-    <profile>
-      <id>spark-3.2</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <modules>
-        <module>common</module>
-        <module>spark32</module>
-      </modules>
-    </profile>
-    <profile>
-      <id>spark-3.3</id>
-      <modules>
-        <module>common</module>
-        <module>spark33</module>
-      </modules>
-    </profile>
-    <profile>
-      <id>spark-3.4</id>
-      <modules>
-        <module>common</module>
-        <module>spark34</module>
-      </modules>
-    </profile>
-  </profiles>
 </project>
diff --git a/shims/spark32/pom.xml b/shims/spark32/pom.xml
@@ -38,21 +38,18 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark32.version}</version>
       <scope>provided</scope>
       <optional>true</optional>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-catalyst_2.12</artifactId>
-      <version>${spark32.version}</version>
       <scope>provided</scope>
       <optional>true</optional>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_2.12</artifactId>
-      <version>${spark32.version}</version>
       <scope>provided</scope>
       <optional>true</optional>
     </dependency>
@@ -72,25 +69,21 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
-      <version>${spark32.version}</version>
       <type>test-jar</type>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark32.version}</version>
       <type>test-jar</type>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
-      <version>${spark32.version}</version>
       <type>test-jar</type>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-hive_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
       <scope>provided</scope>
     </dependency>
   </dependencies>