[GLUTEN-7031][CORE] Initialize new module structure gluten-core / glu…

…ten-substrait (#7057) Closes #7031
apache · Aug 29, 2024 · 3af5bac · 3af5bac
1 parent 96130d1
commit 3af5bac
Show file tree

Hide file tree

Showing 338 changed files with 416 additions and 241 deletions.
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -48,6 +48,7 @@ CORE:
   - changed-files:
     - any-glob-to-any-file: [
       'gluten-core/**/*',
+      'gluten-substrait/**/*',
       'shims/**/*',
       'gluten-ras/**/*',
       'gluten-ui/**/*',

diff --git a/.github/workflows/clickhouse_be_trigger.yml b/.github/workflows/clickhouse_be_trigger.yml
@@ -26,6 +26,7 @@ on:
       - 'gluten-celeborn/package/**'
       - 'gluten-celeborn/clickhouse/**'
       - 'gluten-core/**'
+      - 'gluten-substrait/**'
       - 'gluten-ut/**'
       - 'shims/**'
       - 'tools/gluten-it/**'

diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml
@@ -27,6 +27,7 @@ on:
       - 'gluten-celeborn/velox/**'
       - 'gluten-ras/**'
       - 'gluten-core/**'
+      - 'gluten-substrait/**'
       - 'gluten-data/**'
       - 'gluten-delta/**'
       - 'gluten-iceberg/**'

diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
@@ -16,7 +16,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-core</artifactId>
+      <artifactId>gluten-substrait</artifactId>
       <version>${project.version}</version>
       <scope>compile</scope>
       <exclusions>
@@ -33,7 +33,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-core</artifactId>
+      <artifactId>gluten-substrait</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>

diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/affinity/CHUTAffinity.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/affinity/CHUTAffinity.scala
@@ -33,8 +33,8 @@ object CHUTSoftAffinityManager extends AffinityManager {
   override lazy val minOnTargetHosts: Int =
     GlutenConfig.GLUTEN_SOFT_AFFINITY_MIN_TARGET_HOSTS_DEFAULT_VALUE
 
-  override lazy val detectDuplicateReading = true
+  override lazy val detectDuplicateReading: Boolean = true
 
-  override lazy val duplicateReadingMaxCacheItems =
+  override lazy val duplicateReadingMaxCacheItems: Int =
     GlutenConfig.GLUTEN_SOFT_AFFINITY_DUPLICATE_READING_MAX_CACHE_ITEMS_DEFAULT_VALUE
 }
diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
@@ -34,7 +34,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-core</artifactId>
+      <artifactId>gluten-substrait</artifactId>
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
@@ -45,7 +45,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-core</artifactId>
+      <artifactId>gluten-substrait</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
@@ -78,12 +78,6 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.gluten</groupId>
-      <artifactId>spark-sql-columnar-shims-common</artifactId>
-      <version>${project.version}</version>
-      <scope>provided</scope>
-    </dependency>
     <dependency>
       <groupId>org.scalacheck</groupId>
       <artifactId>scalacheck_${scala.binary.version}</artifactId>

diff --git a/cpp-ch/local-engine/proto/substrait b/cpp-ch/local-engine/proto/substrait
@@ -1 +1 @@
-../../../gluten-core/src/main/resources/substrait/proto/substrait
+../../../gluten-substrait/src/main/resources/substrait/proto/substrait
diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt
@@ -39,11 +39,11 @@ endif()
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
 
 set(SUBSTRAIT_PROTO_SRC_DIR
-    ${GLUTEN_HOME}/gluten-core/src/main/resources/substrait/proto)
+    ${GLUTEN_HOME}/gluten-substrait/src/main/resources/substrait/proto)
 message(STATUS "Set Substrait Proto Directory in ${SUBSTRAIT_PROTO_SRC_DIR}")
 
 set(GLUTEN_PROTO_SRC_DIR
-    ${GLUTEN_HOME}/gluten-core/src/main/resources/org/apache/gluten/proto)
+    ${GLUTEN_HOME}/gluten-substrait/src/main/resources/org/apache/gluten/proto)
 message(STATUS "Set Gluten Proto Directory in ${GLUTEN_PROTO_SRC_DIR}")
 
 find_program(CCACHE_FOUND ccache)

diff --git a/docs/developers/HowTo.md b/docs/developers/HowTo.md
@@ -21,7 +21,7 @@ transforms Spark plan to Substrait plan, and then send the Substrait plan to the
 
 The Gluten codes consist of two parts: the C++ codes and the Java/Scala codes. 
 1. All C++ codes are placed under the directory of `${GLUTEN_HOME}/cpp`, the Java/Scala codes are placed under several directories, such as
-  `${GLUTEN_HOME}/gluten-core` `${GLUTEN_HOME}/gluten-data` `${GLUTEN_HOME}/backends-velox`.
+  `${GLUTEN_HOME}/gluten-substrait` `${GLUTEN_HOME}/gluten-data` `${GLUTEN_HOME}/backends-velox`.
 2. The Java/Scala codes are responsible for validating and transforming the execution plan. Source data should also be provided, the source data may
   come from files or other forms such as networks.
 3. The C++ codes take the Substrait plan and the source data as inputs and transform the Substrait plan to the corresponding backend plan. If the backend

diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md
@@ -489,11 +489,11 @@ This benchmark is tested on AWS EC2 cluster, there are 7 EC2 instances:
 
   Refer to [Deploy Spark 3.2.2](#deploy-spark-322)
 
-- Deploy gluten-core-XXXXX-jar-with-dependencies.jar
+- Deploy gluten-substrait-XXXXX-jar-with-dependencies.jar
 
 ```
-    #deploy 'gluten-core-XXXXX-jar-with-dependencies.jar' to every node, and then
-    cp gluten-core-XXXXX-jar-with-dependencies.jar /path_to_spark/jars/
+    #deploy 'gluten-substrait-XXXXX-jar-with-dependencies.jar' to every node, and then
+    cp gluten-substrait-XXXXX-jar-with-dependencies.jar /path_to_spark/jars/
 ```
 
 - Deploy ClickHouse library

diff --git a/gluten-celeborn/clickhouse/pom.xml b/gluten-celeborn/clickhouse/pom.xml
@@ -30,7 +30,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-core</artifactId>
+      <artifactId>gluten-substrait</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>

diff --git a/gluten-celeborn/pom.xml b/gluten-celeborn/pom.xml
@@ -17,7 +17,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-core</artifactId>
+      <artifactId>gluten-substrait</artifactId>
       <version>${project.version}</version>
       <scope>provided</scope>
     </dependency>

diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml
@@ -1,7 +1,6 @@
-<?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <parent>
     <artifactId>gluten-parent</artifactId>
     <groupId>org.apache.gluten</groupId>
@@ -10,19 +9,15 @@
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>gluten-core</artifactId>
+  <packaging>jar</packaging>
   <name>Gluten Core</name>
 
-  <properties>
-    <build.testJarPhase>none</build.testJarPhase>
-    <build.copyDependenciesPhase>package</build.copyDependenciesPhase>
-    <jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
-  </properties>
-
   <dependencies>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-ui</artifactId>
+      <artifactId>gluten-ras-common</artifactId>
       <version>${project.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.gluten</groupId>
@@ -32,7 +27,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-ras-common</artifactId>
+      <artifactId>spark-sql-columnar-shims-common</artifactId>
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
@@ -121,203 +116,18 @@
       <version>3.1.0.0-RC2</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.seleniumhq.selenium</groupId>
-      <artifactId>selenium-htmlunit-driver</artifactId>
-      <version>2.52.0</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.trino.tpch</groupId>
-      <artifactId>tpch</artifactId>
-      <version>1.1</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.trino.tpcds</groupId>
-      <artifactId>tpcds</artifactId>
-      <version>1.4</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.kohsuke</groupId>
-      <artifactId>github-api</artifactId>
-      <version>1.117</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.jsonwebtoken</groupId>
-      <artifactId>jjwt-api</artifactId>
-      <version>0.10.5</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.jsonwebtoken</groupId>
-      <artifactId>jjwt-impl</artifactId>
-      <version>0.10.5</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.jsonwebtoken</groupId>
-      <artifactId>jjwt-jackson</artifactId>
-      <version>0.10.5</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.knowm.xchart</groupId>
-      <artifactId>xchart</artifactId>
-      <version>3.6.5</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>de.erichseifert.vectorgraphics2d</groupId>
-          <artifactId>VectorGraphics2D</artifactId>		      
-	</exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.gluten</groupId>
-      <artifactId>spark-sql-columnar-shims-common</artifactId>
-      <version>${project.version}</version>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
       <version>${protobuf.version}</version>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>io.glutenproject</groupId>
-      <artifactId>protobuf-java-util</artifactId>
-      <version>${custom.protobuf.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-      <version>2.11.0</version>
-      <scope>provided</scope>
-    </dependency>
-
-    <!-- Fasterxml -->
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.module</groupId>
-      <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
-    </dependency>
-    <!-- Java Faker for generating random data -->
-    <dependency>
-      <groupId>com.github.javafaker</groupId>
-      <artifactId>javafaker</artifactId>
-      <version>1.0.2</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
-  <profiles>
-    <profile>
-      <id>backends-velox</id>
-      <properties>
-        <backend.type>velox</backend.type>
-        <backend.home>${project.basedir}/../ep/build-velox/build/velox_ep</backend.home>
-      </properties>
-    </profile>
-    <profile>
-      <id>backends-clickhouse</id>
-      <properties>
-        <backend.type>ch</backend.type>
-        <backend.home>${project.basedir}/../cpp-ch/ClickHouse</backend.home>
-      </properties>
-    </profile>
-  </profiles>
-
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-    <resources>
-      <resource>
-        <filtering>true</filtering>
-        <!-- Include the properties file to provide the build information. -->
-        <directory>${project.build.directory}/extra-resources</directory>
-      </resource>
-    </resources>
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>build-info</id>
-            <goals>
-              <goal>run</goal>
-            </goals>
-            <phase>generate-resources</phase>
-            <configuration>
-              <target>
-                <exec executable="bash" osfamily="unix">
-                  <arg value="${project.basedir}/../dev/gluten-build-info.sh"/>
-                  <arg value="${backend.type}"/>
-                  <arg value="${backend.home}"/>
-                  <arg value="${project.build.directory}/extra-resources"/>
-                  <arg value="${project.version}"/>
-                  <arg value="${java.version}"/>
-                  <arg value="${scala.version}"/>
-                  <arg value="${spark.version}"/>
-                  <arg value="${hadoop.version}"/>
-                </exec>
-              </target>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <!-- compile proto buffer files using copied protoc binary -->
-      <plugin>
-        <groupId>org.xolstice.maven.plugins</groupId>
-        <artifactId>protobuf-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>compile-substrait-proto</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>compile</goal>
-              <goal>test-compile</goal>
-            </goals>
-            <configuration>
-              <protocArtifact>
-                com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
-              </protocArtifact>
-              <protoSourceRoot>src/main/resources/substrait/proto</protoSourceRoot>
-              <clearOutputDirectory>true</clearOutputDirectory>
-            </configuration>
-          </execution>
-          <execution>
-            <id>compile-gluten-proto</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>compile</goal>
-              <goal>test-compile</goal>
-            </goals>
-            <configuration>
-              <protocArtifact>
-                com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
-              </protocArtifact>
-              <protoSourceRoot>src/main/resources/org/apache/gluten/proto</protoSourceRoot>
-              <clearOutputDirectory>false</clearOutputDirectory>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-resources-plugin</artifactId>
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		../../../gluten-core/src/main/resources/substrait/proto/substrait
		../../../gluten-substrait/src/main/resources/substrait/proto/substrait