Skip to content

Commit

Permalink
[GLUTEN-8208][CORE] A new unified approach of source folder isolation…
Browse files Browse the repository at this point in the history
… for iceberg / hudi / delta with Maven (#8198)

Closes #8208
  • Loading branch information
zhztheplayer authored Dec 16, 2024
1 parent 4823fab commit 08bdb42
Show file tree
Hide file tree
Showing 111 changed files with 395 additions and 231 deletions.
111 changes: 28 additions & 83 deletions backends-clickhouse/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -38,47 +38,6 @@
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>${project.basedir}/src/main-iceberg/resource</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-iceberg-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/main-iceberg/scala</source>
<source>${project.basedir}/src/main-iceberg/java</source>
</sources>
</configuration>
</execution>
<execution>
<id>add-iceberg-test-sources</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/test-iceberg/scala</source>
<source>${project.basedir}/src/test-iceberg/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

Expand Down Expand Up @@ -323,11 +282,6 @@
</dependencies>

<build>
<resources>
<resource>
<directory>${project.basedir}/src/main/resources</directory>
</resource>
</resources>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<plugins>
Expand All @@ -353,6 +307,27 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce-delta-profile</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireActiveProfile>
<!--ClickHouse backend requires -P delta to be set-->
<profiles>delta</profiles>
<message>"-P delta" must be set when building Gluten with ClickHouse backend.</message>
</requireActiveProfile>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
Expand Down Expand Up @@ -436,15 +411,15 @@
<includes>
<include>src/main/scala/**/*.scala</include>
<include>src/test/scala/**/*.scala</include>
<include>src/main/delta-${delta.binary.version}/**/*.scala</include>
<include>src/test/delta-${delta.binary.version}/**/*.scala</include>
<include>src-delta-${delta.binary.version}/main/delta/**/*.scala</include>
<include>src-delta-${delta.binary.version}/test/delta/**/*.scala</include>
</includes>
<excludes>
<exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/*.scala</exclude>
<exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/merge/*.scala</exclude>
<exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/stats/*.scala</exclude>
<exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/DeltaLog.scala</exclude>
<exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/Snapshot.scala</exclude>
<exclude>src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/commands/*.scala</exclude>
<exclude>src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/commands/merge/*.scala</exclude>
<exclude>src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/stats/*.scala</exclude>
<exclude>src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/DeltaLog.scala</exclude>
<exclude>src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/Snapshot.scala</exclude>
</excludes>
</scala>
</configuration>
Expand Down Expand Up @@ -484,36 +459,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>src/main/delta-${delta.binary.version}</source>
</sources>
</configuration>
</execution>
<execution>
<id>add-test-sources</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>src/test/delta-${delta.binary.version}</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
Expand Down
84 changes: 3 additions & 81 deletions backends-velox/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -59,49 +59,7 @@
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>${project.basedir}/src/main-iceberg/resource</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-iceberg-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/main-iceberg/scala</source>
<source>${project.basedir}/src/main-iceberg/java</source>
</sources>
</configuration>
</execution>
<execution>
<id>add-iceberg-test-sources</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/test-iceberg/scala</source>
<source>${project.basedir}/src/test-iceberg/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

<profile>
<id>delta</id>
<dependencies>
Expand All @@ -123,42 +81,6 @@
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-delta-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/main-delta/scala</source>
<source>${project.basedir}/src/main-delta/java</source>
</sources>
</configuration>
</execution>
<execution>
<id>add-delta-test-sources</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/test-delta/scala</source>
<source>${project.basedir}/src/test-delta/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

Expand Down Expand Up @@ -317,11 +239,11 @@
<build>
<resources>
<resource>
<directory>${cpp.releases.dir}</directory>
<targetPath>${platform}/${arch}</targetPath>
<directory>${project.basedir}/src/main/resources</directory>
</resource>
<resource>
<directory>${project.basedir}/src/main/resources</directory>
<directory>${cpp.releases.dir}</directory>
<targetPath>${platform}/${arch}</targetPath>
</resource>
</resources>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging {
val glutenBuildInfo = new mutable.LinkedHashMap[String, String]()

val components = Component.sorted()
glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(","))
glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(", "))
components.foreach {
comp =>
val buildInfo = comp.buildInfo()
Expand Down
39 changes: 4 additions & 35 deletions gluten-delta/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@
<packaging>jar</packaging>
<name>Gluten DeltaLake</name>

<properties>
<resource.dir>${project.basedir}/src/main/resources</resource.dir>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.gluten</groupId>
Expand Down Expand Up @@ -92,11 +88,6 @@
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<resources>
<resource>
<directory>${resource.dir}</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
Expand All @@ -123,10 +114,10 @@
<file>${project.basedir}/../.scalafmt.conf</file>
</scalafmt>
<includes>
<include>src/main/scala/**/*.scala</include>
<include>src/test/scala/**/*.scala</include>
<include>src/main/delta-${delta.binary.version}/**/*.scala</include>
<include>src/test/delta-${delta.binary.version}/**/*.scala</include>
<include>src-delta/main/scala/**/*.scala</include>
<include>src-delta/test/scala/**/*.scala</include>
<include>src-delta-${delta.binary.version}/main/scala/**/*.scala</include>
<include>src-delta-${delta.binary.version}/test/scala/**/*.scala</include>
</includes>
</scala>
</configuration>
Expand All @@ -151,28 +142,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-resources</id>
<phase>generate-sources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>src/main/scala/org/apache/gluten/execution</outputDirectory>
<resources>
<resource>
<directory>src/main/delta-${delta.binary.version}/org/apache/gluten/execution</directory>
</resource>
</resources>
<overwrite>true</overwrite>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.gluten.execution

import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.execution.SparkPlan

case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan)
extends FilterExecTransformerBase(condition, child) {

override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer =
copy(child = newChild)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.gluten.execution

import org.apache.spark.sql.catalyst.expressions.NamedExpression
import org.apache.spark.sql.execution.SparkPlan

case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan)
extends ProjectExecTransformerBase(projectList, child) {

override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer =
copy(child = newChild)
}
Loading

0 comments on commit 08bdb42

Please sign in to comment.