From 08bdb4273d2dea93d4d274735dc43278b2a9e1d6 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 16 Dec 2024 09:54:18 +0800 Subject: [PATCH] [GLUTEN-8208][CORE] A new unified approach of source folder isolation for iceberg / hudi / delta with Maven (#8198) Closes #8208 --- backends-clickhouse/pom.xml | 111 ++------ .../io/delta/tables/ClickhouseTable.scala | 0 .../parser/GlutenCacheFilesSqlParser.scala | 0 .../parser/GlutenClickhouseSqlParser.scala | 0 .../ClickhouseOptimisticTransaction.scala | 0 .../apache/spark/sql/delta/DeltaAdapter.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 0 .../org/apache/spark/sql/delta/Snapshot.scala | 0 .../sql/delta/catalog/ClickHouseTableV2.scala | 0 .../sql/delta/commands/DeleteCommand.scala | 0 .../sql/delta/commands/MergeIntoCommand.scala | 0 .../delta/commands/OptimizeTableCommand.scala | 0 .../OptimizeTableCommandOverwrites.scala | 0 .../sql/delta/commands/UpdateCommand.scala | 0 .../sql/delta/commands/VacuumCommand.scala | 0 .../MergeTreeDelayedCommitProtocol.scala | 0 .../CHOptimizeMetadataOnlyDeltaQuery.scala | 0 .../sql/execution/CHDeltaColumnarWrite.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 0 .../spark/gluten/delta/DeltaStatsUtils.scala | 0 .../io/delta/tables/ClickhouseTable.scala | 0 .../parser/GlutenCacheFilesSqlParser.scala | 0 .../parser/GlutenClickhouseSqlParser.scala | 0 .../ClickhouseOptimisticTransaction.scala | 0 .../apache/spark/sql/delta/DeltaAdapter.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 0 .../org/apache/spark/sql/delta/Snapshot.scala | 0 .../sql/delta/catalog/ClickHouseTableV2.scala | 0 .../sql/delta/commands/DeleteCommand.scala | 0 .../sql/delta/commands/MergeIntoCommand.scala | 0 .../delta/commands/OptimizeTableCommand.scala | 0 .../OptimizeTableCommandOverwrites.scala | 0 .../sql/delta/commands/UpdateCommand.scala | 0 .../sql/delta/commands/VacuumCommand.scala | 0 .../MergeTreeDelayedCommitProtocol.scala | 0 .../CHOptimizeMetadataOnlyDeltaQuery.scala | 0 .../sql/delta/stats/PrepareDeltaScan.scala | 0 .../sql/execution/CHDeltaColumnarWrite.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 0 .../spark/gluten/delta/DeltaStatsUtils.scala | 0 .../io/delta/tables/ClickhouseTable.scala | 0 .../parser/GlutenCacheFilesSqlParser.scala | 0 .../parser/GlutenClickhouseSqlParser.scala | 0 .../ClickhouseOptimisticTransaction.scala | 0 .../apache/spark/sql/delta/DeltaAdapter.scala | 0 .../org/apache/spark/sql/delta/DeltaLog.scala | 0 .../org/apache/spark/sql/delta/Snapshot.scala | 0 .../sql/delta/catalog/ClickHouseTableV2.scala | 0 .../sql/delta/commands/DeleteCommand.scala | 0 .../delta/commands/OptimizeTableCommand.scala | 0 .../OptimizeTableCommandOverwrites.scala | 0 .../sql/delta/commands/UpdateCommand.scala | 0 .../sql/delta/commands/VacuumCommand.scala | 0 .../commands/merge/ClassicMergeExecutor.scala | 0 .../MergeTreeDelayedCommitProtocol.scala | 0 .../CHOptimizeMetadataOnlyDeltaQuery.scala | 0 .../execution/FileDeltaColumnarWrite.scala | 0 .../MergeTreeDeltaColumnarWrite.scala | 0 .../MergeTreeFileFormatDataWriter.scala | 0 .../MergeTreeFileFormatWriter.scala | 0 .../v2/clickhouse/ClickHouseDataSource.scala | 0 .../clickhouse/ClickHouseSparkCatalog.scala | 0 .../source/DeltaMergeTreeFileFormat.scala | 0 .../spark/gluten/delta/DeltaStatsUtils.scala | 0 .../org.apache.gluten.component.Component | 0 .../gluten/component/CHIcebergComponent.scala | 0 .../iceberg/ClickHouseIcebergSuite.scala | 0 backends-velox/pom.xml | 84 +----- .../gluten/execution/VeloxDeltaSuite.scala | 0 .../execution/VeloxTPCHDeltaSuite.scala | 0 .../org.apache.gluten.component.Component | 0 .../component/VeloxIcebergComponent.scala | 0 .../gluten/execution/VeloxIcebergSuite.scala | 0 .../execution/VeloxTPCHIcebergSuite.scala | 0 .../org/apache/gluten/GlutenPlugin.scala | 2 +- gluten-delta/pom.xml | 39 +-- .../DeltaFilterExecTransformer.scala | 0 .../DeltaProjectExecTransformer.scala | 0 .../DeltaFilterExecTransformer.scala | 27 ++ .../DeltaProjectExecTransformer.scala | 27 ++ .../DeltaFilterExecTransformer.scala | 27 ++ .../DeltaProjectExecTransformer.scala | 27 ++ .../DeltaFilterExecTransformer.scala | 0 .../DeltaProjectExecTransformer.scala | 0 ...xecution.DataSourceScanTransformerRegister | 0 ...extension.columnar.RewriteTransformerRules | 0 .../execution/DeltaScanTransformer.scala | 0 .../DeltaScanTransformerProvider.scala | 0 .../DeltaRewriteTransformerRules.scala | 0 .../apache/gluten/execution/DeltaSuite.scala | 2 +- gluten-hudi/pom.xml | 9 - ...xecution.DataSourceScanTransformerRegister | 0 .../execution/HudiScanTransformer.scala | 0 .../HudiScanTransformerProvider.scala | 0 .../gluten/execution/VeloxHudiSuite.scala | 0 .../gluten/execution/VeloxTPCHHudiSuite.scala | 0 gluten-iceberg/pom.xml | 9 - .../rel/IcebergLocalFilesBuilder.java | 0 .../substrait/rel/IcebergLocalFilesNode.java | 0 .../execution/IcebergScanTransformer.scala | 0 .../gluten/execution/OffloadIcebergScan.scala | 0 .../source/GlutenIcebergSourceUtil.scala | 0 .../iceberg/spark/source/TypeUtil.scala | 0 .../gluten/execution/IcebergSuite.scala | 0 gluten-substrait/pom.xml | 9 +- package/pom.xml | 1 - pom.xml | 252 +++++++++++++++++- 111 files changed, 395 insertions(+), 231 deletions(-) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/DeltaAdapter.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/DeltaLog.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/Snapshot.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/commands/VacuumCommand.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) rename backends-clickhouse/{src/main/delta-20 => src-delta-20/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (100%) rename backends-clickhouse/{src/test/delta-20 => src-delta-20/test/scala}/org/apache/spark/gluten/delta/DeltaStatsUtils.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/DeltaAdapter.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/DeltaLog.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/Snapshot.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/commands/VacuumCommand.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) rename backends-clickhouse/{src/main/delta-23 => src-delta-23/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (100%) rename backends-clickhouse/{src/test/delta-23 => src-delta-23/test/scala}/org/apache/spark/gluten/delta/DeltaStatsUtils.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/io/delta/tables/ClickhouseTable.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/DeltaAdapter.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/DeltaLog.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/Snapshot.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/DeleteCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/UpdateCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/VacuumCommand.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala (100%) rename backends-clickhouse/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala (100%) rename backends-clickhouse/{src/test/delta-32 => src-delta-32/test/scala}/org/apache/spark/gluten/delta/DeltaStatsUtils.scala (100%) rename backends-clickhouse/{src/main-iceberg/resource => src-iceberg/main/resources}/META-INF/services/org.apache.gluten.component.Component (100%) rename backends-clickhouse/{src/main-iceberg => src-iceberg/main}/scala/org/apache/gluten/component/CHIcebergComponent.scala (100%) rename backends-clickhouse/{src/test-iceberg => src-iceberg/test}/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala (100%) rename backends-velox/{src/test-delta => src-delta/test}/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala (100%) rename backends-velox/{src/test-delta => src-delta/test}/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala (100%) rename backends-velox/{src/main-iceberg/resource => src-iceberg/main/resources}/META-INF/services/org.apache.gluten.component.Component (100%) rename backends-velox/{src/main-iceberg => src-iceberg/main}/scala/org/apache/gluten/component/VeloxIcebergComponent.scala (100%) rename backends-velox/{src/test-iceberg => src-iceberg/test}/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala (100%) rename backends-velox/{src/test-iceberg => src-iceberg/test}/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala (100%) rename gluten-delta/{src => src-delta-20}/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala (100%) rename gluten-delta/{src => src-delta-20}/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala (100%) create mode 100644 gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala create mode 100644 gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala create mode 100644 gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala create mode 100644 gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala rename gluten-delta/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/execution/DeltaFilterExecTransformer.scala (100%) rename gluten-delta/{src/main/delta-32 => src-delta-32/main/scala}/org/apache/gluten/execution/DeltaProjectExecTransformer.scala (100%) rename gluten-delta/{src => src-delta}/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister (100%) rename gluten-delta/{src => src-delta}/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules (100%) rename gluten-delta/{src => src-delta}/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala (100%) rename gluten-delta/{src => src-delta}/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala (100%) rename gluten-delta/{src => src-delta}/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala (100%) rename gluten-delta/{src => src-delta}/test/scala/org/apache/gluten/execution/DeltaSuite.scala (99%) rename gluten-hudi/{src => src-hudi}/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister (100%) rename gluten-hudi/{src => src-hudi}/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala (100%) rename gluten-hudi/{src => src-hudi}/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala (100%) rename gluten-hudi/{src => src-hudi}/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala (100%) rename gluten-hudi/{src => src-hudi}/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java (100%) rename gluten-iceberg/{src => src-iceberg}/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala (100%) rename gluten-iceberg/{src => src-iceberg}/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala (100%) rename gluten-iceberg/{src => src-iceberg}/test/scala/org/apache/gluten/execution/IcebergSuite.scala (100%) diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index a6042fd1cce5..9a606c48e694 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -38,47 +38,6 @@ test - - - - ${project.basedir}/src/main-iceberg/resource - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-iceberg-sources - generate-sources - - add-source - - - - ${project.basedir}/src/main-iceberg/scala - ${project.basedir}/src/main-iceberg/java - - - - - add-iceberg-test-sources - generate-test-sources - - add-test-source - - - - ${project.basedir}/src/test-iceberg/scala - ${project.basedir}/src/test-iceberg/java - - - - - - - @@ -323,11 +282,6 @@ - - - ${project.basedir}/src/main/resources - - target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes @@ -353,6 +307,27 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-delta-profile + + enforce + + + + + + delta + "-P delta" must be set when building Gluten with ClickHouse backend. + + + + + + org.apache.maven.plugins maven-resources-plugin @@ -436,15 +411,15 @@ src/main/scala/**/*.scala src/test/scala/**/*.scala - src/main/delta-${delta.binary.version}/**/*.scala - src/test/delta-${delta.binary.version}/**/*.scala + src-delta-${delta.binary.version}/main/delta/**/*.scala + src-delta-${delta.binary.version}/test/delta/**/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/merge/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/stats/*.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/DeltaLog.scala - src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/Snapshot.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/commands/*.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/commands/merge/*.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/stats/*.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/DeltaLog.scala + src-delta-${delta.binary.version}/main/delta/org/apache/spark/sql/delta/Snapshot.scala @@ -484,36 +459,6 @@ - - org.codehaus.mojo - build-helper-maven-plugin - - - add-sources - generate-sources - - add-source - - - - src/main/delta-${delta.binary.version} - - - - - add-test-sources - generate-test-sources - - add-test-source - - - - src/test/delta-${delta.binary.version} - - - - - org.antlr antlr4-maven-plugin diff --git a/backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta-20/main/scala/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src-delta-20/main/scala/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/Snapshot.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 100% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src-delta-20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala diff --git a/backends-clickhouse/src/test/delta-20/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta-20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala similarity index 100% rename from backends-clickhouse/src/test/delta-20/org/apache/spark/gluten/delta/DeltaStatsUtils.scala rename to backends-clickhouse/src-delta-20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala diff --git a/backends-clickhouse/src/main/delta-23/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta-23/main/scala/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src-delta-23/main/scala/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaAdapter.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/DeltaLog.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/Snapshot.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/delta/stats/PrepareDeltaScan.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 100% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src-delta-23/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala diff --git a/backends-clickhouse/src/test/delta-23/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta-23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala similarity index 100% rename from backends-clickhouse/src/test/delta-23/org/apache/spark/gluten/delta/DeltaStatsUtils.scala rename to backends-clickhouse/src-delta-23/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala diff --git a/backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta-32/main/scala/io/delta/tables/ClickhouseTable.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/io/delta/tables/ClickhouseTable.scala rename to backends-clickhouse/src-delta-32/main/scala/io/delta/tables/ClickhouseTable.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaAdapter.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaLog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/DeltaLog.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/Snapshot.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/Snapshot.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/DeleteCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/UpdateCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/merge/ClassicMergeExecutor.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/FileDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/MergeTreeDeltaColumnarWrite.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala similarity index 100% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala diff --git a/backends-clickhouse/src/test/delta-32/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta-32/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala similarity index 100% rename from backends-clickhouse/src/test/delta-32/org/apache/spark/gluten/delta/DeltaStatsUtils.scala rename to backends-clickhouse/src-delta-32/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala diff --git a/backends-clickhouse/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component b/backends-clickhouse/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component similarity index 100% rename from backends-clickhouse/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component rename to backends-clickhouse/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component diff --git a/backends-clickhouse/src/main-iceberg/scala/org/apache/gluten/component/CHIcebergComponent.scala b/backends-clickhouse/src-iceberg/main/scala/org/apache/gluten/component/CHIcebergComponent.scala similarity index 100% rename from backends-clickhouse/src/main-iceberg/scala/org/apache/gluten/component/CHIcebergComponent.scala rename to backends-clickhouse/src-iceberg/main/scala/org/apache/gluten/component/CHIcebergComponent.scala diff --git a/backends-clickhouse/src/test-iceberg/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala similarity index 100% rename from backends-clickhouse/src/test-iceberg/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala rename to backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index 2719cc655943..48a044a17f2f 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -59,49 +59,7 @@ test - - - - ${project.basedir}/src/main-iceberg/resource - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-iceberg-sources - generate-sources - - add-source - - - - ${project.basedir}/src/main-iceberg/scala - ${project.basedir}/src/main-iceberg/java - - - - - add-iceberg-test-sources - generate-test-sources - - add-test-source - - - - ${project.basedir}/src/test-iceberg/scala - ${project.basedir}/src/test-iceberg/java - - - - - - - - delta @@ -123,42 +81,6 @@ provided - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-delta-sources - generate-sources - - add-source - - - - ${project.basedir}/src/main-delta/scala - ${project.basedir}/src/main-delta/java - - - - - add-delta-test-sources - generate-test-sources - - add-test-source - - - - ${project.basedir}/src/test-delta/scala - ${project.basedir}/src/test-delta/java - - - - - - - @@ -317,11 +239,11 @@ - ${cpp.releases.dir} - ${platform}/${arch} + ${project.basedir}/src/main/resources - ${project.basedir}/src/main/resources + ${cpp.releases.dir} + ${platform}/${arch} target/scala-${scala.binary.version}/classes diff --git a/backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala b/backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala similarity index 100% rename from backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala rename to backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala diff --git a/backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala b/backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala similarity index 100% rename from backends-velox/src/test-delta/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala rename to backends-velox/src-delta/test/scala/org/apache/gluten/execution/VeloxTPCHDeltaSuite.scala diff --git a/backends-velox/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component b/backends-velox/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component similarity index 100% rename from backends-velox/src/main-iceberg/resource/META-INF/services/org.apache.gluten.component.Component rename to backends-velox/src-iceberg/main/resources/META-INF/services/org.apache.gluten.component.Component diff --git a/backends-velox/src/main-iceberg/scala/org/apache/gluten/component/VeloxIcebergComponent.scala b/backends-velox/src-iceberg/main/scala/org/apache/gluten/component/VeloxIcebergComponent.scala similarity index 100% rename from backends-velox/src/main-iceberg/scala/org/apache/gluten/component/VeloxIcebergComponent.scala rename to backends-velox/src-iceberg/main/scala/org/apache/gluten/component/VeloxIcebergComponent.scala diff --git a/backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala similarity index 100% rename from backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala rename to backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala diff --git a/backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala similarity index 100% rename from backends-velox/src/test-iceberg/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala rename to backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 4b28f23dd9e3..4b27cc75bd70 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -94,7 +94,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { val glutenBuildInfo = new mutable.LinkedHashMap[String, String]() val components = Component.sorted() - glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(",")) + glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(", ")) components.foreach { comp => val buildInfo = comp.buildInfo() diff --git a/gluten-delta/pom.xml b/gluten-delta/pom.xml index f971ec607059..cf2fdd7518fa 100755 --- a/gluten-delta/pom.xml +++ b/gluten-delta/pom.xml @@ -14,10 +14,6 @@ jar Gluten DeltaLake - - ${project.basedir}/src/main/resources - - org.apache.gluten @@ -92,11 +88,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${resource.dir} - - net.alchim31.maven @@ -123,10 +114,10 @@ ${project.basedir}/../.scalafmt.conf - src/main/scala/**/*.scala - src/test/scala/**/*.scala - src/main/delta-${delta.binary.version}/**/*.scala - src/test/delta-${delta.binary.version}/**/*.scala + src-delta/main/scala/**/*.scala + src-delta/test/scala/**/*.scala + src-delta-${delta.binary.version}/main/scala/**/*.scala + src-delta-${delta.binary.version}/test/scala/**/*.scala @@ -151,28 +142,6 @@ - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resources - generate-sources - - copy-resources - - - src/main/scala/org/apache/gluten/execution - - - src/main/delta-${delta.binary.version}/org/apache/gluten/execution - - - true - - - - diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala rename to gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala rename to gluten-delta/src-delta-20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala diff --git a/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala new file mode 100644 index 000000000000..ca4665c0d0cb --- /dev/null +++ b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan) + extends FilterExecTransformerBase(condition, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala new file mode 100644 index 000000000000..9b720b19c5ba --- /dev/null +++ b/gluten-delta/src-delta-23/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.NamedExpression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan) + extends ProjectExecTransformerBase(projectList, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala new file mode 100644 index 000000000000..ca4665c0d0cb --- /dev/null +++ b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan) + extends FilterExecTransformerBase(condition, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala new file mode 100644 index 000000000000..9b720b19c5ba --- /dev/null +++ b/gluten-delta/src-delta-24/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.execution + +import org.apache.spark.sql.catalyst.expressions.NamedExpression +import org.apache.spark.sql.execution.SparkPlan + +case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan) + extends ProjectExecTransformerBase(projectList, child) { + + override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer = + copy(child = newChild) +} diff --git a/gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaFilterExecTransformer.scala rename to gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala diff --git a/gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala similarity index 100% rename from gluten-delta/src/main/delta-32/org/apache/gluten/execution/DeltaProjectExecTransformer.scala rename to gluten-delta/src-delta-32/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala diff --git a/gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister b/gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister similarity index 100% rename from gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister rename to gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister diff --git a/gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules b/gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules similarity index 100% rename from gluten-delta/src/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules rename to gluten-delta/src-delta/main/resources/META-INF/services/org.apache.gluten.extension.columnar.RewriteTransformerRules diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala b/gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala rename to gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformer.scala diff --git a/gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala b/gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala rename to gluten-delta/src-delta/main/scala/org/apache/gluten/execution/DeltaScanTransformerProvider.scala diff --git a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala b/gluten-delta/src-delta/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala similarity index 100% rename from gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala rename to gluten-delta/src-delta/main/scala/org/apache/gluten/extension/DeltaRewriteTransformerRules.scala diff --git a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/DeltaSuite.scala similarity index 99% rename from gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala rename to gluten-delta/src-delta/test/scala/org/apache/gluten/execution/DeltaSuite.scala index caf86189a2d4..df788bec99ad 100644 --- a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala +++ b/gluten-delta/src-delta/test/scala/org/apache/gluten/execution/DeltaSuite.scala @@ -18,7 +18,7 @@ package org.apache.gluten.execution import org.apache.spark.SparkConf import org.apache.spark.sql.Row -import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} +import org.apache.spark.sql.types._ import scala.collection.JavaConverters._ diff --git a/gluten-hudi/pom.xml b/gluten-hudi/pom.xml index b65217ea5b98..7900182f853a 100755 --- a/gluten-hudi/pom.xml +++ b/gluten-hudi/pom.xml @@ -14,10 +14,6 @@ jar Gluten Hudi - - ${project.basedir}/src/main/resources - - org.apache.gluten @@ -107,11 +103,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${resource.dir} - - net.alchim31.maven diff --git a/gluten-hudi/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister b/gluten-hudi/src-hudi/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister similarity index 100% rename from gluten-hudi/src/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister rename to gluten-hudi/src-hudi/main/resources/META-INF/services/org.apache.gluten.execution.DataSourceScanTransformerRegister diff --git a/gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala b/gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala similarity index 100% rename from gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala rename to gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformer.scala diff --git a/gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala b/gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala similarity index 100% rename from gluten-hudi/src/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala rename to gluten-hudi/src-hudi/main/scala/org/apache/gluten/execution/HudiScanTransformerProvider.scala diff --git a/gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala similarity index 100% rename from gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala rename to gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxHudiSuite.scala diff --git a/gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala b/gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala similarity index 100% rename from gluten-hudi/src/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala rename to gluten-hudi/src-hudi/test/scala/org/apache/gluten/execution/VeloxTPCHHudiSuite.scala diff --git a/gluten-iceberg/pom.xml b/gluten-iceberg/pom.xml index bad2ec20e8e8..c8c811674a54 100644 --- a/gluten-iceberg/pom.xml +++ b/gluten-iceberg/pom.xml @@ -14,10 +14,6 @@ jar Gluten Iceberg - - ${project.basedir}/src/main/resources - - org.apache.gluten @@ -99,11 +95,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${resource.dir} - - net.alchim31.maven diff --git a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java similarity index 100% rename from gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java rename to gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesBuilder.java diff --git a/gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java similarity index 100% rename from gluten-iceberg/src/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java rename to gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala diff --git a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/gluten/execution/OffloadIcebergScan.scala diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala similarity index 100% rename from gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala rename to gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/TypeUtil.scala diff --git a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala similarity index 100% rename from gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala rename to gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala diff --git a/gluten-substrait/pom.xml b/gluten-substrait/pom.xml index e8eb6a57e6c4..8a790aa15337 100644 --- a/gluten-substrait/pom.xml +++ b/gluten-substrait/pom.xml @@ -181,13 +181,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - true - - ${project.build.directory}/extra-resources - - org.apache.maven.plugins @@ -205,7 +198,7 @@ - + diff --git a/package/pom.xml b/package/pom.xml index f4cc8d6f7015..e0620e5cf5e1 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -253,7 +253,6 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.3.0 org.codehaus.mojo diff --git a/pom.xml b/pom.xml index 9b27a6839712..3c59b4f19e11 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ - 3.2.0 + 3.6.0 4.8.0 3.12.1 3.2.2 @@ -314,7 +314,7 @@ 2.3.0 23 4.8 - 0.15.0 + 0.15.0 @@ -440,6 +440,82 @@ gluten-delta + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-delta-sources + generate-sources + + add-source + + + + ${project.basedir}/src-delta/main/scala + ${project.basedir}/src-delta/main/java + ${project.basedir}/src-delta-${delta.binary.version}/main/scala + ${project.basedir}/src-delta-${delta.binary.version}/main/java + + + + + add-delta-resources + generate-resources + + add-resource + + + + + + + ${project.basedir}/src-delta/main/resources + + + ${project.basedir}/src-delta-${delta.binary.version}/main/resources + + + + + + add-delta-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/src-delta/test/scala + ${project.basedir}/src-delta/test/java + ${project.basedir}/src-delta-${delta.binary.version}/test/scala + ${project.basedir}/src-delta-${delta.binary.version}/test/java + + + + + add-delta-test-resources + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/src-delta/test/resources + + + ${project.basedir}/src-delta-${delta.binary.version}/test/resources + + + + + + + + iceberg @@ -449,6 +525,70 @@ gluten-iceberg + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-iceberg-sources + generate-sources + + add-source + + + + ${project.basedir}/src-iceberg/main/scala + ${project.basedir}/src-iceberg/main/java + + + + + add-iceberg-resources + generate-resources + + add-resource + + + + + ${project.basedir}/src-iceberg/main/resources + + + + + + add-iceberg-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/src-iceberg/test/scala + ${project.basedir}/src-iceberg/test/java + + + + + add-iceberg-test-resources + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/src-iceberg/test/resources + + + + + + + + hudi @@ -458,6 +598,70 @@ gluten-hudi + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-hudi-sources + generate-sources + + add-source + + + + ${project.basedir}/src-hudi/main/scala + ${project.basedir}/src-hudi/main/java + + + + + add-hudi-resources + generate-resources + + add-resource + + + + + ${project.basedir}/src-hudi/main/resources + + + + + + add-hudi-test-sources + generate-test-sources + + add-test-source + + + + ${project.basedir}/src-hudi/test/scala + ${project.basedir}/src-hudi/test/java + + + + + add-hudi-test-resources + generate-test-resources + + add-test-resource + + + + + ${project.basedir}/src-hudi/test/resources + + + + + + + + backends-velox @@ -877,6 +1081,16 @@ + + + ${project.basedir}/src/main/resources + + + + + ${project.basedir}/src/test/resources + + @@ -1133,13 +1347,17 @@ antlr4-maven-plugin ${antlr4.version} + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + org.apache.maven.plugins maven-enforcer-plugin - 1.0.1 enforce-versions @@ -1175,6 +1393,34 @@ versions-maven-plugin 2.15.0 + + org.codehaus.mojo + build-helper-maven-plugin + + + + + + + + + + add-generated-resources + generate-resources + + add-resource + + + + + + ${project.build.directory}/generated-resources + + + + + +