Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[VL] Add tests for Velox SMJ's coverage #7195

Merged
merged 2 commits into from
Sep 12, 2024
Merged

[VL] Add tests for Velox SMJ's coverage #7195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
edd9920
[VL] Add tests for Velox SMJ's coverage
zhztheplayer Sep 11, 2024
f19bf39
fixup
zhztheplayer Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.datasources.parquet._
import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite}
import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite}
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuiteForceShjOn, GlutenOuterJoinSuiteForceShjOn}
import org.apache.spark.sql.extension.{GlutenCustomerExtensionSuite, GlutenSessionExtensionSuite}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
Expand Down Expand Up @@ -1624,7 +1624,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("test composed unique condition (both non-equal) for left anti join using ShuffledHashJoin (whole-stage-codegen on)")
.exclude("test composed unique condition (both non-equal) for left anti join using SortMergeJoin (whole-stage-codegen off)")
.exclude("test composed unique condition (both non-equal) for left anti join using SortMergeJoin (whole-stage-codegen on)")
enableSuite[GlutenInnerJoinSuite]
enableSuite[GlutenInnerJoinSuiteForceShjOn]
.exclude(
"inner join, one match per row using ShuffledHashJoin (build=left) (whole-stage-codegen off)")
.exclude(
Expand Down Expand Up @@ -1673,7 +1673,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build left (whole-stage-codegen on)")
.exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build right (whole-stage-codegen off)")
.exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin build right (whole-stage-codegen on)")
enableSuite[GlutenOuterJoinSuite]
enableSuite[GlutenOuterJoinSuiteForceShjOn]
.exclude("basic left outer join using ShuffledHashJoin (whole-stage-codegen off)")
.exclude("basic left outer join using ShuffledHashJoin (whole-stage-codegen on)")
.exclude("basic left outer join using SortMergeJoin (whole-stage-codegen off)")
Expand Down
13 changes: 10 additions & 3 deletions gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import org.apache.spark.sql.execution.datasources.parquet._
import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite, GlutenTextV2Suite}
import org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite, GlutenFileTableSuite, GlutenV2PredicateSuite}
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite, GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
import org.apache.spark.sql.execution.joins._
import org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite, GlutenSessionExtensionSuite, TestFileSourceScanExecTransformer}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
Expand Down Expand Up @@ -879,8 +879,15 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("broadcast join where streamed side's output partitioning is HashPartitioning")

enableSuite[GlutenExistenceJoinSuite]
enableSuite[GlutenInnerJoinSuite]
enableSuite[GlutenOuterJoinSuite]
enableSuite[GlutenInnerJoinSuiteForceShjOn]
enableSuite[GlutenInnerJoinSuiteForceShjOff]
enableSuite[GlutenOuterJoinSuiteForceShjOn]
enableSuite[GlutenOuterJoinSuiteForceShjOff]
// Caused by Velox SMJ result mismatches with Spark.
.exclude("basic right outer join using SortMergeJoin (whole-stage-codegen off)")
.exclude("basic right outer join using SortMergeJoin (whole-stage-codegen on)")
.exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen off)")
.exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen on)")
enableSuite[FallbackStrategiesSuite]
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenLocalBroadcastExchangeSuite]
Expand Down
17 changes: 16 additions & 1 deletion ...ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,21 @@
*/
package org.apache.spark.sql.execution.joins

import org.apache.gluten.GlutenConfig

import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsBaseTrait

class GlutenInnerJoinSuite extends InnerJoinSuite with GlutenSQLTestsBaseTrait {}
class GlutenInnerJoinSuiteForceShjOn extends InnerJoinSuite with GlutenSQLTestsBaseTrait {
override def sparkConf: SparkConf = {
super.sparkConf
.set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "true")
}
}

class GlutenInnerJoinSuiteForceShjOff extends InnerJoinSuite with GlutenSQLTestsBaseTrait {
override def sparkConf: SparkConf = {
super.sparkConf
.set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false")
}
}
17 changes: 16 additions & 1 deletion ...ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,21 @@
*/
package org.apache.spark.sql.execution.joins

import org.apache.gluten.GlutenConfig

import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsBaseTrait

class GlutenOuterJoinSuite extends OuterJoinSuite with GlutenSQLTestsBaseTrait {}
class GlutenOuterJoinSuiteForceShjOn extends OuterJoinSuite with GlutenSQLTestsBaseTrait {
override def sparkConf: SparkConf = {
super.sparkConf
.set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "true")
}
}

class GlutenOuterJoinSuiteForceShjOff extends OuterJoinSuite with GlutenSQLTestsBaseTrait {
override def sparkConf: SparkConf = {
super.sparkConf
.set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false")
}
}
Loading