Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[VL] Enable right and anti join in smj #6449

Merged
merged 1 commit into from
Jul 15, 2024
Merged

[VL] Enable right and anti join in smj #6449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1442,6 +1442,68 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}

test("test sort merge join") {
withTable("t1", "t2") {
sql("""
|create table t1 using parquet as
|select cast(id as int) as c1, cast(id as string) c2 from range(100)
|""".stripMargin)
sql("""
|create table t2 using parquet as
|select cast(id as int) as c1, cast(id as string) c2 from range(100) order by c1 desc;
|""".stripMargin)
withSQLConf("spark.gluten.sql.columnar.forceShuffledHashJoin" -> "false") {
runQueryAndCompare(
"""
|select * from t1 inner join t2 on t1.c1 = t2.c1 and t1.c1 > 50;
|""".stripMargin
) {
checkGlutenOperatorMatch[SortMergeJoinExecTransformer]
}
}

withSQLConf("spark.gluten.sql.columnar.forceShuffledHashJoin" -> "false") {
runQueryAndCompare(
"""
|select * from t1 left join t2 on t1.c1 = t2.c1 and t1.c1 > 50;
|""".stripMargin
) {
checkGlutenOperatorMatch[SortMergeJoinExecTransformer]
}
}

withSQLConf("spark.gluten.sql.columnar.forceShuffledHashJoin" -> "false") {
runQueryAndCompare(
"""
|select * from t1 left semi join t2 on t1.c1 = t2.c1 and t1.c1 > 50;
|""".stripMargin
) {
checkGlutenOperatorMatch[SortMergeJoinExecTransformer]
}
}

withSQLConf("spark.gluten.sql.columnar.forceShuffledHashJoin" -> "false") {
runQueryAndCompare(
"""
|select * from t1 right join t2 on t1.c1 = t2.c1 and t1.c1 > 50;
|""".stripMargin
) {
checkGlutenOperatorMatch[SortMergeJoinExecTransformer]
}
}

withSQLConf("spark.gluten.sql.columnar.forceShuffledHashJoin" -> "false") {
runQueryAndCompare(
"""
|select * from t1 left anti join t2 on t1.c1 = t2.c1 and t1.c1 > 50;
|""".stripMargin
) {
checkGlutenOperatorMatch[SortMergeJoinExecTransformer]
}
}
}
}

test("Fix incorrect path by decode") {
val c = "?.+<_>|/"
val path = rootPath + "/test +?.+<_>|"
Expand Down
4 changes: 3 additions & 1 deletion cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -906,11 +906,13 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::JoinRel& joinRel
switch (joinRel.type()) {
case ::substrait::JoinRel_JoinType_JOIN_TYPE_INNER:
case ::substrait::JoinRel_JoinType_JOIN_TYPE_LEFT:
case ::substrait::JoinRel_JoinType_JOIN_TYPE_RIGHT:
case ::substrait::JoinRel_JoinType_JOIN_TYPE_LEFT_SEMI:
case ::substrait::JoinRel_JoinType_JOIN_TYPE_RIGHT_SEMI:
case ::substrait::JoinRel_JoinType_JOIN_TYPE_ANTI:
break;
default:
LOG_VALIDATION_MSG("Sort merge join only support inner, left, left semi and right semi join.");
LOG_VALIDATION_MSG("Sort merge join type is not supported: " + std::to_string(joinRel.type()));
return false;
}
}
Expand Down
Loading