[GLUTEN-3559][VL] Fix UTs failing in spark 3.4 which have assertions …

…for SMJ node (#4109)
apache · Dec 27, 2023 · 1e382bd · 1e382bd
1 parent 73351b9
commit 1e382bd
Show file tree

Hide file tree

Showing 3 changed files with 690 additions and 33 deletions.
diff --git a/gluten-ut/spark34/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala
@@ -68,44 +68,14 @@ class VeloxTestSettings extends BackendTestSettings {
     .exclude("partitioned join: number of buckets mismatch should trigger shuffle")
     .exclude("partitioned join: only one side reports partitioning")
     .exclude("partitioned join: join with two partition keys and different # of partition keys")
+    // disable as both checks for SMJ node
     .excludeByPrefix("SPARK-41413: partitioned join:")
-    .exclude("SPARK-42038: partially clustered: with different partition keys and both sides partially clustered")
-    .exclude("SPARK-42038: partially clustered: with different partition keys and missing keys on left-hand side")
-    .exclude("SPARK-42038: partially clustered: with different partition keys and missing keys on right-hand side")
-    .exclude("SPARK-42038: partially clustered: left outer join")
-    .exclude("SPARK-42038: partially clustered: right outer join")
-    .exclude("SPARK-42038: partially clustered: full outer join is not applicable")
-    .exclude("SPARK-42038: partially clustered: with dynamic partition filtering")
+    .excludeByPrefix("SPARK-42038: partially clustered:")
   enableSuite[GlutenLocalScanSuite]
   enableSuite[GlutenMetadataColumnSuite]
   enableSuite[GlutenSupportsCatalogOptionsSuite]
   enableSuite[GlutenTableCapabilityCheckSuite]
   enableSuite[GlutenWriteDistributionAndOrderingSuite]
-    .exclude("ordered distribution and sort with same exprs: append")
-    .exclude("ordered distribution and sort with same exprs: overwrite")
-    .exclude("ordered distribution and sort with same exprs: overwriteDynamic")
-    .exclude("clustered distribution and sort with same exprs: append")
-    .exclude("clustered distribution and sort with same exprs: overwrite")
-    .exclude("clustered distribution and sort with same exprs: overwriteDynamic")
-    .exclude("clustered distribution and sort with extended exprs: append")
-    .exclude("clustered distribution and sort with extended exprs: overwrite")
-    .exclude("clustered distribution and sort with extended exprs: overwriteDynamic")
-    .exclude("ordered distribution and sort with manual global sort: append")
-    .exclude("ordered distribution and sort with manual global sort: overwrite")
-    .exclude("ordered distribution and sort with manual global sort: overwriteDynamic")
-    .exclude("ordered distribution and sort with incompatible global sort: append")
-    .exclude("ordered distribution and sort with incompatible global sort: overwrite")
-    .exclude("ordered distribution and sort with incompatible global sort: overwriteDynamic")
-    .exclude("ordered distribution and sort with manual local sort: append")
-    .exclude("ordered distribution and sort with manual local sort: overwrite")
-    .exclude("ordered distribution and sort with manual local sort: overwriteDynamic")
-    .exclude("clustered distribution and local sort with manual global sort: append")
-    .exclude("clustered distribution and local sort with manual global sort: overwrite")
-    .exclude("clustered distribution and local sort with manual global sort: overwriteDynamic")
-    .exclude("clustered distribution and local sort with manual local sort: append")
-    .exclude("clustered distribution and local sort with manual local sort: overwrite")
-    .exclude("clustered distribution and local sort with manual local sort: overwriteDynamic")
-
   enableSuite[GlutenQueryCompilationErrorsDSv2Suite]
   enableSuite[GlutenQueryCompilationErrorsSuite]
   enableSuite[GlutenQueryExecutionErrorsSuite]
@@ -1123,6 +1093,7 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenJoinSuite]
     // exclude as it check spark plan
     .exclude("SPARK-36794: Ignore duplicated key when building relation for semi/anti hash join")
+    // exclude as it check for SMJ node
     .exclude(
       "SPARK-43113: Full outer join with duplicate stream-side references in condition (SMJ)")
   enableSuite[GlutenMathFunctionsSuite]

diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenJoinSuite.scala
@@ -16,6 +16,9 @@
  */
 package org.apache.spark.sql
 
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec
+
 class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait {
 
   override def testNameBlackList: Seq[String] = Seq(
@@ -52,4 +55,14 @@ class GlutenJoinSuite extends JoinSuite with GlutenSQLTestsTrait {
         |""".stripMargin
     checkAnswer(spark.sql(sql), Seq(Row(0, 1), Row(1, 2), Row(2, 3)))
   }
+
+  test(
+    GlutenTestConstants.GLUTEN_TEST + "SPARK-43113: Full outer join with duplicate stream-side" +
+      " references in condition (SHJ)") {
+    def check(plan: SparkPlan): Unit = {
+      assert(collect(plan) { case _: ShuffledHashJoinExec => true }.size === 1)
+    }
+
+    dupStreamSideColTest("MERGE", check)
+  }
 }