From 2f0f4555dcd7c1c613e801a4a819f5c1766afb2e Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh <viirya@gmail.com> Date: Mon, 29 Jul 2024 13:53:49 -0700 Subject: [PATCH] Fix --- dev/diffs/3.5.1.diff | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/dev/diffs/3.5.1.diff b/dev/diffs/3.5.1.diff index 9863713752..aa6fcd4b68 100644 --- a/dev/diffs/3.5.1.diff +++ b/dev/diffs/3.5.1.diff @@ -342,7 +342,7 @@ index c2fe31520ac..0f54b233d14 100644 assert(exchanges.size == 2) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala -index f33432ddb6f..9cf7a9dd4e3 100644 +index f33432ddb6f..aef20714168 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala @@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen @@ -393,7 +393,17 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { withTable("fact", "dim") { spark.range(100).select( -@@ -1187,7 +1194,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1027,7 +1034,8 @@ abstract class DynamicPartitionPruningSuiteBase + } + } + +- test("avoid reordering broadcast join keys to match input hash partitioning") { ++ test("avoid reordering broadcast join keys to match input hash partitioning", ++ IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) { + withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { + withTable("large", "dimTwo", "dimThree") { +@@ -1187,7 +1195,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -403,7 +413,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { val df = sql( """ -@@ -1238,7 +1246,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1238,7 +1247,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -413,7 +423,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 Given("dynamic pruning filter on the build side") withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { val df = sql( -@@ -1311,7 +1320,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1311,7 +1321,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -423,7 +433,17 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf( SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true", -@@ -1470,7 +1480,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1423,7 +1434,8 @@ abstract class DynamicPartitionPruningSuiteBase + } + } + +- test("SPARK-34637: DPP side broadcast query stage is created firstly") { ++ test("SPARK-34637: DPP side broadcast query stage is created firstly", ++ IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) { + withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { + val df = sql( + """ WITH v as ( +@@ -1470,7 +1482,8 @@ abstract class DynamicPartitionPruningSuiteBase checkAnswer(df, Row(3, 2) :: Row(3, 2) :: Row(3, 2) :: Row(3, 2) :: Nil) } @@ -433,7 +453,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { val df = sql( """ -@@ -1485,7 +1496,7 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1485,7 +1498,7 @@ abstract class DynamicPartitionPruningSuiteBase } test("SPARK-38148: Do not add dynamic partition pruning if there exists static partition " + @@ -442,7 +462,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { Seq( "f.store_id = 1" -> false, -@@ -1557,7 +1568,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1557,7 +1570,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -452,7 +472,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withTable("duplicate_keys") { withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { Seq[(Int, String)]((1, "NL"), (1, "NL"), (3, "US"), (3, "US"), (3, "US")) -@@ -1588,7 +1600,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1588,7 +1602,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -462,7 +482,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { val df = sql( """ -@@ -1617,7 +1630,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1617,7 +1632,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -472,7 +492,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { val df = sql( """ -@@ -1729,6 +1743,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat +@@ -1729,6 +1745,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat case s: BatchScanExec => // we use f1 col for v2 tables due to schema pruning s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1")))