From 6a9b55b6dc4fcd10c2fdfa009d2176d7d648cd54 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 23 Jul 2024 17:48:08 -0700 Subject: [PATCH] Fix --- dev/diffs/3.4.3.diff | 46 +++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 3c88d6a7ee..ee6e598a37 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -1,5 +1,5 @@ diff --git a/pom.xml b/pom.xml -index d3544881af1..47382e29b5a 100644 +index d3544881af1..7ab326110b0 100644 --- a/pom.xml +++ b/pom.xml @@ -148,6 +148,8 @@ @@ -352,7 +352,7 @@ index daef11ae4d6..9f3cc9181f2 100644 assert(exchanges.size == 2) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala -index f33432ddb6f..9cf7a9dd4e3 100644 +index f33432ddb6f..aef20714168 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala @@ -22,6 +22,7 @@ import org.scalatest.GivenWhenThen @@ -403,7 +403,17 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { withTable("fact", "dim") { spark.range(100).select( -@@ -1187,7 +1194,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1027,7 +1034,8 @@ abstract class DynamicPartitionPruningSuiteBase + } + } + +- test("avoid reordering broadcast join keys to match input hash partitioning") { ++ test("avoid reordering broadcast join keys to match input hash partitioning", ++ IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) { + withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { + withTable("large", "dimTwo", "dimThree") { +@@ -1187,7 +1195,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -413,7 +423,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { val df = sql( """ -@@ -1238,7 +1246,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1238,7 +1247,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -423,7 +433,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 Given("dynamic pruning filter on the build side") withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { val df = sql( -@@ -1311,7 +1320,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1311,7 +1321,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -433,7 +443,17 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf( SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true", SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true", -@@ -1470,7 +1480,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1423,7 +1434,8 @@ abstract class DynamicPartitionPruningSuiteBase + } + } + +- test("SPARK-34637: DPP side broadcast query stage is created firstly") { ++ test("SPARK-34637: DPP side broadcast query stage is created firstly", ++ IgnoreComet("TODO: Support SubqueryBroadcastExec in Comet: #242")) { + withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") { + val df = sql( + """ WITH v as ( +@@ -1470,7 +1482,8 @@ abstract class DynamicPartitionPruningSuiteBase checkAnswer(df, Row(3, 2) :: Row(3, 2) :: Row(3, 2) :: Row(3, 2) :: Nil) } @@ -443,7 +463,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { val df = sql( """ -@@ -1485,7 +1496,7 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1485,7 +1498,7 @@ abstract class DynamicPartitionPruningSuiteBase } test("SPARK-38148: Do not add dynamic partition pruning if there exists static partition " + @@ -452,7 +472,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { Seq( "f.store_id = 1" -> false, -@@ -1557,7 +1568,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1557,7 +1570,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -462,7 +482,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withTable("duplicate_keys") { withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { Seq[(Int, String)]((1, "NL"), (1, "NL"), (3, "US"), (3, "US"), (3, "US")) -@@ -1588,7 +1600,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1588,7 +1602,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -472,7 +492,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { val df = sql( """ -@@ -1617,7 +1630,8 @@ abstract class DynamicPartitionPruningSuiteBase +@@ -1617,7 +1632,8 @@ abstract class DynamicPartitionPruningSuiteBase } } @@ -482,7 +502,7 @@ index f33432ddb6f..9cf7a9dd4e3 100644 withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") { val df = sql( """ -@@ -1729,6 +1743,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat +@@ -1729,6 +1745,8 @@ abstract class DynamicPartitionPruningV1Suite extends DynamicPartitionPruningDat case s: BatchScanExec => // we use f1 col for v2 tables due to schema pruning s.output.exists(_.exists(_.argString(maxFields = 100).contains("f1"))) @@ -2430,7 +2450,7 @@ index abe606ad9c1..2d930b64cca 100644 val tblTargetName = "tbl_target" val tblSourceQualified = s"default.$tblSourceName" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala -index dd55fcfe42c..293e9dc2986 100644 +index dd55fcfe42c..e7fcd0a9e6a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest @@ -2579,7 +2599,7 @@ index 1966e1e64fd..cde97a0aafe 100644 spark.sql( """ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 07361cfdce9..25b0dc3ef7e 100644 +index 07361cfdce9..6673c141c9a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -55,25 +55,53 @@ object TestHive